diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py index ce4e3edfc6be32..dff694ec4b180d 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py +++ b/docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py @@ -29,7 +29,8 @@ - __manifest Directory Namespace V2 manifest table (with its scalar indexes). - all_types.lance The pre-existing compatibility-mode root table, re-registered as-is. - The `doris` namespace with one indexed vector table per ANN algorithm (hash-prefixed - directories), listed in VECTOR_TABLES below. + directories), listed in VECTOR_TABLES below, plus NESTED_TABLE, a nested-field scalar + index fixture used by the SHOW INDEX / index-inspection suites. Every vector table holds identical deterministic data: 1024 rows in two 512-row fragments, 16-dim Float32 `embedding` where embedding[j] = (row_id - 1) + j. For a query equal to the @@ -136,6 +137,16 @@ }, } +# Nested-field scalar index fixture for #66497's SHOW INDEX / lance_index_entries suites. +# The indexed child name deliberately contains a dot so the canonical field path can only +# be written with backtick quoting; the table also carries a __lance_frag_reuse system +# entry (deferred-remap compaction) so every inspection surface proves it filters reserved +# system indexes instead of bricking on them. +NESTED_TABLE = "nested_index" +NESTED_INDEX_NAME = "nested_label_btree" +NESTED_COLUMN = "attributes.`child.with.dot`" +NESTED_ROWS = 16 + # The head query is exactly row 1's vector; the tail query is row 1024's. Only endpoint # vectors are used so that 16 * (n - r)^2 never ties between two different rows n. HEAD_QUERY = [float(j) for j in range(DIM)] @@ -218,10 +229,68 @@ def create_vector_table(namespace, table_name: str) -> str: return location +def make_nested_fragment_table(row_offset_start: int, row_offset_end: int) -> pa.Table: + offsets = list(range(row_offset_start, row_offset_end)) + attributes = pa.StructArray.from_arrays( + [ + pa.array(["even" if offset % 2 == 0 else "odd" for offset in offsets]), + pa.array([f"item-{offset + 1:04d}" for offset in offsets]), + ], + fields=[ + pa.field("source", pa.string(), nullable=False), + pa.field("child.with.dot", pa.string(), nullable=False), + ], + ) + table = pa.Table.from_arrays( + [ + pa.array([offset + 1 for offset in offsets], type=pa.int64()), + attributes, + ], + schema=pa.schema( + [ + pa.field("row_id", pa.int64(), nullable=False), + pa.field( + "attributes", + pa.struct(list(attributes.type)), + nullable=False, + ), + ] + ), + ) + return table + + +def create_nested_index_table(namespace) -> str: + first = make_nested_fragment_table(0, NESTED_ROWS // 2) + buffer = io.BytesIO() + with ipc.new_stream(buffer, first.schema) as writer: + writer.write_table(first) + response = namespace.create_table( + CreateTableRequest(id=[NAMESPACE, NESTED_TABLE]), buffer.getvalue() + ) + location = response.location + lance.write_dataset( + make_nested_fragment_table(NESTED_ROWS // 2, NESTED_ROWS), location, mode="append" + ) + dataset = lance.dataset(location) + # Same physical-dataset indexing detour as the vector tables: the Directory namespace + # does not implement create_table_index. The dotted child name only resolves with + # backtick quoting (NESTED_COLUMN); a plain dotted path raises KeyError. + dataset.create_scalar_index(NESTED_COLUMN, "BTREE", name=NESTED_INDEX_NAME) + # Deferred-remap compaction merges the two fragments but leaves a reserved + # __lance_frag_reuse system index entry behind, which is exactly what the inspection + # surfaces must learn to skip. + lance.dataset(location).optimize.compact_files( + target_rows_per_fragment=NESTED_ROWS, defer_index_remap=True + ) + return location + + def compact_manifest(root: Path) -> None: # Every namespace mutation above leaves a manifest fragment, index delta, and version # behind. Fold them together so the committed fixture stays small and reviewable. Only - # the manifest is compacted: the vector tables must keep exactly two fragments. + # the manifest is compacted: the vector tables keep exactly two fragments, and + # nested_index keeps its deferred-remap system entry. manifest = lance.dataset(str(root / MANIFEST_DIR)) manifest.optimize.compact_files() manifest.optimize.optimize_indices(num_indices_to_merge=len(manifest.list_indices())) @@ -262,6 +331,7 @@ def build(root: Path, all_types_source: Path) -> None: index_file_version="V3", **spec["params"], ) + create_nested_index_table(namespace) compact_manifest(root) @@ -431,10 +501,43 @@ def check_ef_discriminator(name: str, dataset, assert_it: bool) -> None: f"rows {[row for row, _ in wide]} (differs={differs}, asserted={assert_it})") +def check_nested_dataset(location: str): + dataset = lance.dataset(location) + assert dataset.count_rows() == NESTED_ROWS, f"{NESTED_TABLE}: expected {NESTED_ROWS} rows" + fragments = dataset.get_fragments() + assert len(fragments) == 1, f"{NESTED_TABLE}: deferred compaction must leave 1 fragment" + schema = dataset.schema + assert schema.field("row_id").type == pa.int64(), f"{NESTED_TABLE}: row_id type" + attributes = schema.field("attributes") + assert pa.types.is_struct(attributes.type), f"{NESTED_TABLE}: attributes type" + child_names = [field.name for field in attributes.type] + assert child_names == ["source", "child.with.dot"], ( + f"{NESTED_TABLE}: attributes children {child_names}" + ) + indices = {index["name"]: index["type"] for index in dataset.list_indices()} + assert indices.get(NESTED_INDEX_NAME) == "BTree", ( + f"{NESTED_TABLE}: missing BTREE {NESTED_INDEX_NAME}: {indices}" + ) + # The reserved system entry is part of the contract: the Doris FE must filter it out of + # SHOW INDEX and lance_index_entries instead of failing the whole read on it. + assert "__lance_frag_reuse" in indices, ( + f"{NESTED_TABLE}: deferred-remap compaction left no __lance_frag_reuse: {indices}" + ) + # The BTREE must be usable, not just present: one exact-match lookup through it. + probe = ( + dataset.scanner(filter="attributes.`child.with.dot` = 'item-0007'") + .to_table() + .column("row_id") + .to_pylist() + ) + assert probe == [7], f"{NESTED_TABLE}: BTREE probe returned {probe}" + + def check_catalog(root: Path) -> None: namespace = lance_namespace.connect("dir", {"root": str(root)}) tables = namespace.list_tables(ListTablesRequest(id=[NAMESPACE])) - assert sorted(tables.tables) == sorted(VECTOR_TABLES), ( + expected_tables = sorted(list(VECTOR_TABLES) + [NESTED_TABLE]) + assert sorted(tables.tables) == expected_tables, ( f"unexpected {NAMESPACE} tables: {tables.tables}" ) root_tables = namespace.list_tables(ListTablesRequest(id=[])) @@ -478,6 +581,11 @@ def check_catalog(root: Path) -> None: check_boundary_discriminator(table_name, dataset, search) if search.get("ef"): check_ef_discriminator(table_name, dataset, spec.get("ef_discriminator", False)) + + nested = namespace.describe_table(DescribeTableRequest(id=[NAMESPACE, NESTED_TABLE])) + nested_path = Path(nested.location.removeprefix("file://")) + assert nested_path.is_dir(), f"{NESTED_TABLE} location missing: {nested.location}" + check_nested_dataset(nested.location) print(f"self-check OK: {root}") diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_indices/fad74e02-a9fd-4978-90ce-02a8ebb8ff9f/page_data.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_indices/fad74e02-a9fd-4978-90ce-02a8ebb8ff9f/page_data.lance new file mode 100644 index 00000000000000..f0a39364b47338 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_indices/fad74e02-a9fd-4978-90ce-02a8ebb8ff9f/page_data.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_indices/fad74e02-a9fd-4978-90ce-02a8ebb8ff9f/page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_indices/fad74e02-a9fd-4978-90ce-02a8ebb8ff9f/page_lookup.lance new file mode 100644 index 00000000000000..ba8bf14d5a69ad Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_indices/fad74e02-a9fd-4978-90ce-02a8ebb8ff9f/page_lookup.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/0-efecb08b-3af4-4399-a7cb-9bec38a51c5f.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/0-efecb08b-3af4-4399-a7cb-9bec38a51c5f.txn new file mode 100644 index 00000000000000..39f6f1379f20ec Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/0-efecb08b-3af4-4399-a7cb-9bec38a51c5f.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/1-3c17e305-7969-454b-8e12-f9a51797f850.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/1-3c17e305-7969-454b-8e12-f9a51797f850.txn new file mode 100644 index 00000000000000..6b35c5302eeb7d Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/1-3c17e305-7969-454b-8e12-f9a51797f850.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/2-5aef3e99-fb51-4966-a688-2d7e295802af.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/2-5aef3e99-fb51-4966-a688-2d7e295802af.txn new file mode 100644 index 00000000000000..2ec4a10ef93042 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/2-5aef3e99-fb51-4966-a688-2d7e295802af.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/3-7c64f5fa-9002-4dfa-82ac-bad870689f62.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/3-7c64f5fa-9002-4dfa-82ac-bad870689f62.txn new file mode 100644 index 00000000000000..789f7319a99848 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/3-7c64f5fa-9002-4dfa-82ac-bad870689f62.txn @@ -0,0 +1 @@ +$7c64f5fa-9002-4dfa-82ac-bad870689f62� \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/3-9fcbc21e-71ca-49c8-a7f7-e1e5857fd5d3.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/3-9fcbc21e-71ca-49c8-a7f7-e1e5857fd5d3.txn new file mode 100644 index 00000000000000..6b7fb240974d69 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_transactions/3-9fcbc21e-71ca-49c8-a7f7-e1e5857fd5d3.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551610.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551610.manifest new file mode 100644 index 00000000000000..2d66b83fc366de Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551610.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551611.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551611.manifest new file mode 100644 index 00000000000000..af729526a0c7dd Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551611.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551612.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..f9cbdefd918d30 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551612.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551613.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000000..71c543f51d5cdd Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551613.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551614.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..84d77cf7ed295f Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/_versions/18446744073709551614.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/0010101011101010110011103112254514985be719bd5ce4f9.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/0010101011101010110011103112254514985be719bd5ce4f9.lance new file mode 100644 index 00000000000000..31db5bb789409a Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/0010101011101010110011103112254514985be719bd5ce4f9.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/111011110010000011000010a14eb14182bec19779539061b8.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/111011110010000011000010a14eb14182bec19779539061b8.lance new file mode 100644 index 00000000000000..946b10e08f1ec6 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/111011110010000011000010a14eb14182bec19779539061b8.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/111110010111011000000000c11d104b91b767d86cc357b1e1.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/111110010111011000000000c11d104b91b767d86cc357b1e1.lance new file mode 100644 index 00000000000000..2e41a140d6c1b4 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/0a7658e3_doris$nested_index/data/111110010111011000000000c11d104b91b767d86cc357b1e1.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/259cbf62-5bca-412b-a4d7-8bd6b75bd7f2/page_data.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/259cbf62-5bca-412b-a4d7-8bd6b75bd7f2/page_data.lance deleted file mode 100644 index 4e4757ae45c764..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/259cbf62-5bca-412b-a4d7-8bd6b75bd7f2/page_data.lance and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6b01082a-d357-401b-bd03-fb0234d72403/page_data.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6b01082a-d357-401b-bd03-fb0234d72403/page_data.lance new file mode 100644 index 00000000000000..5a56cefc88f5c4 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6b01082a-d357-401b-bd03-fb0234d72403/page_data.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/259cbf62-5bca-412b-a4d7-8bd6b75bd7f2/page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6b01082a-d357-401b-bd03-fb0234d72403/page_lookup.lance similarity index 92% rename from docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/259cbf62-5bca-412b-a4d7-8bd6b75bd7f2/page_lookup.lance rename to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6b01082a-d357-401b-bd03-fb0234d72403/page_lookup.lance index bc94ace1c187ac..b7581f7f8fab09 100644 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/259cbf62-5bca-412b-a4d7-8bd6b75bd7f2/page_lookup.lance and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/6b01082a-d357-401b-bd03-fb0234d72403/page_lookup.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/70cf4b86-c98e-41c8-99b3-9e275524bb75/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/70cf4b86-c98e-41c8-99b3-9e275524bb75/bitmap_page_lookup.lance deleted file mode 100644 index 228f9e617b256c..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/70cf4b86-c98e-41c8-99b3-9e275524bb75/bitmap_page_lookup.lance and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/8bd6fdb0-4402-415a-bf72-8c8b072e3030/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/9dba856e-3de7-4a7f-997a-e448d6ccfd88/bitmap_page_lookup.lance similarity index 67% rename from docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/8bd6fdb0-4402-415a-bf72-8c8b072e3030/bitmap_page_lookup.lance rename to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/9dba856e-3de7-4a7f-997a-e448d6ccfd88/bitmap_page_lookup.lance index d95205bbc07e03..5c8641c1812a3a 100644 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/8bd6fdb0-4402-415a-bf72-8c8b072e3030/bitmap_page_lookup.lance and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/9dba856e-3de7-4a7f-997a-e448d6ccfd88/bitmap_page_lookup.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/e0ff2c67-b56d-41fc-93a4-d766d04a0eb8/bitmap_page_lookup.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/e0ff2c67-b56d-41fc-93a4-d766d04a0eb8/bitmap_page_lookup.lance new file mode 100644 index 00000000000000..60ab136b785507 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_indices/e0ff2c67-b56d-41fc-93a4-d766d04a0eb8/bitmap_page_lookup.lance differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/34-68394369-30e4-4ffa-b61e-197dc759bdb5.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/34-68394369-30e4-4ffa-b61e-197dc759bdb5.txn deleted file mode 100644 index eb3eccf075d9e6..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/34-68394369-30e4-4ffa-b61e-197dc759bdb5.txn and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/39-1624f0d8-c076-4315-9073-a01c9971f8ab.txn b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/39-1624f0d8-c076-4315-9073-a01c9971f8ab.txn new file mode 100644 index 00000000000000..0a85c95df3a3a6 Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_transactions/39-1624f0d8-c076-4315-9073-a01c9971f8ab.txn differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551575.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551575.manifest new file mode 100644 index 00000000000000..e8571883f183da Binary files /dev/null and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551575.manifest differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551580.manifest b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551580.manifest deleted file mode 100644 index a0fc1bd3ea668d..00000000000000 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/18446744073709551580.manifest and /dev/null differ diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json index 6417926911c440..03390d66d786b1 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json +++ b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/_versions/latest_version_hint.json @@ -1 +1 @@ -{"version":35} \ No newline at end of file +{"version":40} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/10110010111110111000110083a5334199aea08c5eacc59581.lance b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/00101000111111010011101078d9df455aa280e225d9336e3f.lance similarity index 53% rename from docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/10110010111110111000110083a5334199aea08c5eacc59581.lance rename to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/00101000111111010011101078d9df455aa280e225d9336e3f.lance index 25730e8ea7a005..51cc624aa15bcb 100644 Binary files a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/10110010111110111000110083a5334199aea08c5eacc59581.lance and b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/__manifest/data/00101000111111010011101078d9df455aa280e225d9336e3f.lance differ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java index 226557e6a63a40..7629d68f6ccc3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.lance; import org.apache.doris.analysis.TableSnapshot; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.CatalogProperty; @@ -28,6 +29,7 @@ import org.apache.doris.datasource.property.metastore.LanceFileSystemMetastoreProperties; import org.apache.doris.datasource.property.metastore.LanceRestMetastoreProperties; +import com.google.common.annotations.VisibleForTesting; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.commons.lang3.StringUtils; @@ -43,6 +45,7 @@ import org.lance.namespace.model.ListTablesResponse; import org.lance.namespace.model.TableExistsRequest; +import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; @@ -73,6 +76,10 @@ public class LanceExternalCatalog extends ExternalCatalog { private static final String DATABASE_NAMESPACE_DELIMITER = "."; private static final int PAGE_SIZE = 1000; private static final long ALLOCATOR_LIMIT = 256L * 1024 * 1024; + private static final int MAX_PROVIDER_MESSAGE_BYTES = 1024; + private static final String[] RUNTIME_SENSITIVE_OPTION_KEYS = { + "aws_access_key_id", "aws_secret_access_key", "aws_session_token" + }; private transient LanceNamespace namespace; private transient BufferAllocator allocator; @@ -155,6 +162,17 @@ private AbstractLanceProperties getLanceProperties() { return (AbstractLanceProperties) catalogProperty.getMetastoreProperties(); } + /** + * Returns whether this catalog is configured to use the Lance REST namespace. + * + *

This deliberately reads only the normalized catalog properties and does not initialize + * the namespace. Callers can therefore reject unsupported REST operations before resolving a + * database or table, both of which may trigger remote metadata requests. + */ + public boolean isRestCatalogConfigured() { + return LANCE_REST.equals(getLanceProperties().getLanceCatalogType()); + } + @Override protected List listDatabaseNames() { makeSureInitialized(); @@ -284,22 +302,7 @@ public LanceTableMetadata loadTableMetadata(String dbName, String tableName, private LanceTableMetadata loadTableMetadata(String dbName, String tableName, Optional tableSnapshot, boolean loadIndexSegments) { makeSureInitialized(); - DescribeTableResponse table = describeTable(dbName, tableName); - if (Boolean.TRUE.equals(table.getManagedVersioning())) { - throw new UnsupportedOperationException( - "Lance managed versioning is not supported by the current BE reader"); - } - String datasetUri = StringUtils.firstNonBlank(table.getTableUri(), table.getLocation()); - if (datasetUri == null) { - throw new RuntimeException("Lance namespace returned no table URI for " + dbName + "." + tableName); - } - - Map storageOptions = new HashMap<>(javaStorageOptions); - if (table.getStorageOptions() != null) { - storageOptions.putAll(table.getStorageOptions()); - } - Map tableBackendStorageOptions = LanceStorageOptions.forBackend( - backendStorageOptions, table.getStorageOptions()); + ResolvedTableAccess tableAccess = resolveTableAccess(dbName, tableName); try { if (tableSnapshot.isPresent()) { TableSnapshot snapshot = tableSnapshot.get(); @@ -313,22 +316,90 @@ private LanceTableMetadata loadTableMetadata(String dbName, String tableName, "Cannot parse Lance FOR TIME AS OF value '" + snapshot.getValue() + "'"); } version = LanceSnapshotResolver.getVersionAtOrBefore( - datasetUri, storageOptions, timestamp, allocator); + tableAccess.datasetUri, tableAccess.javaStorageOptions, timestamp, allocator); } - return LanceMetadataLoader.loadVersion(datasetUri, storageOptions, - tableBackendStorageOptions, version, allocator); + return LanceMetadataLoader.loadVersion(tableAccess.datasetUri, tableAccess.javaStorageOptions, + tableAccess.backendStorageOptions, version, allocator); } return loadIndexSegments - ? LanceMetadataLoader.loadLatestWithIndexSegments(datasetUri, storageOptions, - tableBackendStorageOptions, allocator) - : LanceMetadataLoader.loadLatest(datasetUri, storageOptions, - tableBackendStorageOptions, allocator); + ? LanceMetadataLoader.loadLatestWithIndexSegments(tableAccess.datasetUri, + tableAccess.javaStorageOptions, tableAccess.backendStorageOptions, allocator) + : LanceMetadataLoader.loadLatest(tableAccess.datasetUri, tableAccess.javaStorageOptions, + tableAccess.backendStorageOptions, allocator); } catch (Exception e) { throw new RuntimeException("Failed to load Lance table metadata for " + dbName + "." + tableName + ": " + sanitizedRootCauseMessage(e), safeCause(e)); } } + public List loadTableIndexMetadata( + String dbName, String tableName) throws AnalysisException { + if (isRestCatalogConfigured()) { + throw new AnalysisException("SHOW INDEX is not supported for Lance REST catalogs"); + } + try { + makeSureInitialized(); + } catch (Exception e) { + throw indexMetadataLoadFailure(dbName, tableName, e, null, javaStorageOptions); + } + + ResolvedTableAccess tableAccess = null; + try { + // Keep Directory namespace resolution on the caller while it owns the catalog's + // shared namespace and allocator. Moving that shared owner into a timed task would + // let catalog close release it after the caller returns but before the task ends. + // The deadline below covers the Dataset/JNI index metadata read itself. + tableAccess = resolveTableAccess(dbName, tableName); + String datasetUri = tableAccess.datasetUri; + Map storageOptions = tableAccess.javaStorageOptions; + return LanceMetadataReadExecutor.execute(() -> { + // The caller may return on deadline while JNI is still running. A task-owned + // allocator prevents catalog close from releasing native resources prematurely. + try (BufferAllocator readAllocator = new RootAllocator(ALLOCATOR_LIMIT)) { + return LanceIndexMetadataLoader.load(datasetUri, storageOptions, readAllocator); + } + }); + } catch (Exception e) { + String datasetUri = tableAccess == null ? null : tableAccess.datasetUri; + Map runtimeStorageOptions = tableAccess == null + ? javaStorageOptions : tableAccess.javaStorageOptions; + throw indexMetadataLoadFailure( + dbName, tableName, e, datasetUri, runtimeStorageOptions); + } + } + + @VisibleForTesting + RuntimeException indexMetadataLoadFailure(String dbName, String tableName, + Throwable throwable, String datasetUri, Map runtimeStorageOptions) { + String sanitizedMessage = sanitizedRootCauseMessage( + throwable, datasetUri, runtimeStorageOptions); + Throwable sanitizedCause = throwable instanceof IllegalArgumentException + ? new IllegalArgumentException(sanitizedMessage) + : new RuntimeException(sanitizedMessage); + return new RuntimeException("Failed to load Lance index metadata for " + dbName + "." + tableName + + ": " + sanitizedMessage, sanitizedCause); + } + + private ResolvedTableAccess resolveTableAccess(String dbName, String tableName) { + DescribeTableResponse table = describeTable(dbName, tableName); + if (Boolean.TRUE.equals(table.getManagedVersioning())) { + throw new UnsupportedOperationException( + "Lance managed versioning is not supported by the current BE reader"); + } + String datasetUri = StringUtils.firstNonBlank(table.getTableUri(), table.getLocation()); + if (datasetUri == null) { + throw new RuntimeException("Lance namespace returned no table URI for " + dbName + "." + tableName); + } + + Map tableJavaStorageOptions = new HashMap<>(javaStorageOptions); + if (table.getStorageOptions() != null) { + tableJavaStorageOptions.putAll(table.getStorageOptions()); + } + Map tableBackendStorageOptions = LanceStorageOptions.forBackend( + backendStorageOptions, table.getStorageOptions()); + return new ResolvedTableAccess(datasetUri, tableJavaStorageOptions, tableBackendStorageOptions); + } + private DescribeTableResponse describeTable(String dbName, String tableName) { try { List relativeNamespace = LanceNamespaceName.dorisDatabaseNameToNamespace( @@ -410,6 +481,27 @@ private String sanitizedRootCauseMessage(Throwable throwable) { return message; } + @VisibleForTesting + String sanitizedRootCauseMessage(Throwable throwable, String datasetUri, + Map runtimeStorageOptions) { + String message = ExceptionUtils.getRootCauseMessage(throwable); + Map nonNullStorageOptions = runtimeStorageOptions == null + ? Collections.emptyMap() : runtimeStorageOptions; + List sensitiveValues = new ArrayList<>(); + sensitiveValues.add(catalogProperty.getOrDefault(REST_BEARER_TOKEN, "")); + sensitiveValues.add(catalogProperty.getOrDefault(REST_API_KEY, "")); + for (String sensitiveKey : RUNTIME_SENSITIVE_OPTION_KEYS) { + sensitiveValues.add(nonNullStorageOptions.getOrDefault(sensitiveKey, "")); + } + sensitiveValues.add(datasetUri); + sensitiveValues.removeIf(StringUtils::isEmpty); + sensitiveValues.sort((left, right) -> Integer.compare(right.length(), left.length())); + for (String sensitiveValue : sensitiveValues) { + message = message.replace(sensitiveValue, "***"); + } + return truncateUtf8(removeControlCharacters(message), MAX_PROVIDER_MESSAGE_BYTES); + } + private Throwable safeCause(Throwable throwable) { if (StringUtils.isNotEmpty(catalogProperty.getOrDefault(REST_BEARER_TOKEN, "")) || StringUtils.isNotEmpty(catalogProperty.getOrDefault(REST_API_KEY, ""))) { @@ -417,4 +509,43 @@ private Throwable safeCause(Throwable throwable) { } return throwable; } + + private static String removeControlCharacters(String value) { + StringBuilder sanitized = new StringBuilder(value.length()); + value.codePoints().filter(codePoint -> !Character.isISOControl(codePoint)) + .forEach(sanitized::appendCodePoint); + return sanitized.toString(); + } + + private static String truncateUtf8(String value, int maxBytes) { + if (value.getBytes(StandardCharsets.UTF_8).length <= maxBytes) { + return value; + } + int end = 0; + int bytes = 0; + while (end < value.length()) { + int codePoint = value.codePointAt(end); + int codePointBytes = new String(Character.toChars(codePoint)) + .getBytes(StandardCharsets.UTF_8).length; + if (bytes + codePointBytes > maxBytes) { + break; + } + bytes += codePointBytes; + end += Character.charCount(codePoint); + } + return value.substring(0, end); + } + + private static final class ResolvedTableAccess { + private final String datasetUri; + private final Map javaStorageOptions; + private final Map backendStorageOptions; + + private ResolvedTableAccess(String datasetUri, Map javaStorageOptions, + Map backendStorageOptions) { + this.datasetUri = datasetUri; + this.javaStorageOptions = Collections.unmodifiableMap(new HashMap<>(javaStorageOptions)); + this.backendStorageOptions = Collections.unmodifiableMap(new HashMap<>(backendStorageOptions)); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java index 5089c2925495fe..acba68b63fcbe5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalTable.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.Column; +import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.mvcc.MvccSnapshot; @@ -70,6 +71,11 @@ public LanceTableMetadata loadMetadataForVectorSearch() { db.getRemoteName(), remoteName); } + public List loadIndexMetadata() throws AnalysisException { + return ((LanceExternalCatalog) catalog).loadTableIndexMetadata( + db.getRemoteName(), remoteName); + } + private LanceTableMetadata loadMetadata(Optional tableSnapshot) { return ((LanceExternalCatalog) catalog).loadTableMetadata( db.getRemoteName(), remoteName, tableSnapshot); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java new file mode 100644 index 00000000000000..431b7826bec2b0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoader.java @@ -0,0 +1,408 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance; + +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.common.collect.ImmutableSet; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.commons.lang3.StringUtils; +import org.lance.Dataset; +import org.lance.index.IndexCriteria; +import org.lance.index.IndexDescription; +import org.lance.schema.LanceField; + +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; +import java.util.TreeMap; + +/** Loads and normalizes logical index metadata from one latest Lance dataset snapshot. */ +public final class LanceIndexMetadataLoader { + private static final int MAX_LOGICAL_INDEXES = 256; + private static final int MAX_COLUMNS_PER_INDEX = 64; + // Post-JNI cap for physical entries, independent of the logical-name limit. + private static final int MAX_PHYSICAL_INDEX_ENTRIES = 16 * 1024; + private static final int MAX_COLUMN_NAMES_BYTES = 16 * 1024; + private static final int MAX_SCHEMA_FIELDS = MAX_LOGICAL_INDEXES * MAX_COLUMNS_PER_INDEX; + private static final int MAX_SCHEMA_DEPTH = 64; + private static final int MAX_EXTERNAL_STRING_BYTES = 1024; + private static final int MAX_PROPERTIES_BYTES = 400; + + // System entries exposed by the pinned Lance producers and SDK paths. + private static final Set SYSTEM_INDEX_NAMES = ImmutableSet.of( + "__lance_frag_reuse", + "__lance_mem_wal"); + private static final Set TOP_LEVEL_PROPERTY_ALLOWLIST = ImmutableSet.of( + "metric_type", + "target_partition_size"); + private static final Set HNSW_PROPERTY_ALLOWLIST = ImmutableSet.of( + "construction_ef", + "max_connections", + "max_level"); + private static final Set COMPRESSION_PROPERTY_ALLOWLIST = ImmutableSet.of( + "type", + "num_bits", + "num_sub_vectors", + "rotation_type"); + + private LanceIndexMetadataLoader() { + } + + /** Loads logical indexes and schema fields from the same latest dataset snapshot. */ + public static List load(String datasetUri, + Map javaStorageOptions, BufferAllocator allocator) throws Exception { + try (Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri) + .readOptions(LanceReadOptions.build(javaStorageOptions, OptionalLong.empty())).build()) { + Map fieldNames = buildFieldNamesById(dataset.getLanceSchema().fields()); + return normalize(describeUserIndexes(dataset), fieldNames); + } + } + + /** + * Describes only user-created indexes. The Lance JNI bulk describe path also tries to + * materialize details for internal indexes, whose details are not supported by the SDK. + */ + static List describeUserIndexes(Dataset dataset) { + List listedNames = dataset.listIndexes(); + if (listedNames == null) { + throw new IllegalArgumentException("Lance index names must not be null"); + } + if (listedNames.size() > MAX_PHYSICAL_INDEX_ENTRIES) { + throw new IllegalArgumentException( + "Lance physical index entry count exceeds limit " + + MAX_PHYSICAL_INDEX_ENTRIES); + } + + Set userIndexNames = new LinkedHashSet<>(); + for (String listedName : listedNames) { + if (SYSTEM_INDEX_NAMES.contains(listedName)) { + continue; + } + String name = requireExternalString(listedName, "Lance logical index name"); + // nativeListIndexes returns physical entries, so one logical name can repeat. + userIndexNames.add(name); + if (userIndexNames.size() > MAX_LOGICAL_INDEXES) { + throw new IllegalArgumentException( + "Lance logical index count exceeds limit " + MAX_LOGICAL_INDEXES); + } + } + + List descriptions = new ArrayList<>(userIndexNames.size()); + for (String name : userIndexNames) { + IndexCriteria criteria = new IndexCriteria.Builder().hasName(name).build(); + List matching = dataset.describeIndices(criteria); + if (matching == null) { + throw new IllegalArgumentException("Lance index descriptions must not be null"); + } + // A criteria query is an exact lookup; any other cardinality is inconsistent metadata. + if (matching.size() != 1) { + throw new IllegalArgumentException( + "Lance index criteria must return exactly one description"); + } + IndexDescription description = matching.get(0); + if (description == null) { + throw new IllegalArgumentException( + "Lance logical index description must not be null"); + } + String describedName = requireExternalString( + description.getName(), "Lance logical index name"); + if (!name.equals(describedName)) { + throw new IllegalArgumentException( + "Lance index description name does not match requested name"); + } + descriptions.add(description); + } + return descriptions; + } + + static Map buildFieldNamesById(List fields) { + if (fields == null) { + throw new IllegalArgumentException("Lance schema fields must not be null"); + } + if (fields.size() > MAX_SCHEMA_FIELDS) { + throw new IllegalArgumentException( + "Lance schema field count exceeds limit " + MAX_SCHEMA_FIELDS); + } + Map fieldNames = new HashMap<>(); + SchemaTraversalState traversalState = new SchemaTraversalState(); + for (LanceField field : fields) { + collectFieldNames(field, "", 1, fieldNames, traversalState); + } + return fieldNames; + } + + private static void collectFieldNames(LanceField field, String parentPath, int depth, + Map fieldNames, SchemaTraversalState traversalState) { + if (depth > MAX_SCHEMA_DEPTH) { + throw new IllegalArgumentException( + "Lance schema depth exceeds limit " + MAX_SCHEMA_DEPTH); + } + if (field == null) { + throw new IllegalArgumentException("Lance schema field must not be null"); + } + ++traversalState.fieldCount; + if (traversalState.fieldCount > MAX_SCHEMA_FIELDS) { + throw new IllegalArgumentException( + "Lance schema field count exceeds limit " + MAX_SCHEMA_FIELDS); + } + String segment = formatFieldPathSegment( + requireExternalString(field.getName(), "Lance schema field name")); + String path = requireExternalString( + parentPath.isEmpty() ? segment : parentPath + "." + segment, + "Lance schema field path"); + if (fieldNames.put(field.getId(), path) != null) { + throw new IllegalArgumentException("Duplicate Lance schema field id " + field.getId()); + } + List children = field.getChildren(); + if (children == null) { + throw new IllegalArgumentException("Lance schema field children must not be null"); + } + for (LanceField child : children) { + collectFieldNames(child, path, depth + 1, fieldNames, traversalState); + } + } + + private static String formatFieldPathSegment(String segment) { + boolean requiresQuoting = segment.codePoints() + .anyMatch(codePoint -> !Character.isLetterOrDigit(codePoint) + && codePoint != '_'); + if (requiresQuoting) { + return "`" + segment.replace("`", "``") + "`"; + } + return segment; + } + + /** Converts SDK descriptions into bounded immutable Java-only metadata. */ + static List normalize(List descriptions, + Map fieldNames) { + if (descriptions == null) { + throw new IllegalArgumentException("Lance index descriptions must not be null"); + } + if (descriptions.size() > MAX_LOGICAL_INDEXES) { + throw new IllegalArgumentException( + "Lance logical index count exceeds limit " + MAX_LOGICAL_INDEXES); + } + if (fieldNames == null) { + throw new IllegalArgumentException("Lance field names must not be null"); + } + + List normalized = new ArrayList<>(descriptions.size()); + int aggregateColumnNamesBytes = 0; + for (int position = 0; position < descriptions.size(); ++position) { + IndexDescription description = descriptions.get(position); + if (description == null) { + throw new IllegalArgumentException("Lance logical index description must not be null"); + } + + String name = requireExternalString( + description.getName(), "Lance logical index name"); + String indexType = requireExternalString( + description.getIndexType(), "Lance logical index type"); + List fieldIds = description.getFieldIds(); + if (fieldIds == null || fieldIds.isEmpty()) { + throw new IllegalArgumentException( + "Lance logical index field IDs must not be null or empty"); + } + if (fieldIds.size() > MAX_COLUMNS_PER_INDEX) { + throw new IllegalArgumentException( + "Lance logical index column count exceeds limit " + MAX_COLUMNS_PER_INDEX); + } + + List columns = new ArrayList<>(fieldIds.size()); + Set uniqueFieldIds = new HashSet<>(); + for (Integer fieldId : fieldIds) { + if (fieldId == null) { + throw new IllegalArgumentException( + "Lance logical index field ID must not be null"); + } + if (!uniqueFieldIds.add(fieldId)) { + throw new IllegalArgumentException( + "Duplicate field id " + fieldId + " in Lance logical index metadata"); + } + if (!fieldNames.containsKey(fieldId)) { + throw new IllegalArgumentException( + "Lance index metadata references unknown field id " + fieldId); + } + String column = requireExternalString( + fieldNames.get(fieldId), "Lance logical index column name"); + aggregateColumnNamesBytes += utf8Length(column); + if (aggregateColumnNamesBytes > MAX_COLUMN_NAMES_BYTES) { + throw new IllegalArgumentException( + "Lance logical index column names exceed aggregate limit " + + MAX_COLUMN_NAMES_BYTES + " UTF-8 bytes"); + } + columns.add(column); + } + + String properties = normalizeProperties(name, description.getDetailsJson()); + LanceLogicalIndex index = new LanceLogicalIndex(name, columns, indexType, properties); + normalized.add(new IndexedLogicalIndex(index, position)); + } + + Set logicalIndexNames = new HashSet<>(); + for (IndexedLogicalIndex indexed : normalized) { + String name = indexed.index.getName(); + if (!logicalIndexNames.add(name)) { + throw new IllegalArgumentException( + "Duplicate Lance logical index name '" + name + "'"); + } + } + normalized.sort(Comparator.comparing( + (IndexedLogicalIndex indexed) -> indexed.index.getName()) + .thenComparingInt(indexed -> indexed.position)); + List result = new ArrayList<>(normalized.size()); + for (IndexedLogicalIndex indexed : normalized) { + result.add(indexed.index); + } + return Collections.unmodifiableList(result); + } + + private static String normalizeProperties(String indexName, String detailsJson) { + if (detailsJson == null) { + return "{}"; + } + if (utf8Length(detailsJson) > MAX_EXTERNAL_STRING_BYTES) { + throw new IllegalArgumentException( + "Lance index details JSON exceeds limit " + + MAX_EXTERNAL_STRING_BYTES + " UTF-8 bytes"); + } + if (StringUtils.isBlank(detailsJson)) { + return "{}"; + } + + JsonElement parsed; + try (JsonReader reader = new JsonReader(new StringReader(detailsJson))) { + reader.setLenient(false); + parsed = GsonUtils.GSON.getAdapter(JsonElement.class).read(reader); + if (reader.peek() != JsonToken.END_DOCUMENT) { + throw invalidDetailsJson(indexName); + } + } catch (IOException | RuntimeException e) { + throw invalidDetailsJson(indexName); + } + if (!parsed.isJsonObject()) { + throw invalidDetailsJson(indexName); + } + + TreeMap allowedProperties = new TreeMap<>(); + JsonObject object = parsed.getAsJsonObject(); + copyPrimitiveProperties( + object, TOP_LEVEL_PROPERTY_ALLOWLIST, allowedProperties, indexName); + copyNestedProperties( + object, "compression", COMPRESSION_PROPERTY_ALLOWLIST, + allowedProperties, indexName); + copyNestedProperties( + object, "hnsw", HNSW_PROPERTY_ALLOWLIST, allowedProperties, indexName); + + String properties = GsonUtils.GSON.toJson(allowedProperties); + if (utf8Length(properties) > MAX_PROPERTIES_BYTES) { + throw new IllegalArgumentException( + "Lance index properties exceed limit " + + MAX_PROPERTIES_BYTES + " UTF-8 bytes"); + } + return properties; + } + + private static void copyNestedProperties(JsonObject source, String propertyName, + Set allowlist, Map target, String indexName) { + JsonElement nested = source.get(propertyName); + if (nested == null || nested.isJsonNull()) { + return; + } + if (!nested.isJsonObject()) { + throw invalidDetailsJson(indexName); + } + + TreeMap allowedNested = new TreeMap<>(); + copyPrimitiveProperties(nested.getAsJsonObject(), allowlist, allowedNested, indexName); + if (allowedNested.isEmpty()) { + return; + } + JsonObject normalizedNested = new JsonObject(); + for (Map.Entry entry : allowedNested.entrySet()) { + normalizedNested.add(entry.getKey(), entry.getValue()); + } + target.put(propertyName, normalizedNested); + } + + private static void copyPrimitiveProperties(JsonObject source, Set allowlist, + Map target, String indexName) { + for (Map.Entry entry : source.entrySet()) { + if (!allowlist.contains(entry.getKey())) { + continue; + } + JsonElement value = entry.getValue(); + if (value == null || value.isJsonNull()) { + continue; + } + if (!value.isJsonPrimitive()) { + throw invalidDetailsJson(indexName); + } + target.put(entry.getKey(), value); + } + } + + private static IllegalArgumentException invalidDetailsJson(String indexName) { + return new IllegalArgumentException( + "Invalid Lance index details JSON for '" + indexName + "'"); + } + + private static String requireExternalString(String value, String valueType) { + if (value == null || value.isEmpty()) { + throw new IllegalArgumentException(valueType + " must not be null or empty"); + } + if (utf8Length(value) > MAX_EXTERNAL_STRING_BYTES) { + throw new IllegalArgumentException(valueType + " exceeds limit " + + MAX_EXTERNAL_STRING_BYTES + " UTF-8 bytes"); + } + return value; + } + + private static int utf8Length(String value) { + return value.getBytes(StandardCharsets.UTF_8).length; + } + + private static final class SchemaTraversalState { + private int fieldCount; + } + + private static final class IndexedLogicalIndex { + private final LanceLogicalIndex index; + private final int position; + + private IndexedLogicalIndex(LanceLogicalIndex index, int position) { + this.index = index; + this.position = position; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceLogicalIndex.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceLogicalIndex.java new file mode 100644 index 00000000000000..2003243f438ca9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceLogicalIndex.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** Immutable logical index metadata read from one Lance dataset snapshot. */ +public final class LanceLogicalIndex { + private final String name; + private final List columns; + private final String indexType; + private final String properties; + + public LanceLogicalIndex(String name, List columns, + String indexType, String properties) { + this.name = Objects.requireNonNull(name, "name must not be null"); + List columnCopy = new ArrayList<>( + Objects.requireNonNull(columns, "columns must not be null")); + for (String column : columnCopy) { + Objects.requireNonNull(column, "column must not be null"); + } + this.columns = Collections.unmodifiableList(columnCopy); + this.indexType = Objects.requireNonNull(indexType, "indexType must not be null"); + this.properties = Objects.requireNonNull(properties, "properties must not be null"); + } + + public String getName() { + return name; + } + + public List getColumns() { + return columns; + } + + public String getIndexType() { + return indexType; + } + + public String getProperties() { + return properties; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataReadExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataReadExecutor.java new file mode 100644 index 00000000000000..0e6ab36b239ee8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataReadExecutor.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance; + +import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.annotations.VisibleForTesting; + +import java.util.Locale; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** Runs Lance JNI metadata reads behind a finite FE concurrency and deadline boundary. */ +final class LanceMetadataReadExecutor { + private static final int DEFAULT_TIMEOUT_SECONDS = 60; + private static final int MAX_CONCURRENT_READS = 4; + private static final int MAX_QUEUED_READS = 16; + private static final ThreadPoolExecutor EXECUTOR = ThreadPoolManager.newDaemonFixedThreadPool( + MAX_CONCURRENT_READS, + MAX_QUEUED_READS, + "lance-metadata-read", + false, + new ThreadPoolExecutor.AbortPolicy()); + + private LanceMetadataReadExecutor() { + } + + static T execute(Callable task) throws Exception { + ConnectContext context = ConnectContext.get(); + int queryTimeoutSeconds = context == null + ? DEFAULT_TIMEOUT_SECONDS : context.getQueryTimeoutS(); + int timeoutSeconds = queryTimeoutSeconds > 0 + ? Math.min(queryTimeoutSeconds, DEFAULT_TIMEOUT_SECONDS) : DEFAULT_TIMEOUT_SECONDS; + return execute(task, EXECUTOR, timeoutSeconds, TimeUnit.SECONDS); + } + + @VisibleForTesting + static T execute(Callable task, ExecutorService executor, + long timeout, TimeUnit timeoutUnit) throws Exception { + if (timeout <= 0) { + throw new IllegalArgumentException("Lance metadata read timeout must be positive"); + } + long timeoutNanos = timeoutUnit.toNanos(timeout); + if (timeoutNanos <= 0) { + throw new IllegalArgumentException("Lance metadata read timeout is too small"); + } + + long deadlineNanos = System.nanoTime() + timeoutNanos; + Future future; + try { + future = executor.submit(() -> { + // A request can expire while waiting in the finite queue. Do not enter JNI for a + // result whose caller has already timed out. + if (remainingNanos(deadlineNanos) <= 0) { + throw timeoutFailure(timeout, timeoutUnit); + } + return task.call(); + }); + } catch (RejectedExecutionException e) { + throw new MetadataReadCapacityException( + "Lance metadata read capacity is exhausted"); + } + + try { + long remainingNanos = remainingNanos(deadlineNanos); + if (remainingNanos <= 0) { + throw timeoutFailure(timeout, timeoutUnit); + } + return future.get(remainingNanos, TimeUnit.NANOSECONDS); + } catch (TimeoutException e) { + // Deliberately do not cancel or interrupt the Future. If JNI has started, the worker + // remains the sole owner of its Dataset and allocator until the native call returns. + throw timeoutFailure(timeout, timeoutUnit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // As with timeout, interruption of the waiting caller must not interrupt JNI. + throw new MetadataReadInterruptedException( + "Interrupted while waiting for Lance metadata read"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new RuntimeException(cause); + } + } + + private static long remainingNanos(long deadlineNanos) { + return deadlineNanos - System.nanoTime(); + } + + private static MetadataReadTimeoutException timeoutFailure(long timeout, TimeUnit timeoutUnit) { + return new MetadataReadTimeoutException("Lance metadata read timed out after " + + timeout + " " + timeoutUnit.name().toLowerCase(Locale.ROOT)); + } + + static final class MetadataReadTimeoutException extends RuntimeException { + MetadataReadTimeoutException(String message) { + super(message); + } + } + + static final class MetadataReadCapacityException extends RuntimeException { + MetadataReadCapacityException(String message) { + super(message); + } + } + + static final class MetadataReadInterruptedException extends RuntimeException { + MetadataReadInterruptedException(String message) { + super(message); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommand.java index 3315ec8d3aabbd..2086aa97580b66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommand.java @@ -29,7 +29,11 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.lance.LanceExternalCatalog; +import org.apache.doris.datasource.lance.LanceExternalTable; +import org.apache.doris.datasource.lance.LanceLogicalIndex; import org.apache.doris.info.TableNameInfo; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; @@ -108,30 +112,62 @@ private ShowResultSet handleShowIndex(ConnectContext ctx, StmtExecutor executor) analyze(ctx); List> rows = Lists.newArrayList(); - // in show index, only support internal catalog - DatabaseIf db = Env.getCurrentEnv().getCatalogMgr() - .getCatalogOrAnalysisException(tableNameInfo.getCtl()) - .getDbOrAnalysisException(tableNameInfo.getDb()); + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrAnalysisException(tableNameInfo.getCtl()); + if (catalog instanceof LanceExternalCatalog + && ((LanceExternalCatalog) catalog).isRestCatalogConfigured()) { + throw new AnalysisException("SHOW INDEX is not supported for Lance REST catalogs"); + } + DatabaseIf db = catalog.getDbOrAnalysisException(tableNameInfo.getDb()); if (db instanceof Database) { + rows = getInternalIndexRows(db); + } else if (catalog instanceof LanceExternalCatalog) { TableIf table = db.getTableOrAnalysisException(tableNameInfo.getTbl()); - if (table instanceof OlapTable) { - OlapTable olapTable = (OlapTable) table; - olapTable.readLock(); - try { - List indexes = olapTable.getIndexes(); - for (Index index : indexes) { - rows.add(Lists.newArrayList(tableNameInfo.getTbl(), "", index.getIndexName(), - "", String.join(",", index.getColumns()), "", "", "", "", - "", index.getIndexType().name(), index.getComment(), index.getPropertiesString())); - } - } finally { - olapTable.readUnlock(); - } + if (!(table instanceof LanceExternalTable)) { + throw new AnalysisException("Table " + tableNameInfo.getTbl() + " is not a Lance table"); } + rows = getLanceIndexRows((LanceExternalTable) table); } return new ShowResultSet(getMetaData(), rows); } + private List> getInternalIndexRows(DatabaseIf db) throws Exception { + List> rows = Lists.newArrayList(); + TableIf table = db.getTableOrAnalysisException(tableNameInfo.getTbl()); + if (table instanceof OlapTable) { + OlapTable olapTable = (OlapTable) table; + olapTable.readLock(); + try { + List indexes = olapTable.getIndexes(); + for (Index index : indexes) { + rows.add(Lists.newArrayList(tableNameInfo.getTbl(), "", index.getIndexName(), + "", String.join(",", index.getColumns()), "", "", "", "", + "", index.getIndexType().name(), index.getComment(), index.getPropertiesString())); + } + } finally { + olapTable.readUnlock(); + } + } + return rows; + } + + private List> getLanceIndexRows(LanceExternalTable table) throws AnalysisException { + return buildLanceRows(table.getName(), table.loadIndexMetadata()); + } + + @VisibleForTesting + static List> buildLanceRows(String tableName, List indexes) { + List> rows = Lists.newArrayList(); + for (LanceLogicalIndex index : indexes) { + List columns = index.getColumns(); + for (int i = 0; i < columns.size(); i++) { + rows.add(Lists.newArrayList(tableName, "", index.getName(), String.valueOf(i + 1), + columns.get(i), "", "", "", "", "", index.getIndexType(), "", index.getProperties())); + } + } + return rows; + } + @Override public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { return handleShowIndex(ctx, executor); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceFilesystemCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceFilesystemCatalogTest.java index 6b4a9f9c2bca7d..0e1a787bddaad1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceFilesystemCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceFilesystemCatalogTest.java @@ -20,9 +20,27 @@ import org.junit.Assert; import org.junit.Test; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RunnableFuture; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; public class LanceFilesystemCatalogTest { @@ -90,4 +108,268 @@ public void testNamespaceNameRoundTrip() throws Exception { Assert.assertEquals(Collections.singletonList("default"), LanceNamespaceName.dorisDatabaseNameToNamespace(rootCollision, ".", "default")); } + + @Test + public void testIndexMetadataErrorSanitization() { + String bearerToken = "sentinel-bearer-token"; + String apiKey = "sentinel-api-key"; + String accessKey = "sentinel-access-key"; + String secretKey = "sentinel-secret-key"; + String sessionToken = "sentinel-session-token"; + String datasetUri = "s3://sentinel-user:sentinel-password@bucket/private/table.lance"; + + Map catalogProperties = new HashMap<>(); + catalogProperties.put(LanceExternalCatalog.REST_BEARER_TOKEN, bearerToken); + catalogProperties.put(LanceExternalCatalog.REST_API_KEY, apiKey); + LanceExternalCatalog catalog = new LanceExternalCatalog( + 1, "lance_filesystem", null, catalogProperties, ""); + + Map runtimeStorageOptions = new HashMap<>(); + runtimeStorageOptions.put("aws_access_key_id", accessKey); + runtimeStorageOptions.put("aws_secret_access_key", secretKey); + runtimeStorageOptions.put("aws_session_token", sessionToken); + String providerMessage = "provider failure\nuri=" + datasetUri + + " bearer=" + bearerToken + " api-key=" + apiKey + + " access=" + accessKey + " secret=" + secretKey + " session=" + sessionToken; + + RuntimeException providerFailure = new RuntimeException(providerMessage); + RuntimeException exposed = catalog.indexMetadataLoadFailure( + "db", "table", providerFailure, datasetUri, runtimeStorageOptions); + StringWriter stackTrace = new StringWriter(); + exposed.printStackTrace(new PrintWriter(stackTrace)); + + for (String sentinel : Arrays.asList(bearerToken, apiKey, accessKey, secretKey, + sessionToken, datasetUri)) { + Assert.assertFalse(exposed.getMessage().contains(sentinel)); + Assert.assertFalse(exposed.getCause().getMessage().contains(sentinel)); + Assert.assertFalse(stackTrace.toString().contains(sentinel)); + } + Assert.assertNotSame(providerFailure, exposed.getCause()); + Assert.assertTrue(exposed.getCause().getMessage().contains("***")); + Assert.assertFalse(exposed.getCause().getMessage().contains("\n")); + Assert.assertTrue(exposed.getCause().getMessage().getBytes(StandardCharsets.UTF_8).length <= 1024); + } + + @Test + public void testIndexMetadataErrorSanitizationUsesUtf8ByteLimit() { + LanceExternalCatalog catalog = new LanceExternalCatalog( + 2, "lance_filesystem", null, Collections.emptyMap(), ""); + char[] multibyteCharacters = new char[1024]; + Arrays.fill(multibyteCharacters, '界'); + + String sanitized = catalog.sanitizedRootCauseMessage( + new RuntimeException(new String(multibyteCharacters)), null, Collections.emptyMap()); + + Assert.assertTrue(sanitized.getBytes(StandardCharsets.UTF_8).length <= 1024); + } + + @Test + public void testIndexMetadataErrorSanitizationReplacesOverlappingSecrets() { + Map catalogProperties = new HashMap<>(); + catalogProperties.put(LanceExternalCatalog.REST_BEARER_TOKEN, "overlapping-secret"); + LanceExternalCatalog catalog = new LanceExternalCatalog( + 3, "lance_filesystem", null, catalogProperties, ""); + Map runtimeStorageOptions = Collections.singletonMap( + "aws_secret_access_key", "overlapping-secret-with-suffix"); + + String sanitized = catalog.sanitizedRootCauseMessage( + new RuntimeException("overlapping-secret-with-suffix"), + null, runtimeStorageOptions); + + Assert.assertEquals("RuntimeException: ***", sanitized); + } + + @Test + public void testIndexMetadataFailurePreservesSanitizedMetadataErrorType() { + LanceExternalCatalog catalog = new LanceExternalCatalog( + 4, "lance_filesystem", null, Collections.emptyMap(), ""); + IllegalArgumentException metadataFailure = new IllegalArgumentException("invalid metadata"); + + RuntimeException exposed = catalog.indexMetadataLoadFailure( + "db", "table", metadataFailure, null, null); + + Assert.assertTrue(exposed.getCause() instanceof IllegalArgumentException); + Assert.assertNotSame(metadataFailure, exposed.getCause()); + Assert.assertEquals("IllegalArgumentException: invalid metadata", + exposed.getCause().getMessage()); + } + + @Test + public void testIndexMetadataReadTimeoutKeepsWorkerOwnershipUntilReturn() throws Exception { + CountDownLatch taskStarted = new CountDownLatch(1); + CountDownLatch releaseTask = new CountDownLatch(1); + CountDownLatch taskFinished = new CountDownLatch(1); + AtomicBoolean ownerOpen = new AtomicBoolean(false); + AtomicReference callerFailure = new AtomicReference<>(); + ThreadPoolExecutor executor = new ThreadPoolExecutor( + 1, 1, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()) { + @Override + protected RunnableFuture newTaskFor(Callable callable) { + return new FutureTask(callable) { + @Override + public T get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + if (!taskStarted.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("Metadata read task did not start"); + } + throw new TimeoutException("deterministic test deadline"); + } + }; + } + }; + Thread caller = new Thread(() -> { + try { + LanceMetadataReadExecutor.execute(() -> { + ownerOpen.set(true); + taskStarted.countDown(); + try { + releaseTask.await(); + return Collections.emptyList(); + } finally { + ownerOpen.set(false); + taskFinished.countDown(); + } + }, executor, 5, TimeUnit.SECONDS); + } catch (Throwable throwable) { + callerFailure.set(throwable); + } + }, "lance-metadata-read-timeout-caller-test"); + try { + caller.start(); + Assert.assertTrue(taskStarted.await(5, TimeUnit.SECONDS)); + caller.join(TimeUnit.SECONDS.toMillis(5)); + + Assert.assertFalse(caller.isAlive()); + Assert.assertTrue(callerFailure.get() + instanceof LanceMetadataReadExecutor.MetadataReadTimeoutException); + Assert.assertEquals("Lance metadata read timed out after 5 seconds", + callerFailure.get().getMessage()); + Assert.assertTrue(ownerOpen.get()); + Assert.assertEquals(1, taskFinished.getCount()); + + releaseTask.countDown(); + Assert.assertTrue(taskFinished.await(5, TimeUnit.SECONDS)); + Assert.assertFalse(ownerOpen.get()); + } finally { + releaseTask.countDown(); + caller.interrupt(); + caller.join(TimeUnit.SECONDS.toMillis(5)); + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + public void testInterruptedIndexMetadataWaitKeepsWorkerOwnershipUntilReturn() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch taskStarted = new CountDownLatch(1); + CountDownLatch releaseTask = new CountDownLatch(1); + CountDownLatch taskFinished = new CountDownLatch(1); + AtomicBoolean ownerOpen = new AtomicBoolean(false); + AtomicReference callerFailure = new AtomicReference<>(); + Thread caller = new Thread(() -> { + try { + LanceMetadataReadExecutor.execute(() -> { + ownerOpen.set(true); + taskStarted.countDown(); + try { + releaseTask.await(); + return Collections.emptyList(); + } finally { + ownerOpen.set(false); + taskFinished.countDown(); + } + }, executor, 5, TimeUnit.SECONDS); + } catch (Throwable throwable) { + callerFailure.set(throwable); + } + }, "lance-metadata-read-interrupted-caller-test"); + try { + caller.start(); + Assert.assertTrue(taskStarted.await(5, TimeUnit.SECONDS)); + caller.interrupt(); + caller.join(TimeUnit.SECONDS.toMillis(5)); + + Assert.assertFalse(caller.isAlive()); + Assert.assertTrue(callerFailure.get() + instanceof LanceMetadataReadExecutor.MetadataReadInterruptedException); + Assert.assertTrue(ownerOpen.get()); + Assert.assertEquals(1, taskFinished.getCount()); + + releaseTask.countDown(); + Assert.assertTrue(taskFinished.await(5, TimeUnit.SECONDS)); + Assert.assertFalse(ownerOpen.get()); + } finally { + releaseTask.countDown(); + caller.interrupt(); + caller.join(TimeUnit.SECONDS.toMillis(5)); + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + public void testExpiredQueuedIndexMetadataReadDoesNotEnterProvider() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch blockerStarted = new CountDownLatch(1); + CountDownLatch releaseBlocker = new CountDownLatch(1); + AtomicBoolean providerEntered = new AtomicBoolean(false); + try { + executor.submit(() -> { + blockerStarted.countDown(); + releaseBlocker.await(); + return null; + }); + Assert.assertTrue(blockerStarted.await(5, TimeUnit.SECONDS)); + + try { + LanceMetadataReadExecutor.execute(() -> { + providerEntered.set(true); + return Collections.emptyList(); + }, executor, 20, TimeUnit.MILLISECONDS); + Assert.fail("Expected Lance metadata read timeout"); + } catch (LanceMetadataReadExecutor.MetadataReadTimeoutException expected) { + Assert.assertTrue(expected.getMessage().contains("timed out")); + } + + releaseBlocker.countDown(); + executor.shutdown(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + Assert.assertFalse(providerEntered.get()); + } finally { + releaseBlocker.countDown(); + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + public void testIndexMetadataReadRejectsWhenCapacityIsExhausted() throws Exception { + CountDownLatch blockerStarted = new CountDownLatch(1); + CountDownLatch releaseBlocker = new CountDownLatch(1); + ThreadPoolExecutor executor = new ThreadPoolExecutor( + 1, 1, 0, TimeUnit.MILLISECONDS, new SynchronousQueue<>(), + new ThreadPoolExecutor.AbortPolicy()); + try { + executor.submit(() -> { + blockerStarted.countDown(); + releaseBlocker.await(); + return null; + }); + Assert.assertTrue(blockerStarted.await(5, TimeUnit.SECONDS)); + + try { + LanceMetadataReadExecutor.execute( + Collections::emptyList, executor, 1, TimeUnit.SECONDS); + Assert.fail("Expected Lance metadata read capacity rejection"); + } catch (LanceMetadataReadExecutor.MetadataReadCapacityException expected) { + Assert.assertEquals( + "Lance metadata read capacity is exhausted", expected.getMessage()); + } + } finally { + releaseBlocker.countDown(); + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoaderTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoaderTest.java new file mode 100644 index 00000000000000..68a5a79acfff5f --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexMetadataLoaderTest.java @@ -0,0 +1,736 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.lance.Dataset; +import org.lance.index.IndexCriteria; +import org.lance.index.IndexDescription; +import org.lance.schema.LanceField; +import org.mockito.Mockito; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class LanceIndexMetadataLoaderTest { + + @Test + public void testEmptyDescriptionsReturnImmutableList() { + List indexes = LanceIndexMetadataLoader.normalize( + Collections.emptyList(), Collections.emptyMap()); + + Assertions.assertTrue(indexes.isEmpty()); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> indexes.add(new LanceLogicalIndex( + "idx", Collections.singletonList("column"), "BTREE", "{}"))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize(null, Collections.emptyMap())); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(null), Collections.emptyMap())); + } + + @Test + public void testDescribeUserIndexesSkipsSystemIndexesAndDeduplicatesPhysicalEntries() { + Dataset dataset = Mockito.mock(Dataset.class); + IndexDescription first = description( + "first_idx", Collections.singletonList(1), "BTREE", null); + IndexDescription prefixedUserName = description( + "__user_idx", Collections.singletonList(2), "BTREE", null); + Mockito.when(dataset.listIndexes()).thenReturn(Arrays.asList( + "__lance_frag_reuse", "first_idx", "__lance_mem_wal", + "first_idx", "__user_idx")); + Mockito.when(dataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenAnswer(invocation -> { + IndexCriteria criteria = invocation.getArgument(0); + String name = criteria.getHasName().orElseThrow(AssertionError::new); + if ("first_idx".equals(name)) { + return Collections.singletonList(first); + } + if ("__user_idx".equals(name)) { + return Collections.singletonList(prefixedUserName); + } + throw new AssertionError("Unexpected index criteria: " + name); + }); + + List descriptions = + LanceIndexMetadataLoader.describeUserIndexes(dataset); + + Assertions.assertEquals(Arrays.asList(first, prefixedUserName), descriptions); + Mockito.verify(dataset, Mockito.times(2)) + .describeIndices(Mockito.any(IndexCriteria.class)); + Mockito.verify(dataset, Mockito.never()).describeIndices(); + } + + @Test + public void testDescribeUserIndexesReturnsEmptyForOnlySystemIndexes() { + Dataset dataset = Mockito.mock(Dataset.class); + Mockito.when(dataset.listIndexes()).thenReturn(Arrays.asList( + "__lance_frag_reuse", "__lance_mem_wal", + "__lance_frag_reuse")); + + Assertions.assertTrue( + LanceIndexMetadataLoader.describeUserIndexes(dataset).isEmpty()); + + Mockito.verify(dataset, Mockito.never()) + .describeIndices(Mockito.any(IndexCriteria.class)); + Mockito.verify(dataset, Mockito.never()).describeIndices(); + } + + @Test + public void testShortMemWalNameIsNotTreatedAsSystemIndex() { + Dataset dataset = Mockito.mock(Dataset.class); + IndexDescription userDescription = description( + "__mem_wal", Collections.singletonList(1), "BTREE", null); + Mockito.when(dataset.listIndexes()) + .thenReturn(Collections.singletonList("__mem_wal")); + Mockito.when(dataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenReturn(Collections.singletonList(userDescription)); + + Assertions.assertEquals(Collections.singletonList(userDescription), + LanceIndexMetadataLoader.describeUserIndexes(dataset)); + Mockito.verify(dataset).describeIndices(Mockito.argThat(criteria -> + criteria.getHasName().filter("__mem_wal"::equals).isPresent())); + } + + @Test + public void testDescribeUserIndexesEnforcesUniqueUserIndexLimitBeforeDescribe() { + Dataset atLimitDataset = Mockito.mock(Dataset.class); + List atLimitNames = new ArrayList<>(); + for (int index = 0; index < 256; ++index) { + atLimitNames.add("idx_" + index); + } + atLimitNames.addAll(Arrays.asList( + "__lance_frag_reuse", "__lance_mem_wal")); + Mockito.when(atLimitDataset.listIndexes()).thenReturn(atLimitNames); + Mockito.when(atLimitDataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenAnswer(invocation -> { + IndexCriteria criteria = invocation.getArgument(0); + String name = criteria.getHasName().orElseThrow(AssertionError::new); + return Collections.singletonList(description( + name, Collections.singletonList(1), "BTREE", null)); + }); + + Assertions.assertEquals(256, + LanceIndexMetadataLoader.describeUserIndexes(atLimitDataset).size()); + Mockito.verify(atLimitDataset, Mockito.times(256)) + .describeIndices(Mockito.any(IndexCriteria.class)); + + Dataset overLimitDataset = Mockito.mock(Dataset.class); + List overLimitNames = new ArrayList<>(atLimitNames.subList(0, 256)); + overLimitNames.add("idx_256"); + Mockito.when(overLimitDataset.listIndexes()).thenReturn(overLimitNames); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(overLimitDataset)); + Assertions.assertTrue(exception.getMessage().contains("256")); + Mockito.verify(overLimitDataset, Mockito.never()) + .describeIndices(Mockito.any(IndexCriteria.class)); + } + + @Test + public void testDescribeUserIndexesBoundsRawPhysicalEntriesIncludingSystemEntries() { + Dataset atLimitDataset = Mockito.mock(Dataset.class); + Mockito.when(atLimitDataset.listIndexes()).thenReturn( + Collections.nCopies(16384, "__lance_frag_reuse")); + Assertions.assertTrue( + LanceIndexMetadataLoader.describeUserIndexes(atLimitDataset).isEmpty()); + + Dataset overLimitDataset = Mockito.mock(Dataset.class); + Mockito.when(overLimitDataset.listIndexes()).thenReturn( + Collections.nCopies(16385, "__lance_frag_reuse")); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(overLimitDataset)); + + Assertions.assertTrue(exception.getMessage().contains("16384")); + Mockito.verify(overLimitDataset, Mockito.never()) + .describeIndices(Mockito.any(IndexCriteria.class)); + } + + @Test + public void testDescribeUserIndexesRejectsInvalidProviderResults() { + Dataset nullNamesDataset = Mockito.mock(Dataset.class); + Mockito.when(nullNamesDataset.listIndexes()).thenReturn(null); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(nullNamesDataset)); + + for (String invalidName : Arrays.asList(null, "")) { + Dataset invalidNameDataset = Mockito.mock(Dataset.class); + Mockito.when(invalidNameDataset.listIndexes()) + .thenReturn(Collections.singletonList(invalidName)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(invalidNameDataset)); + Mockito.verify(invalidNameDataset, Mockito.never()) + .describeIndices(Mockito.any(IndexCriteria.class)); + } + + Dataset nullDescriptionsDataset = Mockito.mock(Dataset.class); + Mockito.when(nullDescriptionsDataset.listIndexes()) + .thenReturn(Collections.singletonList("user_idx")); + Mockito.when(nullDescriptionsDataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenReturn(null); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(nullDescriptionsDataset)); + + Dataset failingDataset = Mockito.mock(Dataset.class); + RuntimeException sdkFailure = new RuntimeException("SDK failure"); + Mockito.when(failingDataset.listIndexes()) + .thenReturn(Collections.singletonList("user_idx")); + Mockito.when(failingDataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenThrow(sdkFailure); + Assertions.assertSame(sdkFailure, Assertions.assertThrows(RuntimeException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(failingDataset))); + } + + @Test + public void testDescribeUserIndexesRequiresExactlyOneMatchingDescription() { + IndexDescription requested = description( + "user_idx", Collections.singletonList(1), "BTREE", null); + for (List invalidDescriptions : Arrays.>asList( + Collections.emptyList(), Arrays.asList(requested, requested))) { + Dataset dataset = Mockito.mock(Dataset.class); + Mockito.when(dataset.listIndexes()) + .thenReturn(Collections.singletonList("user_idx")); + Mockito.when(dataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenReturn(invalidDescriptions); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(dataset)); + Assertions.assertTrue(exception.getMessage().contains("exactly one")); + } + + Dataset nullDescriptionDataset = Mockito.mock(Dataset.class); + Mockito.when(nullDescriptionDataset.listIndexes()) + .thenReturn(Collections.singletonList("user_idx")); + Mockito.when(nullDescriptionDataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenReturn(Collections.singletonList(null)); + IllegalArgumentException nullDescription = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(nullDescriptionDataset)); + Assertions.assertTrue(nullDescription.getMessage().contains("must not be null")); + + Dataset mismatchedNameDataset = Mockito.mock(Dataset.class); + Mockito.when(mismatchedNameDataset.listIndexes()) + .thenReturn(Collections.singletonList("user_idx")); + Mockito.when(mismatchedNameDataset.describeIndices(Mockito.any(IndexCriteria.class))) + .thenReturn(Collections.singletonList(description( + "different_idx", Collections.singletonList(1), "BTREE", null))); + IllegalArgumentException mismatchedName = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.describeUserIndexes(mismatchedNameDataset)); + Assertions.assertTrue(mismatchedName.getMessage().contains( + "does not match requested name")); + } + + @Test + public void testNormalizesIvfPqDescriptionAndDefensivelyCopiesColumns() { + IndexDescription description = description("embedding_idx", + Collections.singletonList(7), "IVF_PQ", + "{\"target_partition_size\":256,\"unknown\":\"secret\"," + + "\"runtime_hints\":{\"secret\":\"opaque\"}," + + "\"compression\":{\"type\":\"pq\",\"num_sub_vectors\":16," + + "\"unknown\":\"opaque\",\"num_bits\":8}," + + "\"metric_type\":\"cosine\"," + + "\"hnsw\":{\"max_level\":3,\"max_connections\":32," + + "\"construction_ef\":200}}"); + + List indexes = LanceIndexMetadataLoader.normalize( + Collections.singletonList(description), + Collections.singletonMap(7, "embedding")); + + Assertions.assertEquals(1, indexes.size()); + LanceLogicalIndex index = indexes.get(0); + Assertions.assertEquals("embedding_idx", index.getName()); + Assertions.assertEquals(Collections.singletonList("embedding"), index.getColumns()); + Assertions.assertEquals("IVF_PQ", index.getIndexType()); + Assertions.assertEquals( + "{\"compression\":{\"num_bits\":8,\"num_sub_vectors\":16,\"type\":\"pq\"}," + + "\"hnsw\":{\"construction_ef\":200,\"max_connections\":32," + + "\"max_level\":3},\"metric_type\":\"cosine\"," + + "\"target_partition_size\":256}", + index.getProperties()); + Assertions.assertFalse(index.getProperties().contains("runtime_hints")); + Assertions.assertFalse(index.getProperties().contains("opaque")); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> index.getColumns().add("another")); + + List mutableColumns = new ArrayList<>(Collections.singletonList("first")); + LanceLogicalIndex directlyConstructed = new LanceLogicalIndex( + "direct", mutableColumns, "BTREE", "{}"); + mutableColumns.add("second"); + Assertions.assertEquals(Collections.singletonList("first"), + directlyConstructed.getColumns()); + } + + @Test + public void testNormalizesRqCompressionRotationType() { + LanceLogicalIndex index = LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "rq_idx", Collections.singletonList(1), "IVF_RQ", + "{\"compression\":{\"rotation_type\":\"matrix\"," + + "\"type\":\"rq\",\"num_bits\":4}," + + "\"metric_type\":\"L2\"}")), + fieldNames("embedding")).get(0); + + Assertions.assertEquals( + "{\"compression\":{\"num_bits\":4,\"rotation_type\":\"matrix\"," + + "\"type\":\"rq\"},\"metric_type\":\"L2\"}", + index.getProperties()); + } + + @Test + public void testSortsIndexesAndPreservesCompositeFieldOrder() { + List descriptions = Arrays.asList( + description("z_idx", Arrays.asList(3, 1, 2), "BTREE", null), + description("a_idx", Collections.singletonList(1), "BITMAP", "{}")); + + List indexes = LanceIndexMetadataLoader.normalize( + descriptions, fieldNames("alpha", "beta", "gamma")); + + Assertions.assertEquals(Arrays.asList("a_idx", "z_idx"), + Arrays.asList(indexes.get(0).getName(), indexes.get(1).getName())); + Assertions.assertEquals(Arrays.asList("gamma", "alpha", "beta"), + indexes.get(1).getColumns()); + } + + @Test + public void testRejectsExactDuplicateNameButPreservesCaseOnlyNames() { + List duplicates = Arrays.asList( + description("idx", Collections.singletonList(1), "BTREE", null), + description("idx", Collections.singletonList(2), "BITMAP", null)); + IllegalArgumentException duplicate = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + duplicates, fieldNames("alpha", "beta"))); + Assertions.assertTrue(duplicate.getMessage().contains( + "Duplicate Lance logical index name 'idx'")); + + List caseOnlyNames = Arrays.asList( + description("idx", Collections.singletonList(1), "BTREE", null), + description("IDX", Collections.singletonList(2), "BITMAP", null)); + List indexes = LanceIndexMetadataLoader.normalize( + caseOnlyNames, fieldNames("alpha", "beta")); + Assertions.assertEquals(Arrays.asList("IDX", "idx"), + Arrays.asList(indexes.get(0).getName(), indexes.get(1).getName())); + } + + @Test + public void testBuildsCanonicalEscapedPathsForNestedFieldIds() { + LanceField parent = field(1, "parent"); + LanceField childWithDot = field(2, "child.with.dot"); + LanceField grandchildWithBacktick = field(3, "tick`name"); + LanceField unicodeChild = field(4, "字段"); + Mockito.when(parent.getChildren()).thenReturn(Arrays.asList(childWithDot, unicodeChild)); + Mockito.when(childWithDot.getChildren()) + .thenReturn(Collections.singletonList(grandchildWithBacktick)); + + Map fieldNames = LanceIndexMetadataLoader.buildFieldNamesById( + Collections.singletonList(parent)); + + Assertions.assertEquals("parent", fieldNames.get(1)); + Assertions.assertEquals("parent.`child.with.dot`", fieldNames.get(2)); + Assertions.assertEquals( + "parent.`child.with.dot`.`tick``name`", fieldNames.get(3)); + Assertions.assertEquals("parent.字段", fieldNames.get(4)); + + LanceLogicalIndex index = LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "nested", Arrays.asList(2, 3, 4), "BTREE", null)), + fieldNames).get(0); + Assertions.assertEquals(Arrays.asList( + "parent.`child.with.dot`", + "parent.`child.with.dot`.`tick``name`", + "parent.字段"), index.getColumns()); + } + + @Test + public void testRejectsSchemaDepthOverLimit() { + Map atLimit = LanceIndexMetadataLoader.buildFieldNamesById( + Collections.singletonList(nestedFieldChain(64))); + Assertions.assertEquals(64, atLimit.size()); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.buildFieldNamesById( + Collections.singletonList(nestedFieldChain(65)))); + Assertions.assertTrue(exception.getMessage().contains("depth")); + Assertions.assertTrue(exception.getMessage().contains("64")); + } + + @Test + public void testRejectsAggregateSchemaFieldCountOverLimit() { + LanceField root = field(0, "root"); + LanceField repeatedChild = field(1, "child"); + AtomicInteger childId = new AtomicInteger(); + Mockito.when(repeatedChild.getId()).thenAnswer( + invocation -> childId.incrementAndGet()); + + Mockito.when(root.getChildren()).thenReturn( + Collections.nCopies(16383, repeatedChild)); + Map atLimit = LanceIndexMetadataLoader.buildFieldNamesById( + Collections.singletonList(root)); + Assertions.assertEquals(16384, atLimit.size()); + + childId.set(0); + Mockito.when(root.getChildren()).thenReturn( + Collections.nCopies(16384, repeatedChild)); + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.buildFieldNamesById( + Collections.singletonList(root))); + Assertions.assertTrue(exception.getMessage().contains("field count")); + Assertions.assertTrue(exception.getMessage().contains("16384")); + } + + @Test + public void testRejectsUnknownDuplicateNullAndEmptyFieldIds() { + IllegalArgumentException unknown = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "unknown", Collections.singletonList(99), "BTREE", null)), + fieldNames("alpha"))); + Assertions.assertTrue(unknown.getMessage().contains( + "Lance index metadata references unknown field id 99")); + + IllegalArgumentException duplicate = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "duplicate", Arrays.asList(1, 1), "BTREE", null)), + fieldNames("alpha"))); + Assertions.assertTrue(duplicate.getMessage().contains("Duplicate field id 1")); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "null", Arrays.asList(1, null), "BTREE", null)), + fieldNames("alpha"))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "empty", Collections.emptyList(), "BTREE", null)), + fieldNames("alpha"))); + } + + @Test + public void testRejectsTooManyColumnsAndLogicalIndexes() { + List tooManyFieldIds = new ArrayList<>(); + for (int id = 1; id <= 65; ++id) { + tooManyFieldIds.add(id); + } + IllegalArgumentException columns = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "many_columns", tooManyFieldIds, "BTREE", null)), + Collections.emptyMap())); + Assertions.assertTrue(columns.getMessage().contains("64")); + + List atLimit = new ArrayList<>(); + for (int index = 0; index < 256; ++index) { + atLimit.add(description( + "idx_" + index, Collections.singletonList(1), "BTREE", null)); + } + Assertions.assertEquals(256, + LanceIndexMetadataLoader.normalize(atLimit, fieldNames("alpha")).size()); + + List overLimit = new ArrayList<>(atLimit); + overLimit.add(description( + "idx_over_limit", Collections.singletonList(1), "BTREE", null)); + IllegalArgumentException indexes = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + overLimit, fieldNames("alpha"))); + Assertions.assertTrue(indexes.getMessage().contains("256")); + } + + @Test + public void testRejectsExternalStringsOverUtf8ByteLimit() { + assertBoundFailure( + description(repeat("x", 1025), Collections.singletonList(1), "BTREE", null), + fieldNames("alpha"), "name", "1024"); + assertBoundFailure( + description("idx", Collections.singletonList(1), repeat("x", 1025), null), + fieldNames("alpha"), "type", "1024"); + assertBoundFailure( + description("idx", Collections.singletonList(1), "BTREE", null), + Collections.singletonMap(1, repeat("x", 1025)), "column", "1024"); + assertBoundFailure( + description("idx", Collections.singletonList(1), "BTREE", repeat("x", 1025)), + fieldNames("alpha"), "details", "1024"); + + String multibyte = repeat("界", 342); + Assertions.assertTrue(multibyte.length() < 1024); + assertBoundFailure( + description(multibyte, Collections.singletonList(1), "BTREE", null), + fieldNames("alpha"), "name", "1024"); + + String withinUtf8Limit = repeat("界", 341); + List indexes = LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + withinUtf8Limit, Collections.singletonList(1), "BTREE", null)), + fieldNames("alpha")); + Assertions.assertEquals(withinUtf8Limit, indexes.get(0).getName()); + } + + @Test + public void testRejectsAggregateColumnNameBytesOverLimit() { + List fieldIds = new ArrayList<>(); + Map fields = new HashMap<>(); + for (int id = 1; id <= 17; ++id) { + fieldIds.add(id); + fields.put(id, repeat("x", 1024)); + } + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "wide", fieldIds, "BTREE", null)), fields)); + Assertions.assertTrue(exception.getMessage().contains("16384")); + } + + @Test + public void testRejectsAggregateColumnNameBytesAcrossIndexes() { + List firstFieldIds = new ArrayList<>(); + List secondFieldIds = new ArrayList<>(); + Map fields = new HashMap<>(); + for (int id = 1; id <= 18; ++id) { + if (id <= 9) { + firstFieldIds.add(id); + } else { + secondFieldIds.add(id); + } + fields.put(id, repeat("x", 1024)); + } + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize(Arrays.asList( + description("first", firstFieldIds, "BTREE", null), + description("second", secondFieldIds, "BTREE", null)), fields)); + Assertions.assertTrue(exception.getMessage().contains("aggregate")); + Assertions.assertTrue(exception.getMessage().contains("16384")); + } + + @Test + public void testNullAndBlankDetailsProduceEmptyProperties() { + List descriptions = Arrays.asList( + description("blank", Collections.singletonList(1), "BTREE", " \n\t"), + description("empty", Collections.singletonList(1), "BTREE", ""), + description("null", Collections.singletonList(1), "BTREE", null), + description("unicode_blank", Collections.singletonList(1), "BTREE", "\u2003")); + + List indexes = LanceIndexMetadataLoader.normalize( + descriptions, fieldNames("alpha")); + + Assertions.assertEquals("{}", indexes.get(0).getProperties()); + Assertions.assertEquals("{}", indexes.get(1).getProperties()); + Assertions.assertEquals("{}", indexes.get(2).getProperties()); + Assertions.assertEquals("{}", indexes.get(3).getProperties()); + } + + @Test + public void testRejectsMalformedAndNonObjectJsonWithoutEchoingInput() { + String malformedDetails = "{\"metric_type\":\"raw-secret\""; + IllegalArgumentException malformed = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "malformed", Collections.singletonList(1), + "IVF_PQ", malformedDetails)), + fieldNames("embedding"))); + Assertions.assertTrue(malformed.getMessage().contains( + "Invalid Lance index details JSON for 'malformed'")); + Assertions.assertFalse(stackTrace(malformed).contains("raw-secret")); + + IllegalArgumentException array = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "array", Collections.singletonList(1), "BTREE", "[1,2]")), + fieldNames("alpha"))); + Assertions.assertTrue(array.getMessage().contains( + "Invalid Lance index details JSON for 'array'")); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "lenient", Collections.singletonList(1), + "BTREE", "{metric_type:cosine}")), + fieldNames("alpha"))); + } + + @Test + public void testFiltersUnknownPropertiesAndSortsAllowlistedNestedScalars() { + String details = "{\"target_partition_size\":256,\"num_sub_vectors\":16," + + "\"unknown\":\"opaque-raw-details\",\"metric_type\":\"cosine\"," + + "\"runtime_hints\":{\"secret\":\"credential\"}," + + "\"compression\":{\"type\":\"pq\",\"num_sub_vectors\":16," + + "\"unknown\":\"nested-opaque\",\"num_bits\":8}," + + "\"hnsw\":{\"max_connections\":32,\"construction_ef\":200," + + "\"max_level\":7,\"unknown\":false}}"; + + LanceLogicalIndex index = LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "idx", Collections.singletonList(1), "IVF_PQ", details)), + fieldNames("embedding")).get(0); + + Assertions.assertEquals( + "{\"compression\":{\"num_bits\":8,\"num_sub_vectors\":16,\"type\":\"pq\"}," + + "\"hnsw\":{\"construction_ef\":200,\"max_connections\":32," + + "\"max_level\":7},\"metric_type\":\"cosine\"," + + "\"target_partition_size\":256}", + index.getProperties()); + Assertions.assertFalse(index.getProperties().contains("opaque-raw-details")); + Assertions.assertFalse(index.getProperties().contains("nested-opaque")); + Assertions.assertFalse(index.getProperties().contains("credential")); + Assertions.assertFalse(index.getProperties().contains("num_partitions")); + + LanceLogicalIndex nullProperty = LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "null", Collections.singletonList(1), "IVF_PQ", + "{\"metric_type\":null}")), + fieldNames("embedding")).get(0); + Assertions.assertEquals("{}", nullProperty.getProperties()); + } + + @Test + public void testRejectsAllowlistedObjectAndArrayValues() { + for (String details : Arrays.asList( + "{\"metric_type\":{\"name\":\"cosine\"}}", + "{\"compression\":{\"num_bits\":[8]}}", + "{\"compression\":\"pq\"}", + "{\"hnsw\":[32]}")) { + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "idx", Collections.singletonList(1), "IVF_PQ", details)), + fieldNames("embedding"))); + Assertions.assertTrue(exception.getMessage().contains( + "Invalid Lance index details JSON for 'idx'")); + } + } + + @Test + public void testRejectsPropertiesOverFinalJsonLimit() { + String details = "{\"compression\":{\"type\":\"" + + repeat("x", 390) + "\"}}"; + Assertions.assertTrue(details.length() < 1024); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "idx", Collections.singletonList(1), "IVF_PQ", details)), + fieldNames("embedding"))); + Assertions.assertTrue(exception.getMessage().contains("400")); + Assertions.assertFalse(exception.getMessage().contains(repeat("x", 390))); + } + + @Test + public void testCredentialSentinelsNeverAppearInInvalidJsonExceptionChain() { + String accessKey = "SENTINEL_ACCESS_KEY"; + String secretKey = "SENTINEL_SECRET_KEY"; + String sessionToken = "SENTINEL_SESSION_TOKEN"; + String details = "{\"metric_type\":{\"access\":\"" + accessKey + + "\",\"secret\":\"" + secretKey + "\",\"session\":\"" + + sessionToken + "\"}}"; + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description( + "idx", Collections.singletonList(1), "IVF_PQ", details)), + fieldNames("embedding"))); + String stackTrace = stackTrace(exception); + Assertions.assertFalse(stackTrace.contains(accessKey)); + Assertions.assertFalse(stackTrace.contains(secretKey)); + Assertions.assertFalse(stackTrace.contains(sessionToken)); + Assertions.assertNull(exception.getCause()); + } + + private static void assertBoundFailure(IndexDescription description, + Map fields, String expectedType, String expectedLimit) { + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> LanceIndexMetadataLoader.normalize( + Collections.singletonList(description), fields)); + Assertions.assertTrue(exception.getMessage().contains(expectedType)); + Assertions.assertTrue(exception.getMessage().contains(expectedLimit)); + } + + private static IndexDescription description(String name, List fieldIds, + String indexType, String detailsJson) { + return new IndexDescription(name, fieldIds, "type.googleapis.com/lance.index", + indexType, 0, Collections.emptyList(), detailsJson); + } + + private static LanceField field(int id, String name) { + LanceField field = Mockito.mock(LanceField.class); + Mockito.when(field.getId()).thenReturn(id); + Mockito.when(field.getName()).thenReturn(name); + Mockito.when(field.getChildren()).thenReturn(Collections.emptyList()); + return field; + } + + private static LanceField nestedFieldChain(int depth) { + LanceField child = null; + for (int level = depth; level >= 1; --level) { + LanceField parent = field(level, "level_" + level); + if (child != null) { + Mockito.when(parent.getChildren()) + .thenReturn(Collections.singletonList(child)); + } + child = parent; + } + return child; + } + + private static Map fieldNames(String... names) { + Map fields = new HashMap<>(); + for (int index = 0; index < names.length; ++index) { + fields.put(index + 1, names[index]); + } + return fields; + } + + private static String repeat(String value, int count) { + return String.join("", Collections.nCopies(count, value)); + } + + private static String stackTrace(Throwable throwable) { + StringWriter writer = new StringWriter(); + throwable.printStackTrace(new PrintWriter(writer)); + return writer.toString(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommandTest.java index 28b0fece357601..3a4c433430e031 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCommandTest.java @@ -17,19 +17,75 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.lance.LanceExternalCatalog; +import org.apache.doris.datasource.lance.LanceExternalDatabase; +import org.apache.doris.datasource.lance.LanceLogicalIndex; +import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.info.TableNameInfo; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class ShowIndexCommandTest extends TestWithFeService { + private static final String INTERNAL_TABLE = "show_index_internal"; + private static final String EXTERNAL_CATALOG = "show_index_test_external"; + private static final String UNREACHABLE_LANCE_CATALOG = "show_index_unreachable_lance"; + private static final String REST_LANCE_CATALOG = "show_index_rest_lance"; @Override protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; createDatabase("test"); connectContext.setDatabase("test"); + createTable("CREATE TABLE test." + INTERNAL_TABLE + " (\n" + + " k1 INT,\n" + + " value STRING,\n" + + " INDEX idx_value(value) USING INVERTED COMMENT 'internal index'\n" + + ") DUPLICATE KEY(k1)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1')"); + createCatalog("CREATE CATALOG " + EXTERNAL_CATALOG + " PROPERTIES (\n" + + " 'type' = 'test',\n" + + " 'catalog_provider.class' = '" + + ShowIndexCommandTest.class.getName() + "$ExternalCatalogProvider'\n" + + ")"); + createCatalog("CREATE CATALOG " + UNREACHABLE_LANCE_CATALOG + " PROPERTIES (\n" + + " 'type' = 'lance',\n" + + " 'lance.catalog.type' = 'rest',\n" + + " 'lance.rest.uri' = 'http://127.0.0.1:1',\n" + + " 'test_connection' = 'false'\n" + + ")"); + createCatalog("CREATE CATALOG " + REST_LANCE_CATALOG + " PROPERTIES (\n" + + " 'type' = 'lance',\n" + + " 'lance.catalog.type' = 'rest',\n" + + " 'lance.rest.uri' = 'http://127.0.0.1:1',\n" + + " 'test_connection' = 'false'\n" + + ")"); } @Test @@ -50,9 +106,154 @@ void testAnalyze() throws Exception { Assertions.assertThrows(AnalysisException.class, () -> finalSi1.analyze(connectContext)); connectContext.setDatabase(null); - tableName = new TableNameInfo("", "test"); - si = new ShowIndexCommand(tableName); - ShowIndexCommand finalSi2 = si; - Assertions.assertThrows(AnalysisException.class, () -> finalSi2.analyze(connectContext)); + try { + tableName = new TableNameInfo("", "test"); + si = new ShowIndexCommand(tableName); + ShowIndexCommand finalSi2 = si; + Assertions.assertThrows(AnalysisException.class, () -> finalSi2.analyze(connectContext)); + } finally { + connectContext.setDatabase("test"); + } + } + + @Test + void testInternalIndexRowsRemainUnchanged() throws Exception { + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("test"); + OlapTable table = (OlapTable) db.getTableOrAnalysisException(INTERNAL_TABLE); + List indexes = table.getIndexes(); + Assertions.assertEquals(1, indexes.size()); + Index index = indexes.get(0); + + ShowIndexCommand command = new ShowIndexCommand( + new TableNameInfo(InternalCatalog.INTERNAL_CATALOG_NAME, "test", INTERNAL_TABLE)); + ShowResultSet result = command.doRun(connectContext, null); + + Assertions.assertEquals(Collections.singletonList(Lists.newArrayList( + INTERNAL_TABLE, "", index.getIndexName(), "", String.join(",", index.getColumns()), + "", "", "", "", "", index.getIndexType().name(), index.getComment(), + index.getPropertiesString())), result.getResultRows()); + } + + @Test + void testBuildLanceRowsMapsAllThirteenColumns() { + LanceLogicalIndex index = new LanceLogicalIndex( + "VectorIndex", Collections.singletonList("embedding"), "IVF_PQ", + "{\"metric_type\":\"cosine\"}"); + + List> rows = ShowIndexCommand.buildLanceRows( + "documents", Collections.singletonList(index)); + + Assertions.assertEquals(1, rows.size()); + Assertions.assertEquals(13, rows.get(0).size()); + Assertions.assertEquals(Arrays.asList( + "documents", "", "VectorIndex", "1", "embedding", "", "", "", "", "", + "IVF_PQ", "", "{\"metric_type\":\"cosine\"}"), rows.get(0)); + } + + @Test + void testBuildLanceRowsExpandsCompositeIndexWithoutReorderingInput() { + LanceLogicalIndex first = new LanceLogicalIndex( + "z_index", Arrays.asList("first", "second", "third"), "BTREE", "{}"); + LanceLogicalIndex second = new LanceLogicalIndex( + "a_index", Collections.singletonList("fourth"), "BITMAP", "{}"); + List indexes = Lists.newArrayList(first, second); + + List> rows = ShowIndexCommand.buildLanceRows("events", indexes); + + Assertions.assertSame(first, indexes.get(0)); + Assertions.assertSame(second, indexes.get(1)); + Assertions.assertEquals(4, rows.size()); + Assertions.assertEquals(Arrays.asList("z_index", "z_index", "z_index", "a_index"), + Arrays.asList(rows.get(0).get(2), rows.get(1).get(2), rows.get(2).get(2), rows.get(3).get(2))); + Assertions.assertEquals(Arrays.asList("1", "2", "3", "1"), + Arrays.asList(rows.get(0).get(3), rows.get(1).get(3), rows.get(2).get(3), rows.get(3).get(3))); + Assertions.assertEquals(Arrays.asList("first", "second", "third", "fourth"), + Arrays.asList(rows.get(0).get(4), rows.get(1).get(4), rows.get(2).get(4), rows.get(3).get(4))); + } + + @Test + void testOtherExternalCatalogStillReturnsEmptyWithoutResolvingTable() throws Exception { + ShowIndexCommand command = new ShowIndexCommand( + new TableNameInfo(EXTERNAL_CATALOG, "external_db", "missing_table")); + + ShowResultSet result = command.doRun(connectContext, null); + + Assertions.assertTrue(result.getResultRows().isEmpty()); + } + + @Test + void testDeniedUserDoesNotInitializeUnreachableLanceCatalog() throws Exception { + LanceExternalCatalog catalog = (LanceExternalCatalog) Env.getCurrentEnv().getCatalogMgr() + .getCatalog(UNREACHABLE_LANCE_CATALOG); + Assertions.assertFalse(catalog.isInitialized()); + ConnectContext deniedContext = createCtx( + UserIdentity.createAnalyzedUserIdentWithIp("show_index_denied_user", "%"), "127.0.0.1"); + try { + ShowIndexCommand command = new ShowIndexCommand( + new TableNameInfo(UNREACHABLE_LANCE_CATALOG, "unreachable_db", "unreachable_table")); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, () -> command.doRun(deniedContext, null)); + + Assertions.assertTrue(exception.getMessage().contains("denied")); + Assertions.assertFalse(catalog.isInitialized()); + } finally { + connectContext.setThreadLocalInfo(); + } + } + + @Test + void testAuthorizedLanceRestRejectedBeforeCatalogInitialization() throws Exception { + LanceExternalCatalog catalog = (LanceExternalCatalog) Env.getCurrentEnv().getCatalogMgr() + .getCatalog(REST_LANCE_CATALOG); + Assertions.assertFalse(catalog.isInitialized()); + ShowIndexCommand command = new ShowIndexCommand( + new TableNameInfo(REST_LANCE_CATALOG, "unreachable_db", "unreachable_table")); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, () -> command.doRun(connectContext, null)); + + Assertions.assertEquals( + "SHOW INDEX is not supported for Lance REST catalogs", exception.getDetailMessage()); + Assertions.assertFalse(catalog.isInitialized()); + } + + @Test + void testNonLanceTableInLanceCatalogRejected() throws Exception { + Env mockedEnvironment = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class); + LanceExternalDatabase database = Mockito.mock(LanceExternalDatabase.class); + TableIf notLanceTable = Mockito.mock(TableIf.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(mockedEnvironment); + Mockito.when(mockedEnvironment.getAccessManager()).thenReturn(accessManager); + Mockito.when(accessManager.checkTblPriv( + Mockito.any(ConnectContext.class), Mockito.eq("lance_fs"), Mockito.eq("db"), + Mockito.eq("table"), + Mockito.eq(PrivPredicate.SHOW))).thenReturn(true); + Mockito.when(mockedEnvironment.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalogOrAnalysisException("lance_fs")).thenReturn(catalog); + Mockito.when(catalog.isRestCatalogConfigured()).thenReturn(false); + Mockito.doReturn(database).when(catalog).getDbOrAnalysisException("db"); + Mockito.doReturn(notLanceTable).when(database).getTableOrAnalysisException("table"); + ShowIndexCommand command = new ShowIndexCommand( + new TableNameInfo("lance_fs", "db", "table")); + + AnalysisException exception = Assertions.assertThrows( + AnalysisException.class, () -> command.doRun(connectContext, null)); + + Assertions.assertEquals("Table table is not a Lance table", exception.getDetailMessage()); + } + } + + public static class ExternalCatalogProvider implements TestExternalCatalog.TestCatalogProvider { + @Override + public Map>> getMetadata() { + Map>> metadata = new HashMap<>(); + metadata.put("external_db", Collections.emptyMap()); + return metadata; + } } } diff --git a/regression-test/data/external_table_p0/lance/test_lance_show_index.out b/regression-test/data/external_table_p0/lance/test_lance_show_index.out new file mode 100644 index 00000000000000..d7b7f1e9f1e02a --- /dev/null +++ b/regression-test/data/external_table_p0/lance/test_lance_show_index.out @@ -0,0 +1,18 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !show_index -- +vs_ivf_pq_f32 embedding_ivf_pq_f32 1 embedding IVF_PQ {"compression":{"num_bits":4,"num_sub_vectors":4,"type":"pq"},"metric_type":"L2"} + +-- !show_indexes -- +vs_ivf_pq_f32 embedding_ivf_pq_f32 1 embedding IVF_PQ {"compression":{"num_bits":4,"num_sub_vectors":4,"type":"pq"},"metric_type":"L2"} + +-- !show_key -- +vs_ivf_pq_f32 embedding_ivf_pq_f32 1 embedding IVF_PQ {"compression":{"num_bits":4,"num_sub_vectors":4,"type":"pq"},"metric_type":"L2"} + +-- !show_keys -- +vs_ivf_pq_f32 embedding_ivf_pq_f32 1 embedding IVF_PQ {"compression":{"num_bits":4,"num_sub_vectors":4,"type":"pq"},"metric_type":"L2"} + +-- !nested_index -- +nested_index nested_label_btree 1 attributes.`child.with.dot` BTree {} + +-- !no_indexes -- + diff --git a/regression-test/suites/external_table_p0/lance/test_lance_show_index.groovy b/regression-test/suites/external_table_p0/lance/test_lance_show_index.groovy new file mode 100644 index 00000000000000..36908c36e68475 --- /dev/null +++ b/regression-test/suites/external_table_p0/lance/test_lance_show_index.groovy @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lance_show_index", "p0,external") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Lance SHOW INDEX test because the Iceberg MinIO environment is disabled.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String lanceRestPort = context.config.otherConfigs.get("lance_rest_port") + String filesystemCatalog = "test_lance_show_index" + String restCatalog = "test_lance_show_index_rest" + String user = "test_lance_show_index_user" + String password = "C123_567p" + + sql """DROP CATALOG IF EXISTS `${filesystemCatalog}`""" + sql """DROP CATALOG IF EXISTS `${restCatalog}`""" + try_sql "DROP USER '${user}'@'%'" + + try { + sql """ + CREATE CATALOG `${filesystemCatalog}` PROPERTIES ( + "type" = "lance", + "lance.catalog.type" = "filesystem", + "warehouse" = "s3://warehouse/lance", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.region" = "us-east-1", + "use_path_style" = "true" + ) + """ + + order_qt_show_index """SHOW INDEX FROM `${filesystemCatalog}`.`doris`.`vs_ivf_pq_f32`""" + order_qt_show_indexes """SHOW INDEXES FROM `${filesystemCatalog}`.`doris`.`vs_ivf_pq_f32`""" + order_qt_show_key """SHOW KEY FROM `${filesystemCatalog}`.`doris`.`vs_ivf_pq_f32`""" + order_qt_show_keys """SHOW KEYS FROM `${filesystemCatalog}`.`doris`.`vs_ivf_pq_f32`""" + order_qt_nested_index """SHOW INDEX FROM `${filesystemCatalog}`.`doris`.`nested_index`""" + order_qt_no_indexes """SHOW INDEX FROM `${filesystemCatalog}`.`doris`.`predicate_pushdown`""" + + sql """ + CREATE CATALOG `${restCatalog}` PROPERTIES ( + "type" = "lance", + "lance.catalog.type" = "rest", + "lance.rest.uri" = "http://${externalEnvIp}:${lanceRestPort}", + "lance.rest.security.type" = "bearer", + "lance.rest.bearer-token" = "doris-lance-rest-test-token", + "lance.namespace.root_database" = "default", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "use_path_style" = "true", + "test_connection" = "true" + ) + """ + + test { + sql """SHOW INDEX FROM `${restCatalog}`.`default`.`all_types`""" + exception "SHOW INDEX is not supported for Lance REST catalogs" + } + + sql """CREATE USER '${user}'@'%' IDENTIFIED BY '${password}'""" + sql """GRANT SELECT_PRIV ON regression_test TO '${user}'@'%'""" + if (isCloudMode()) { + def clusters = sql "SHOW CLUSTERS" + assertTrue(!clusters.isEmpty()) + sql """GRANT USAGE_PRIV ON CLUSTER `${clusters[0][0]}` TO '${user}'@'%'""" + } + + connect(user, password, context.config.jdbcUrl) { + test { + sql """SHOW INDEX FROM `${filesystemCatalog}`.`doris`.`vs_ivf_pq_f32`""" + exception "denied" + } + } + } finally { + try_sql "DROP USER '${user}'@'%'" + // Keep both catalogs for debugging when the suite fails. + } +}