From 21cb3c62521c4f70d1e6a2630aed31396328a1b6 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Sat, 31 Jan 2026 12:48:49 +0100 Subject: [PATCH 1/2] plain encoding, uncompressed, no dict for videos in parquet --- src/datasets/io/parquet.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/datasets/io/parquet.py b/src/datasets/io/parquet.py index b7b13b6b8a4..7f53d9491de 100644 --- a/src/datasets/io/parquet.py +++ b/src/datasets/io/parquet.py @@ -7,6 +7,7 @@ from .. import Dataset, Features, NamedSplit, config from ..arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features +from ..features.features import require_storage_embed from ..formatting import query_table from ..packaged_modules import _PACKAGED_DATASETS_MODULES from ..packaged_modules.parquet.parquet import Parquet @@ -126,6 +127,13 @@ def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) - schema=schema, use_content_defined_chunking=self.use_content_defined_chunking, write_page_index=self.write_page_index, + compression={ + col: "none" if require_storage_embed(feature) else "snappy" for col, feature in self.dataset.features.items() + }, + use_dictionary=[col for col, feature in self.dataset.features.items() if not require_storage_embed(feature)], + column_encoding={ + col: "PLAIN" for col, feature in self.dataset.features.items() if require_storage_embed(feature) + }, **parquet_writer_kwargs, ) From 1b8bd00b85ef664851b844ffc4588d61879b564b Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Sat, 31 Jan 2026 12:48:58 +0100 Subject: [PATCH 2/2] style --- src/datasets/io/parquet.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/datasets/io/parquet.py b/src/datasets/io/parquet.py index 7f53d9491de..e2617658b02 100644 --- a/src/datasets/io/parquet.py +++ b/src/datasets/io/parquet.py @@ -128,9 +128,12 @@ def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) - use_content_defined_chunking=self.use_content_defined_chunking, write_page_index=self.write_page_index, compression={ - col: "none" if require_storage_embed(feature) else "snappy" for col, feature in self.dataset.features.items() + col: "none" if require_storage_embed(feature) else "snappy" + for col, feature in self.dataset.features.items() }, - use_dictionary=[col for col, feature in self.dataset.features.items() if not require_storage_embed(feature)], + use_dictionary=[ + col for col, feature in self.dataset.features.items() if not require_storage_embed(feature) + ], column_encoding={ col: "PLAIN" for col, feature in self.dataset.features.items() if require_storage_embed(feature) },