diff --git a/src/datasets/io/parquet.py b/src/datasets/io/parquet.py index b7b13b6b8a4..e2617658b02 100644 --- a/src/datasets/io/parquet.py +++ b/src/datasets/io/parquet.py @@ -7,6 +7,7 @@ from .. import Dataset, Features, NamedSplit, config from ..arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features +from ..features.features import require_storage_embed from ..formatting import query_table from ..packaged_modules import _PACKAGED_DATASETS_MODULES from ..packaged_modules.parquet.parquet import Parquet @@ -126,6 +127,16 @@ def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) - schema=schema, use_content_defined_chunking=self.use_content_defined_chunking, write_page_index=self.write_page_index, + compression={ + col: "none" if require_storage_embed(feature) else "snappy" + for col, feature in self.dataset.features.items() + }, + use_dictionary=[ + col for col, feature in self.dataset.features.items() if not require_storage_embed(feature) + ], + column_encoding={ + col: "PLAIN" for col, feature in self.dataset.features.items() if require_storage_embed(feature) + }, **parquet_writer_kwargs, )