From 58929da2fecf1aca30bb44c4739f53e38b132853 Mon Sep 17 00:00:00 2001 From: Renze Post Date: Tue, 16 Jan 2024 15:22:14 +0100 Subject: [PATCH 1/6] Change default parquet_row_group_size in BaseSQLToGCSOperator --- airflow/providers/google/cloud/transfers/sql_to_gcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/google/cloud/transfers/sql_to_gcs.py b/airflow/providers/google/cloud/transfers/sql_to_gcs.py index dcadaf7859dc9..1529430c97a07 100644 --- a/airflow/providers/google/cloud/transfers/sql_to_gcs.py +++ b/airflow/providers/google/cloud/transfers/sql_to_gcs.py @@ -85,7 +85,7 @@ class BaseSQLToGCSOperator(BaseOperator): :param parquet_row_group_size: The approximate number of rows in each row group when using parquet format. Using a large row group size can reduce the file size and improve the performance of reading the data, but it needs more memory to - execute the operator. (default: 1) + execute the operator. (default: 100000) """ template_fields: Sequence[str] = ( @@ -123,7 +123,7 @@ def __init__( exclude_columns: set | None = None, partition_columns: list | None = None, write_on_empty: bool = False, - parquet_row_group_size: int = 1, + parquet_row_group_size: int = 100000, **kwargs, ) -> None: super().__init__(**kwargs) From 677c252312b651dd88d03c05c8fb9e5a07f56e8a Mon Sep 17 00:00:00 2001 From: Renze Post Date: Wed, 17 Jan 2024 12:34:10 +0100 Subject: [PATCH 2/6] Add change to changelog --- airflow/providers/google/CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 3187a397b4f79..02d1840aafc98 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -26,6 +26,7 @@ Changelog --------- +* ``Change default parquet_row_group_size in BaseSQLToGCSOperator (#36817)`` 10.13.1 ....... From 0f365408849ab6e3748fd8e1967012bc64d1b5a9 Mon Sep 17 00:00:00 2001 From: Renze Post Date: Wed, 17 Jan 2024 17:27:10 +0100 Subject: [PATCH 3/6] Added a better change description --- airflow/providers/google/CHANGELOG.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 02d1840aafc98..df789144467b3 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -26,7 +26,15 @@ Changelog --------- -* ``Change default parquet_row_group_size in BaseSQLToGCSOperator (#36817)`` +The default value of ``parquet_row_group_size`` in ``BaseSQLToGCSOperator`` has changed from 1 to +100000, in order to have a default that provides better compression efficiency and performance of +reading the data in the output Parquet files. In many cases, the previous value of 1 resulted in +very large files, long task durations and out of memory issues. A default value of 100000 may require +more memory to execute the operator, in which case users can override the ``parquet_row_group_size`` +parameter in the operator. All operators that are derived from ``BaseSQLToGCSOperator`` are affected +when ``export_format`` is ``parquet``: ``MySQLToGCSOperator``, ``PrestoToGCSOperator``, +``OracleToGCSOperator``, ``TrinoToGCSOperator``, ``MSSQLToGCSOperator`` and ``PostgresToGCSOperator``. + 10.13.1 ....... From b526985d46a278e63c72dcbe3bf6b599515420e3 Mon Sep 17 00:00:00 2001 From: Renze Post Date: Wed, 17 Jan 2024 17:30:10 +0100 Subject: [PATCH 4/6] Remove unnecessary extra newline --- airflow/providers/google/CHANGELOG.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index df789144467b3..76e1ad3281fd1 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -35,7 +35,6 @@ parameter in the operator. All operators that are derived from ``BaseSQLToGCSOpe when ``export_format`` is ``parquet``: ``MySQLToGCSOperator``, ``PrestoToGCSOperator``, ``OracleToGCSOperator``, ``TrinoToGCSOperator``, ``MSSQLToGCSOperator`` and ``PostgresToGCSOperator``. - 10.13.1 ....... From a4d278e449d906170ee4dfe089c70bc6c0d069a7 Mon Sep 17 00:00:00 2001 From: Renze Post Date: Thu, 18 Jan 2024 12:54:51 +0100 Subject: [PATCH 5/6] Applied suggested changes Co-authored-by: Andrey Anshin --- airflow/providers/google/CHANGELOG.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 76e1ad3281fd1..0c242cc3de6c4 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -26,14 +26,16 @@ Changelog --------- -The default value of ``parquet_row_group_size`` in ``BaseSQLToGCSOperator`` has changed from 1 to -100000, in order to have a default that provides better compression efficiency and performance of -reading the data in the output Parquet files. In many cases, the previous value of 1 resulted in -very large files, long task durations and out of memory issues. A default value of 100000 may require -more memory to execute the operator, in which case users can override the ``parquet_row_group_size`` -parameter in the operator. All operators that are derived from ``BaseSQLToGCSOperator`` are affected -when ``export_format`` is ``parquet``: ``MySQLToGCSOperator``, ``PrestoToGCSOperator``, -``OracleToGCSOperator``, ``TrinoToGCSOperator``, ``MSSQLToGCSOperator`` and ``PostgresToGCSOperator``. + +.. note:: + The default value of ``parquet_row_group_size`` in ``BaseSQLToGCSOperator`` has changed from 1 to + 100000, in order to have a default that provides better compression efficiency and performance of + reading the data in the output Parquet files. In many cases, the previous value of 1 resulted in + very large files, long task durations and out of memory issues. A default value of 100000 may require + more memory to execute the operator, in which case users can override the ``parquet_row_group_size`` + parameter in the operator. All operators that are derived from ``BaseSQLToGCSOperator`` are affected + when ``export_format`` is ``parquet``: ``MySQLToGCSOperator``, ``PrestoToGCSOperator``, + ``OracleToGCSOperator``, ``TrinoToGCSOperator``, ``MSSQLToGCSOperator`` and ``PostgresToGCSOperator``. 10.13.1 ....... From 5a5d7b1874de44ac07d7d91cf967116c3b497719 Mon Sep 17 00:00:00 2001 From: Renze Post Date: Thu, 18 Jan 2024 13:20:45 +0100 Subject: [PATCH 6/6] Applied suggested changes Co-authored-by: Elad Kalif <45845474+eladkal@users.noreply.github.com> --- airflow/providers/google/CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 0c242cc3de6c4..9249209235a76 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -35,7 +35,7 @@ Changelog more memory to execute the operator, in which case users can override the ``parquet_row_group_size`` parameter in the operator. All operators that are derived from ``BaseSQLToGCSOperator`` are affected when ``export_format`` is ``parquet``: ``MySQLToGCSOperator``, ``PrestoToGCSOperator``, - ``OracleToGCSOperator``, ``TrinoToGCSOperator``, ``MSSQLToGCSOperator`` and ``PostgresToGCSOperator``. + ``OracleToGCSOperator``, ``TrinoToGCSOperator``, ``MSSQLToGCSOperator`` and ``PostgresToGCSOperator``. Due to the above we treat this change as bug fix. 10.13.1 .......