From a9ce146935e6a6f6a8c1e79bb79ccab10d5ddf6d Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Fri, 30 Apr 2021 10:48:30 +0800 Subject: [PATCH 01/18] Update s3.py Not removing the trailing slash to s3 to avoid over matching the files --- airflow/providers/amazon/aws/hooks/s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index 2e18754d51d53..39b4ac1e57fa0 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -143,9 +143,9 @@ def parse_s3_url(s3url: str) -> Tuple[str, str]: raise AirflowException(f'Please provide a bucket_name instead of "{s3url}"') bucket_name = parsed_url.netloc - key = parsed_url.path.strip('/') + # key = parsed_url.path.strip('/') - return bucket_name, key + return bucket_name, parsed_url.path @provide_bucket_name def check_for_bucket(self, bucket_name: Optional[str] = None) -> bool: From 9f65ff715849403e160891d7bded487090823164 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Wed, 5 May 2021 18:29:49 +0800 Subject: [PATCH 02/18] Update airflow/providers/amazon/aws/hooks/s3.py Co-authored-by: Felix Uellendall --- airflow/providers/amazon/aws/hooks/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index 39b4ac1e57fa0..a5aabb785aeee 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -143,7 +143,7 @@ def parse_s3_url(s3url: str) -> Tuple[str, str]: raise AirflowException(f'Please provide a bucket_name instead of "{s3url}"') bucket_name = parsed_url.netloc - # key = parsed_url.path.strip('/') + key = parsed_url.path return bucket_name, parsed_url.path From e603f1ec7eceed9de687b0b7b3f27e864825cebc Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Wed, 5 May 2021 18:29:56 +0800 Subject: [PATCH 03/18] Update airflow/providers/amazon/aws/hooks/s3.py Co-authored-by: Felix Uellendall --- airflow/providers/amazon/aws/hooks/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index a5aabb785aeee..f5808258b7f95 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -145,7 +145,7 @@ def parse_s3_url(s3url: str) -> Tuple[str, str]: bucket_name = parsed_url.netloc key = parsed_url.path - return bucket_name, parsed_url.path + return bucket_name, key @provide_bucket_name def check_for_bucket(self, bucket_name: Optional[str] = None) -> bool: From cf1cbbf31a6d43ca758c156e35e2e2c29fbef43c Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Thu, 6 May 2021 22:24:54 +0800 Subject: [PATCH 04/18] Update test_s3.py --- tests/providers/amazon/aws/hooks/test_s3.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/providers/amazon/aws/hooks/test_s3.py b/tests/providers/amazon/aws/hooks/test_s3.py index b56d4fca7bf1e..0f7da9b5023d7 100644 --- a/tests/providers/amazon/aws/hooks/test_s3.py +++ b/tests/providers/amazon/aws/hooks/test_s3.py @@ -69,6 +69,10 @@ def test_parse_s3_url(self): parsed = S3Hook.parse_s3_url("s3://test/this/is/not/a-real-key.txt") assert parsed == ("test", "this/is/not/a-real-key.txt"), "Incorrect parsing of the s3 url" + def test_parse_s3_object_directory(self): + parsed = S3Hook.parse_s3_url("s3://test/this/is/not/a-real-s3-directory/") + assert parsed == ("test", "this/is/not/a-real-s3-directory/"), "Incorrect parsing of the s3 url" + def test_check_for_bucket(self, s3_bucket): hook = S3Hook() assert hook.check_for_bucket(s3_bucket) is True From 66454fcc0b248dc2ab769b246bcc1e7cf1bafe40 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Thu, 6 May 2021 23:17:10 +0800 Subject: [PATCH 05/18] Update s3.py --- airflow/providers/amazon/aws/hooks/s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index f5808258b7f95..25c015e95dfac 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -143,9 +143,9 @@ def parse_s3_url(s3url: str) -> Tuple[str, str]: raise AirflowException(f'Please provide a bucket_name instead of "{s3url}"') bucket_name = parsed_url.netloc - key = parsed_url.path + # key = parsed_url.path - return bucket_name, key + return bucket_name, parsed_url.path @provide_bucket_name def check_for_bucket(self, bucket_name: Optional[str] = None) -> bool: From d8701c01545ecdabdef4d2f60d7c6f053387a1b7 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Fri, 7 May 2021 06:06:37 +0800 Subject: [PATCH 06/18] Update test_s3.py --- tests/providers/amazon/aws/hooks/test_s3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/providers/amazon/aws/hooks/test_s3.py b/tests/providers/amazon/aws/hooks/test_s3.py index 0f7da9b5023d7..e2ebf50d91f81 100644 --- a/tests/providers/amazon/aws/hooks/test_s3.py +++ b/tests/providers/amazon/aws/hooks/test_s3.py @@ -71,8 +71,10 @@ def test_parse_s3_url(self): def test_parse_s3_object_directory(self): parsed = S3Hook.parse_s3_url("s3://test/this/is/not/a-real-s3-directory/") + assert parsed != ("test", "this/is/not/a-real-s3-directory"), "Incorrect parsing of the s3 url" assert parsed == ("test", "this/is/not/a-real-s3-directory/"), "Incorrect parsing of the s3 url" + def test_check_for_bucket(self, s3_bucket): hook = S3Hook() assert hook.check_for_bucket(s3_bucket) is True From 7281e423d758b25bba138e7bc8fd1e9c06e4af27 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Mon, 10 May 2021 11:24:05 +0800 Subject: [PATCH 07/18] Update test_s3.py --- tests/providers/amazon/aws/hooks/test_s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/providers/amazon/aws/hooks/test_s3.py b/tests/providers/amazon/aws/hooks/test_s3.py index e2ebf50d91f81..41328dc7aa20f 100644 --- a/tests/providers/amazon/aws/hooks/test_s3.py +++ b/tests/providers/amazon/aws/hooks/test_s3.py @@ -74,7 +74,6 @@ def test_parse_s3_object_directory(self): assert parsed != ("test", "this/is/not/a-real-s3-directory"), "Incorrect parsing of the s3 url" assert parsed == ("test", "this/is/not/a-real-s3-directory/"), "Incorrect parsing of the s3 url" - def test_check_for_bucket(self, s3_bucket): hook = S3Hook() assert hook.check_for_bucket(s3_bucket) is True From 7a4da1894d9b468eacfc0ebac4aace3168ba14c5 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Mon, 10 May 2021 11:24:21 +0800 Subject: [PATCH 08/18] Create test_s3.py From 124e086cc2f6953d8448da45c5f9096d2fe9f8c5 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Fri, 14 May 2021 15:53:11 +0800 Subject: [PATCH 09/18] Update s3.py --- airflow/providers/amazon/aws/hooks/s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index 25c015e95dfac..f5808258b7f95 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -143,9 +143,9 @@ def parse_s3_url(s3url: str) -> Tuple[str, str]: raise AirflowException(f'Please provide a bucket_name instead of "{s3url}"') bucket_name = parsed_url.netloc - # key = parsed_url.path + key = parsed_url.path - return bucket_name, parsed_url.path + return bucket_name, key @provide_bucket_name def check_for_bucket(self, bucket_name: Optional[str] = None) -> bool: From 6defbdc7cb276f6df7b4020e08564f7ecab3f000 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Fri, 14 May 2021 15:59:10 +0800 Subject: [PATCH 10/18] Update UPDATING.md --- UPDATING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/UPDATING.md b/UPDATING.md index 24435b5e0db9f..cf520de27ea7c 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -100,6 +100,10 @@ serve as a DagBag cache burst time. The `default_queue` configuration option has been moved from `[celery]` section to `[operators]` section to allow for re-use between different executors. +### Update `parse_s3_url` in airflow.providers.amazon.aws.hooks.s3.S3Hook + +The `parse_s3_url` method in `airflow.providers.amazon.aws.hooks.s3.S3Hook` will now not remove the trailing slash by default. This will allow user to more flexibly implement their preferred prefix-level or subdirectory-level s3 operations. + ## Airflow 2.0.2 ### Default `[kubernetes] enable_tcp_keepalive` is changed to `True` From 0e36c78dfdfa7ee941bddc644cd432c29f3529e3 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Fri, 14 May 2021 16:01:46 +0800 Subject: [PATCH 11/18] Update test_s3.py --- tests/providers/amazon/aws/hooks/test_s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/providers/amazon/aws/hooks/test_s3.py b/tests/providers/amazon/aws/hooks/test_s3.py index 41328dc7aa20f..0f7da9b5023d7 100644 --- a/tests/providers/amazon/aws/hooks/test_s3.py +++ b/tests/providers/amazon/aws/hooks/test_s3.py @@ -71,7 +71,6 @@ def test_parse_s3_url(self): def test_parse_s3_object_directory(self): parsed = S3Hook.parse_s3_url("s3://test/this/is/not/a-real-s3-directory/") - assert parsed != ("test", "this/is/not/a-real-s3-directory"), "Incorrect parsing of the s3 url" assert parsed == ("test", "this/is/not/a-real-s3-directory/"), "Incorrect parsing of the s3 url" def test_check_for_bucket(self, s3_bucket): From ce40db83625d9a488e192d3a71f79cd40733a00c Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Fri, 14 May 2021 17:59:59 +0800 Subject: [PATCH 12/18] Update s3.py --- airflow/providers/amazon/aws/hooks/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py index f5808258b7f95..990ee829d97e7 100644 --- a/airflow/providers/amazon/aws/hooks/s3.py +++ b/airflow/providers/amazon/aws/hooks/s3.py @@ -143,7 +143,7 @@ def parse_s3_url(s3url: str) -> Tuple[str, str]: raise AirflowException(f'Please provide a bucket_name instead of "{s3url}"') bucket_name = parsed_url.netloc - key = parsed_url.path + key = parsed_url.path.lstrip('/') return bucket_name, key From 14ae122cebb9f8f9abd903148503282325aa1c71 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Tue, 18 May 2021 22:54:03 +0800 Subject: [PATCH 13/18] Update CHANGELOG.rst --- airflow/providers/amazon/CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index dc2fdcd26edc8..45ac856415ba4 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -19,6 +19,10 @@ Changelog --------- +2.0.0 + +* ``Update `parse_s3_url` in airflow.providers.amazon.aws.hooks.s3.S3Hook(#15609) + 1.3.0 ..... From 73ccec19cb055aa788f23b7ea67e153ef1292550 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Tue, 18 May 2021 22:55:00 +0800 Subject: [PATCH 14/18] Update UPDATING.md --- UPDATING.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index cf520de27ea7c..24435b5e0db9f 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -100,10 +100,6 @@ serve as a DagBag cache burst time. The `default_queue` configuration option has been moved from `[celery]` section to `[operators]` section to allow for re-use between different executors. -### Update `parse_s3_url` in airflow.providers.amazon.aws.hooks.s3.S3Hook - -The `parse_s3_url` method in `airflow.providers.amazon.aws.hooks.s3.S3Hook` will now not remove the trailing slash by default. This will allow user to more flexibly implement their preferred prefix-level or subdirectory-level s3 operations. - ## Airflow 2.0.2 ### Default `[kubernetes] enable_tcp_keepalive` is changed to `True` From 8c893031bc866f03f735dbf6f593ad7476e556fa Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Tue, 18 May 2021 22:57:20 +0800 Subject: [PATCH 15/18] Update CHANGELOG.rst --- airflow/providers/amazon/CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index 45ac856415ba4..d7cb16fd2a184 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -20,6 +20,10 @@ Changelog --------- 2.0.0 +..... + +Bug fixes +~~~~~~~~~ * ``Update `parse_s3_url` in airflow.providers.amazon.aws.hooks.s3.S3Hook(#15609) From f74102be059b90d7ac5c4602867e9a293a530d36 Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Wed, 19 May 2021 17:17:51 +0800 Subject: [PATCH 16/18] Update provider.yaml --- airflow/providers/amazon/provider.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index fb5f67f1b34fc..b619a40409eda 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -22,6 +22,7 @@ description: | Amazon integration (including `Amazon Web Services (AWS) `__). versions: + - 2.0.0 - 1.3.0 - 1.2.0 - 1.1.0 From ef0a8d43b75c87e8cd5e8086a83b78573bcbd11e Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Wed, 26 May 2021 19:57:13 +0800 Subject: [PATCH 17/18] Update CHANGELOG.rst --- airflow/providers/amazon/CHANGELOG.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index 194d08379035a..e2cc824847e36 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -19,15 +19,6 @@ Changelog --------- -2.0.0 -..... - -Bug fixes -~~~~~~~~~ - -* ``parse_s3_url in airflow.providers.amazon.aws.hooks.s3.S3Hook(#15609) should accept trailing slash`` - -======= 1.4.0 ..... From a6da71df760730b71c33efb45e641ed4c33c9f4f Mon Sep 17 00:00:00 2001 From: Isaacwhyuenac <13378284+Isaacwhyuenac@users.noreply.github.com> Date: Wed, 26 May 2021 19:57:59 +0800 Subject: [PATCH 18/18] Update provider.yaml --- airflow/providers/amazon/provider.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index f6e3990dc35d9..4144f486c6be7 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -22,7 +22,6 @@ description: | Amazon integration (including `Amazon Web Services (AWS) `__). versions: - - 2.0.0 - 1.4.0 - 1.3.0 - 1.2.0