From 07504ce0566289ced975fe08b9e91e3f22f91cdf Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Mon, 14 Nov 2022 14:42:06 +0530 Subject: [PATCH 01/34] Added new configs for fixed SAS Token --- .../hadoop/fs/azurebfs/constants/ConfigurationKeys.java | 4 ++++ .../hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 9d3b2d5e82c6e3..7571a7dd21817c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -230,6 +230,10 @@ public static String accountProperty(String property, String account) { public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token"; public static final String FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE = "fs.azure.delegation.token.provider.type"; + /** Key for fixed SAS token **/ + public static final String FS_AZURE_SAS_FIXED_TOKEN = "fs.azure.sas.fixed.token"; + /** Key for fixed SAS token provider class **/ + public static final String FS_AZURE_SAS_FIXED_TOKEN_PROVIDER = "fs.azure.sas.fixed.token.provider"; /** Key for SAS token provider **/ public static final String FS_AZURE_SAS_TOKEN_PROVIDER_TYPE = "fs.azure.sas.token.provider.type"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java new file mode 100644 index 00000000000000..861b02b26be0b0 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java @@ -0,0 +1,2 @@ +package org.apache.hadoop.fs.azurebfs.extensions;public class FixedSASTokenProvider { +} From 289b99399e152abc933cc14e4295c1cabaf41881 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 15 Nov 2022 11:12:04 +0530 Subject: [PATCH 02/34] Modified config keys for fixed sas token --- .../apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 7571a7dd21817c..32c54f7c0ee8ff 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -232,8 +232,7 @@ public static String accountProperty(String property, String account) { /** Key for fixed SAS token **/ public static final String FS_AZURE_SAS_FIXED_TOKEN = "fs.azure.sas.fixed.token"; - /** Key for fixed SAS token provider class **/ - public static final String FS_AZURE_SAS_FIXED_TOKEN_PROVIDER = "fs.azure.sas.fixed.token.provider"; + /** Key for SAS token provider **/ public static final String FS_AZURE_SAS_TOKEN_PROVIDER_TYPE = "fs.azure.sas.token.provider.type"; From 087c6b740eb6dd41d7b2634646c33168d70f4826 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 15 Nov 2022 11:15:39 +0530 Subject: [PATCH 03/34] Added new FixedTokenProvider class to read fixed SAS --- .../extensions/FixedSASTokenProvider.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java index 861b02b26be0b0..7d4be92e5c6f89 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java @@ -1,2 +1,28 @@ -package org.apache.hadoop.fs.azurebfs.extensions;public class FixedSASTokenProvider { +package org.apache.hadoop.fs.azurebfs.extensions; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; +import org.apache.hadoop.security.AccessControlException; + +import java.io.IOException; + +public class FixedSASTokenProvider implements SASTokenProvider{ + Configuration configuration; + String accountName; + + @Override + public void initialize(Configuration configuration, String accountName) throws IOException { + this.configuration = configuration; + this.accountName = accountName; + } + + @Override + public String getSASToken(String account, String fileSystem, String path, String operation) throws IOException { + String fixedToken = configuration.get(ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN, null); + if (fixedToken == null) + throw new InvalidConfigurationValueException("The fixed SAS Token configuration value is invalid."); + else + return fixedToken; + } } From ac827302a0076241bbd2b2d03d25c0212c5f9ea0 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 15 Nov 2022 11:16:37 +0530 Subject: [PATCH 04/34] Enabled passing tracingContext --- .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 11 ++++++----- .../hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index d0bdd9818db24b..e5758294127510 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -191,9 +191,6 @@ public void initialize(URI uri, Configuration configuration) .withBlockOutputActiveBlocks(blockOutputActiveBlocks) .build(); - this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder); - LOG.trace("AzureBlobFileSystemStore init complete"); - final AbfsConfiguration abfsConfiguration = abfsStore .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( @@ -201,9 +198,13 @@ public void initialize(URI uri, Configuration configuration) tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); + TracingContext tracingContext = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); + + this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder, tracingContext); + LOG.trace("AzureBlobFileSystemStore init complete"); + if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { - TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), tracingContext) == null) { try { this.createFileSystem(tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 11397e03e5c5b6..f0bfc9cad87a90 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -197,7 +197,7 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport { * @throws IOException Throw IOE in case of failure during constructing. */ public AzureBlobFileSystemStore( - AzureBlobFileSystemStoreBuilder abfsStoreBuilder) throws IOException { + AzureBlobFileSystemStoreBuilder abfsStoreBuilder, TracingContext tracingContext) throws IOException { this.uri = abfsStoreBuilder.uri; String[] authorityParts = authorityParts(uri); final String fileSystemName = authorityParts[0]; @@ -239,7 +239,7 @@ public AzureBlobFileSystemStore( boolean useHttps = (usingOauth || abfsConfiguration.isHttpsAlwaysUsed()) ? true : abfsStoreBuilder.isSecureScheme; this.abfsPerfTracker = new AbfsPerfTracker(fileSystemName, accountName, this.abfsConfiguration); this.abfsCounters = abfsStoreBuilder.abfsCounters; - initializeClient(uri, fileSystemName, accountName, useHttps); + initializeClient(uri, fileSystemName, accountName, useHttps, tracingContext); final Class identityTransformerClass = abfsStoreBuilder.configuration.getClass(FS_AZURE_IDENTITY_TRANSFORM_CLASS, IdentityTransformer.class, IdentityTransformerInterface.class); @@ -1572,7 +1572,7 @@ public boolean isInfiniteLeaseKey(String key) { * @throws IOException */ private void initializeClient(URI uri, String fileSystemName, - String accountName, boolean isSecure) + String accountName, boolean isSecure, TracingContext tracingContext) throws IOException { if (this.client != null) { return; @@ -1608,20 +1608,21 @@ private void initializeClient(URI uri, String fileSystemName, abfsConfiguration.getStorageAccountKey()); } else if (authType == AuthType.SAS) { LOG.trace("Fetching SAS token provider"); - sasTokenProvider = abfsConfiguration.getSASTokenProvider(); + sasTokenProvider = abfsConfiguration.getSASTokenProvider(getIsNamespaceEnabled(tracingContext)); } else { LOG.trace("Fetching token provider"); tokenProvider = abfsConfiguration.getTokenProvider(); ExtensionHelper.bind(tokenProvider, uri, abfsConfiguration.getRawConfiguration()); } - LOG.trace("Initializing AbfsClient for {}", baseUrl); if (tokenProvider != null) { this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, tokenProvider, populateAbfsClientContext()); } else { + // determine whether to use config or tokenProvider + this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, sasTokenProvider, populateAbfsClientContext()); From 072f358c6a2d5185ddf1ab85e2cbc6b6a9c8f075 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 15 Nov 2022 11:17:59 +0530 Subject: [PATCH 05/34] added functionality for choosing SAS provider modes --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 52 +++++++++++++++++++ .../fs/azurebfs/services/AbfsClient.java | 8 ++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index fafc30372b4a54..52ce8255b2f7b4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -22,6 +22,7 @@ import java.lang.reflect.Field; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.azurebfs.extensions.FixedSASTokenProvider; import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.StringUtils; @@ -915,6 +916,57 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio } } + public SASTokenProvider getSASTokenProvider(boolean isNamespaceEnabled) throws AzureBlobFileSystemException { + // currently kept as a second method definition to not disturb the TestAccountConfiguration tests + // which test the precedence of account specific and global sas provider + AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); + if (authType != AuthType.SAS) { + throw new SASTokenProviderException(String.format( + "Invalid auth type: %s is being used, expecting SAS", authType)); + } + + try { + String configKey = FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; + Class sasTokenProviderClass = + getTokenProviderClass(authType, configKey, null, + SASTokenProvider.class); + String fixedConfigKey = FS_AZURE_SAS_FIXED_TOKEN; + String fixedToken = this.rawConfig.get(fixedConfigKey, null); + + Preconditions.checkArgument(sasTokenProviderClass != null || fixedToken != null, + String.format("The configuration value for both \"%s\" and \"%s\" cannot be invalid.", configKey, fixedConfigKey)); + + Class finalSasTokenProviderClass = null; + if (sasTokenProviderClass != null && fixedToken != null) { + if (isNamespaceEnabled) + throw new InvalidConfigurationValueException("A clear setting of either global or uesr delegation SAS provider is required."); + else { + // precedence given to fixed SAS in case of non hns account + finalSasTokenProviderClass = FixedSASTokenProvider.class; + } + } + else if (sasTokenProviderClass != null) { + // document that access control exceptions might be encountered with filesystem level operations + finalSasTokenProviderClass = sasTokenProviderClass; + } + else { + finalSasTokenProviderClass = FixedSASTokenProvider.class; + } + + SASTokenProvider sasTokenProvider = ReflectionUtils + .newInstance(finalSasTokenProviderClass, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, + String.format("Failed to initialize %s", finalSasTokenProviderClass)); + + LOG.trace("Initializing {}", finalSasTokenProviderClass.getName()); + sasTokenProvider.initialize(rawConfig, accountName); + LOG.trace("{} init complete", finalSasTokenProviderClass.getName()); + return sasTokenProvider; + } catch (Exception e) { + throw new TokenAccessProviderException("Unable to load SAS token provider class: " + e, e); + } + } + public int getReadAheadRange() { return this.readAheadRange; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index aa72ed64e6e5d2..33a6e78344ae29 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -38,7 +38,12 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.enums.Trilean; import org.apache.hadoop.fs.store.LogExactlyOnce; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; @@ -58,6 +63,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; +import org.apache.hadoop.fs.azurebfs.enums.Trilean; import org.apache.hadoop.fs.azurebfs.extensions.ExtensionHelper; import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider; import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; @@ -1122,7 +1128,7 @@ private String appendSASTokenToQuery(String path, LOG.trace("Fetch SAS token for {} on {}", operation, path); if (cachedSasToken == null) { sasToken = sasTokenProvider.getSASToken(this.accountName, - this.filesystem, path, operation); + this.filesystem, path, operation); if ((sasToken == null) || sasToken.isEmpty()) { throw new UnsupportedOperationException("SASToken received is empty or null"); } From 306ce9b39aaf2159104a95c9b3edaa87c0bd65af Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 17 Nov 2022 11:53:59 +0530 Subject: [PATCH 06/34] Changed passing tracingContext --- hadoop-tools/hadoop-azure/.attach_pid103782 | 0 .../fs/azurebfs/AzureBlobFileSystem.java | 5 ++-- .../fs/azurebfs/AzureBlobFileSystemStore.java | 23 +++++++++---------- .../ITestAzureBlobFileSystemChooseSAS.java | 2 ++ 4 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/.attach_pid103782 create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java diff --git a/hadoop-tools/hadoop-azure/.attach_pid103782 b/hadoop-tools/hadoop-azure/.attach_pid103782 new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index e5758294127510..13aac345730ff0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -191,6 +191,8 @@ public void initialize(URI uri, Configuration configuration) .withBlockOutputActiveBlocks(blockOutputActiveBlocks) .build(); + this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder, fileSystemId, listener); + LOG.trace("AzureBlobFileSystemStore init complete"); final AbfsConfiguration abfsConfiguration = abfsStore .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( @@ -201,9 +203,6 @@ public void initialize(URI uri, Configuration configuration) TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); - this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder, tracingContext); - LOG.trace("AzureBlobFileSystemStore init complete"); - if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), tracingContext) == null) { try { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index f0bfc9cad87a90..db3577e456076e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -55,6 +55,8 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.azurebfs.constants.*; +import org.apache.hadoop.fs.azurebfs.utils.*; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; @@ -69,10 +71,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; -import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; -import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations; -import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConcurrentWriteOperationDetectedException; @@ -114,11 +112,6 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker; import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo; import org.apache.hadoop.fs.azurebfs.services.ListingSupport; -import org.apache.hadoop.fs.azurebfs.utils.Base64; -import org.apache.hadoop.fs.azurebfs.utils.CRC64; -import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils; -import org.apache.hadoop.fs.azurebfs.utils.TracingContext; -import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.fs.impl.OpenFileParameters; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -197,7 +190,7 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport { * @throws IOException Throw IOE in case of failure during constructing. */ public AzureBlobFileSystemStore( - AzureBlobFileSystemStoreBuilder abfsStoreBuilder, TracingContext tracingContext) throws IOException { + AzureBlobFileSystemStoreBuilder abfsStoreBuilder, String fileSystemId, Listener listener) throws IOException { this.uri = abfsStoreBuilder.uri; String[] authorityParts = authorityParts(uri); final String fileSystemName = authorityParts[0]; @@ -239,6 +232,14 @@ public AzureBlobFileSystemStore( boolean useHttps = (usingOauth || abfsConfiguration.isHttpsAlwaysUsed()) ? true : abfsStoreBuilder.isSecureScheme; this.abfsPerfTracker = new AbfsPerfTracker(fileSystemName, accountName, this.abfsConfiguration); this.abfsCounters = abfsStoreBuilder.abfsCounters; + + // creating tracing context to be used by getSASTokenProvider + String clientCorrelationId = TracingContext.validateClientCorrelationID( + abfsConfiguration.getClientCorrelationId()); + TracingHeaderFormat tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); + TracingContext tracingContext = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); + initializeClient(uri, fileSystemName, accountName, useHttps, tracingContext); final Class identityTransformerClass = abfsStoreBuilder.configuration.getClass(FS_AZURE_IDENTITY_TRANSFORM_CLASS, IdentityTransformer.class, @@ -1621,8 +1622,6 @@ private void initializeClient(URI uri, String fileSystemName, tokenProvider, populateAbfsClientContext()); } else { - // determine whether to use config or tokenProvider - this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, sasTokenProvider, populateAbfsClientContext()); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java new file mode 100644 index 00000000000000..9a26d50d31de1c --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -0,0 +1,2 @@ +package org.apache.hadoop.fs.azurebfs;public class ITestAzureBlobFileSystemChooseSAS { +} From 5712f67183879725101a0e1e6b408f1bed62d42a Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 17 Nov 2022 13:39:59 +0530 Subject: [PATCH 07/34] Using setter for sasTokenProvider --- .../hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 8 +++++--- .../apache/hadoop/fs/azurebfs/services/AbfsClient.java | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index db3577e456076e..87aa6d80137df4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1608,8 +1608,8 @@ private void initializeClient(URI uri, String fileSystemName, creds = new SharedKeyCredentials(accountName.substring(0, dotIndex), abfsConfiguration.getStorageAccountKey()); } else if (authType == AuthType.SAS) { - LOG.trace("Fetching SAS token provider"); - sasTokenProvider = abfsConfiguration.getSASTokenProvider(getIsNamespaceEnabled(tracingContext)); + LOG.trace("Setting SAS token provider to temporary value"); + sasTokenProvider = null; } else { LOG.trace("Fetching token provider"); tokenProvider = abfsConfiguration.getTokenProvider(); @@ -1623,8 +1623,10 @@ private void initializeClient(URI uri, String fileSystemName, populateAbfsClientContext()); } else { this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, - sasTokenProvider, populateAbfsClientContext()); + LOG.trace("Fetching actual SAS token provider"); + sasTokenProvider = this.abfsConfiguration.getSASTokenProvider(getIsNamespaceEnabled(tracingContext)); + client.setSasTokenProvider(sasTokenProvider); } LOG.trace("AbfsClient init complete"); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 33a6e78344ae29..5ceaa6e9c20cb4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -198,6 +198,9 @@ private String getBase64EncodedString(byte[] bytes) { return Base64.getEncoder().encodeToString(bytes); } + public void setSasTokenProvider(final SASTokenProvider sasTokenProvider) { + this.sasTokenProvider = sasTokenProvider; + } @Override public void close() throws IOException { if (tokenProvider instanceof Closeable) { From c34f09bc56eff65bf687b98203a8da4e79723e3e Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 17 Nov 2022 13:43:34 +0530 Subject: [PATCH 08/34] Added setter for sasTokenProvider --- .../org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 87aa6d80137df4..a28f63860031c7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1623,6 +1623,7 @@ private void initializeClient(URI uri, String fileSystemName, populateAbfsClientContext()); } else { this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, + sasTokenProvider, populateAbfsClientContext()); LOG.trace("Fetching actual SAS token provider"); sasTokenProvider = this.abfsConfiguration.getSASTokenProvider(getIsNamespaceEnabled(tracingContext)); From 2e95f5daba413c5c4ff52d9223b507ecfe4ef395 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Fri, 18 Nov 2022 19:33:16 +0530 Subject: [PATCH 09/34] SAS token chosen without namespace knowledge --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 40 +------------------ .../fs/azurebfs/AzureBlobFileSystem.java | 8 ++-- .../fs/azurebfs/AzureBlobFileSystemStore.java | 20 +++------- .../fs/azurebfs/services/AbfsClient.java | 3 -- 4 files changed, 11 insertions(+), 60 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 52ce8255b2f7b4..9087ccfe17ec58 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -887,38 +887,6 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio } public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { - AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); - if (authType != AuthType.SAS) { - throw new SASTokenProviderException(String.format( - "Invalid auth type: %s is being used, expecting SAS", authType)); - } - - try { - String configKey = FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; - Class sasTokenProviderClass = - getTokenProviderClass(authType, configKey, null, - SASTokenProvider.class); - - Preconditions.checkArgument(sasTokenProviderClass != null, - String.format("The configuration value for \"%s\" is invalid.", configKey)); - - SASTokenProvider sasTokenProvider = ReflectionUtils - .newInstance(sasTokenProviderClass, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, - String.format("Failed to initialize %s", sasTokenProviderClass)); - - LOG.trace("Initializing {}", sasTokenProviderClass.getName()); - sasTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", sasTokenProviderClass.getName()); - return sasTokenProvider; - } catch (Exception e) { - throw new TokenAccessProviderException("Unable to load SAS token provider class: " + e, e); - } - } - - public SASTokenProvider getSASTokenProvider(boolean isNamespaceEnabled) throws AzureBlobFileSystemException { - // currently kept as a second method definition to not disturb the TestAccountConfiguration tests - // which test the precedence of account specific and global sas provider AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); if (authType != AuthType.SAS) { throw new SASTokenProviderException(String.format( @@ -938,12 +906,8 @@ public SASTokenProvider getSASTokenProvider(boolean isNamespaceEnabled) throws A Class finalSasTokenProviderClass = null; if (sasTokenProviderClass != null && fixedToken != null) { - if (isNamespaceEnabled) - throw new InvalidConfigurationValueException("A clear setting of either global or uesr delegation SAS provider is required."); - else { - // precedence given to fixed SAS in case of non hns account - finalSasTokenProviderClass = FixedSASTokenProvider.class; - } + LOG.trace("Using SASTokenProvider class instead of config although both are available for use"); + finalSasTokenProviderClass = sasTokenProviderClass; } else if (sasTokenProviderClass != null) { // document that access control exceptions might be encountered with filesystem level operations diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 13aac345730ff0..f94fcb02160ac5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -191,7 +191,7 @@ public void initialize(URI uri, Configuration configuration) .withBlockOutputActiveBlocks(blockOutputActiveBlocks) .build(); - this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder, fileSystemId, listener); + this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder); LOG.trace("AzureBlobFileSystemStore init complete"); final AbfsConfiguration abfsConfiguration = abfsStore .getAbfsConfiguration(); @@ -200,10 +200,10 @@ public void initialize(URI uri, Configuration configuration) tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); - TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); - if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { + TracingContext tracingContext = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); + if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), tracingContext) == null) { try { this.createFileSystem(tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index a28f63860031c7..752d7b1a29eb49 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -190,7 +190,7 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport { * @throws IOException Throw IOE in case of failure during constructing. */ public AzureBlobFileSystemStore( - AzureBlobFileSystemStoreBuilder abfsStoreBuilder, String fileSystemId, Listener listener) throws IOException { + AzureBlobFileSystemStoreBuilder abfsStoreBuilder) throws IOException { this.uri = abfsStoreBuilder.uri; String[] authorityParts = authorityParts(uri); final String fileSystemName = authorityParts[0]; @@ -233,14 +233,7 @@ public AzureBlobFileSystemStore( this.abfsPerfTracker = new AbfsPerfTracker(fileSystemName, accountName, this.abfsConfiguration); this.abfsCounters = abfsStoreBuilder.abfsCounters; - // creating tracing context to be used by getSASTokenProvider - String clientCorrelationId = TracingContext.validateClientCorrelationID( - abfsConfiguration.getClientCorrelationId()); - TracingHeaderFormat tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); - TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); - - initializeClient(uri, fileSystemName, accountName, useHttps, tracingContext); + initializeClient(uri, fileSystemName, accountName, useHttps); final Class identityTransformerClass = abfsStoreBuilder.configuration.getClass(FS_AZURE_IDENTITY_TRANSFORM_CLASS, IdentityTransformer.class, IdentityTransformerInterface.class); @@ -1573,7 +1566,7 @@ public boolean isInfiniteLeaseKey(String key) { * @throws IOException */ private void initializeClient(URI uri, String fileSystemName, - String accountName, boolean isSecure, TracingContext tracingContext) + String accountName, boolean isSecure) throws IOException { if (this.client != null) { return; @@ -1608,8 +1601,8 @@ private void initializeClient(URI uri, String fileSystemName, creds = new SharedKeyCredentials(accountName.substring(0, dotIndex), abfsConfiguration.getStorageAccountKey()); } else if (authType == AuthType.SAS) { - LOG.trace("Setting SAS token provider to temporary value"); - sasTokenProvider = null; + LOG.trace("Fetching SAS Token Provider"); + sasTokenProvider = this.abfsConfiguration.getSASTokenProvider(); } else { LOG.trace("Fetching token provider"); tokenProvider = abfsConfiguration.getTokenProvider(); @@ -1625,9 +1618,6 @@ private void initializeClient(URI uri, String fileSystemName, this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, sasTokenProvider, populateAbfsClientContext()); - LOG.trace("Fetching actual SAS token provider"); - sasTokenProvider = this.abfsConfiguration.getSASTokenProvider(getIsNamespaceEnabled(tracingContext)); - client.setSasTokenProvider(sasTokenProvider); } LOG.trace("AbfsClient init complete"); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 5ceaa6e9c20cb4..33a6e78344ae29 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -198,9 +198,6 @@ private String getBase64EncodedString(byte[] bytes) { return Base64.getEncoder().encodeToString(bytes); } - public void setSasTokenProvider(final SASTokenProvider sasTokenProvider) { - this.sasTokenProvider = sasTokenProvider; - } @Override public void close() throws IOException { if (tokenProvider instanceof Closeable) { From 3a809b4bc05dde9ce1f12e64d59a1639639c4aa0 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Fri, 18 Nov 2022 19:33:53 +0530 Subject: [PATCH 10/34] Added tests for choosing SAS Token --- .../ITestAzureBlobFileSystemChooseSAS.java | 121 +++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 9a26d50d31de1c..741d0e436d3748 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -1,2 +1,121 @@ -package org.apache.hadoop.fs.azurebfs;public class ITestAzureBlobFileSystemChooseSAS { +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException; +import org.apache.hadoop.fs.azurebfs.extensions.FixedSASTokenProvider; +import org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider; +import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider; +import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider; +import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.assertj.core.api.Assertions; +import org.junit.Assume; +import org.junit.Test; + +import java.io.IOException; +import java.lang.reflect.Executable; + +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; + +public class ITestAzureBlobFileSystemChooseSAS extends AbstractAbfsIntegrationTest{ + + MockSASTokenProvider fixedTokenProvider; + public ITestAzureBlobFileSystemChooseSAS() throws Exception { + // These tests rely on specific settings in azure-auth-keys.xml: + Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_ID)); + Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET)); + Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID)); + Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID)); + // The test uses shared key to create a random filesystem and then creates another + // instance of this filesystem using SAS authorization. + Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); + } + + @Override + public void setup() throws Exception { + createFilesystemForSASTests(); + fixedTokenProvider = new MockSASTokenProvider(); + fixedTokenProvider.initialize(getRawConfiguration(), getAccountName()); + super.setup(); + } + + @Test + public void bothProviderConfigSet() throws IOException { + Configuration testConfig = getRawConfiguration(); + AzureBlobFileSystem testFs = getFileSystem(); + testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, "org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider"); + // Not setting any operation as Service SAS generator does not use it + testConfig.set(FS_AZURE_SAS_FIXED_TOKEN, fixedTokenProvider.getSASToken(getAccountName(), testFs.toString(), "/", "")); + SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); + // the tokenProvider class should be chosen + assertEquals(MockDelegationSASTokenProvider.class, actualClass.getClass()); + + // attempting an operation using the selected SAS Token + // creating a new fs instance with the new configs + AzureBlobFileSystem newTestFs = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()); + Path testPath = new Path("/testCorrectSASToken"); + + newTestFs.create(testPath).close(); + } + + @Test + public void onlyConfigSet() throws IOException { + Configuration testConfig = getRawConfiguration(); + AzureBlobFileSystem testFs = getFileSystem(); + testConfig.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE); + // Not setting any operation as Service SAS generator does not use it + testConfig.set(FS_AZURE_SAS_FIXED_TOKEN, fixedTokenProvider.getSASToken(getAccountName(), testFs.toString(), "/", "")); + SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); + // the tokenProvider class should be chosen + assertEquals(FixedSASTokenProvider.class, actualClass.getClass()); + + // attempting an operation using the selected SAS Token + Path testPath = new Path("/testCorrectSASToken"); + testFs.create(testPath).close(); + } + + @Test + public void onlyProviderSet() throws IOException { + Configuration testConfig = getRawConfiguration(); + AzureBlobFileSystem testFs = getFileSystem(); + testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, "org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider"); + testConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); + SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); + // the tokenProvider class should be chosen + assertEquals(MockDelegationSASTokenProvider.class, actualClass.getClass()); + + // attempting an operation using the selected SAS Token + Path testPath = new Path("/testCorrectSASToken"); + testFs.create(testPath).close(); + } + + @Test(expected = TokenAccessProviderException.class) + public void bothProviderConfigUnset() throws IOException { + Configuration testConfig = getRawConfiguration(); + + testConfig.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE); + testConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); + + SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); + } } From 4e1fe4b87cb5ff636e2715497ebb3dab19a9c822 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Fri, 18 Nov 2022 19:51:51 +0530 Subject: [PATCH 11/34] Style changes --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 9 +++------ .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 2 +- .../fs/azurebfs/AzureBlobFileSystemStore.java | 15 +++++++++++---- .../extensions/FixedSASTokenProvider.java | 17 +++++++++++++++++ .../hadoop/fs/azurebfs/services/AbfsClient.java | 3 +-- 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 9087ccfe17ec58..f906b443f9b1ff 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -889,8 +889,7 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); if (authType != AuthType.SAS) { - throw new SASTokenProviderException(String.format( - "Invalid auth type: %s is being used, expecting SAS", authType)); + throw new SASTokenProviderException(String.format("Invalid auth type: %s is being used, expecting SAS", authType)); } try { @@ -917,10 +916,8 @@ else if (sasTokenProviderClass != null) { finalSasTokenProviderClass = FixedSASTokenProvider.class; } - SASTokenProvider sasTokenProvider = ReflectionUtils - .newInstance(finalSasTokenProviderClass, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, - String.format("Failed to initialize %s", finalSasTokenProviderClass)); + SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(finalSasTokenProviderClass, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", finalSasTokenProviderClass)); LOG.trace("Initializing {}", finalSasTokenProviderClass.getName()); sasTokenProvider.initialize(rawConfig, accountName); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index f94fcb02160ac5..a5b09d6abd4eb1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -193,6 +193,7 @@ public void initialize(URI uri, Configuration configuration) this.abfsStore = new AzureBlobFileSystemStore(systemStoreBuilder); LOG.trace("AzureBlobFileSystemStore init complete"); + final AbfsConfiguration abfsConfiguration = abfsStore .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( @@ -203,7 +204,6 @@ public void initialize(URI uri, Configuration configuration) if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); - if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), tracingContext) == null) { try { this.createFileSystem(tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 752d7b1a29eb49..4a1537eeaf88ae 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -55,8 +55,6 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.azurebfs.constants.*; -import org.apache.hadoop.fs.azurebfs.utils.*; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; @@ -71,6 +69,10 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConcurrentWriteOperationDetectedException; @@ -112,6 +114,11 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsPerfTracker; import org.apache.hadoop.fs.azurebfs.services.AbfsPerfInfo; import org.apache.hadoop.fs.azurebfs.services.ListingSupport; +import org.apache.hadoop.fs.azurebfs.utils.Base64; +import org.apache.hadoop.fs.azurebfs.utils.CRC64; +import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; import org.apache.hadoop.fs.impl.OpenFileParameters; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -232,7 +239,6 @@ public AzureBlobFileSystemStore( boolean useHttps = (usingOauth || abfsConfiguration.isHttpsAlwaysUsed()) ? true : abfsStoreBuilder.isSecureScheme; this.abfsPerfTracker = new AbfsPerfTracker(fileSystemName, accountName, this.abfsConfiguration); this.abfsCounters = abfsStoreBuilder.abfsCounters; - initializeClient(uri, fileSystemName, accountName, useHttps); final Class identityTransformerClass = abfsStoreBuilder.configuration.getClass(FS_AZURE_IDENTITY_TRANSFORM_CLASS, IdentityTransformer.class, @@ -1602,13 +1608,14 @@ private void initializeClient(URI uri, String fileSystemName, abfsConfiguration.getStorageAccountKey()); } else if (authType == AuthType.SAS) { LOG.trace("Fetching SAS Token Provider"); - sasTokenProvider = this.abfsConfiguration.getSASTokenProvider(); + sasTokenProvider = abfsConfiguration.getSASTokenProvider(); } else { LOG.trace("Fetching token provider"); tokenProvider = abfsConfiguration.getTokenProvider(); ExtensionHelper.bind(tokenProvider, uri, abfsConfiguration.getRawConfiguration()); } + LOG.trace("Initializing AbfsClient for {}", baseUrl); if (tokenProvider != null) { this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java index 7d4be92e5c6f89..786f466f644028 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.fs.azurebfs.extensions; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 33a6e78344ae29..3acbfc06a3f673 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1127,8 +1127,7 @@ private String appendSASTokenToQuery(String path, try { LOG.trace("Fetch SAS token for {} on {}", operation, path); if (cachedSasToken == null) { - sasToken = sasTokenProvider.getSASToken(this.accountName, - this.filesystem, path, operation); + sasToken = sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); if ((sasToken == null) || sasToken.isEmpty()) { throw new UnsupportedOperationException("SASToken received is empty or null"); } From d7179de1d791071fd35e510680e8e290b9d731bb Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Mon, 28 Nov 2022 13:07:08 +0530 Subject: [PATCH 12/34] simplifying SAS Token choice --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 33 +++++++------------ .../fs/azurebfs/services/AbfsClient.java | 27 ++++++++++++--- .../MockActualSASTokenProvider.java | 2 ++ 3 files changed, 36 insertions(+), 26 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index f906b443f9b1ff..70ac7188eb3954 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -893,36 +893,27 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio } try { - String configKey = FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; Class sasTokenProviderClass = - getTokenProviderClass(authType, configKey, null, + getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, null, SASTokenProvider.class); - String fixedConfigKey = FS_AZURE_SAS_FIXED_TOKEN; - String fixedToken = this.rawConfig.get(fixedConfigKey, null); + String fixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); Preconditions.checkArgument(sasTokenProviderClass != null || fixedToken != null, - String.format("The configuration value for both \"%s\" and \"%s\" cannot be invalid.", configKey, fixedConfigKey)); + String.format("The configuration value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); - Class finalSasTokenProviderClass = null; - if (sasTokenProviderClass != null && fixedToken != null) { + if (sasTokenProviderClass != null) { LOG.trace("Using SASTokenProvider class instead of config although both are available for use"); - finalSasTokenProviderClass = sasTokenProviderClass; - } - else if (sasTokenProviderClass != null) { - // document that access control exceptions might be encountered with filesystem level operations - finalSasTokenProviderClass = sasTokenProviderClass; + SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderClass, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderClass)); + + LOG.trace("Initializing {}", sasTokenProviderClass.getName()); + sasTokenProvider.initialize(rawConfig, accountName); + LOG.trace("{} init complete", sasTokenProviderClass.getName()); + return sasTokenProvider; } else { - finalSasTokenProviderClass = FixedSASTokenProvider.class; + return null; } - - SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(finalSasTokenProviderClass, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", finalSasTokenProviderClass)); - - LOG.trace("Initializing {}", finalSasTokenProviderClass.getName()); - sasTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", finalSasTokenProviderClass.getName()); - return sasTokenProvider; } catch (Exception e) { throw new TokenAccessProviderException("Unable to load SAS token provider class: " + e, e); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 3acbfc06a3f673..34ea689577fae0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -38,12 +38,8 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; -import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; -import org.apache.hadoop.fs.azurebfs.enums.Trilean; import org.apache.hadoop.fs.store.LogExactlyOnce; -import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; @@ -258,6 +254,9 @@ public AbfsRestOperation createFilesystem(TracingContext tracingContext) throws final AbfsUriQueryBuilder abfsUriQueryBuilder = new AbfsUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); + // appending SAS Token to query + appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); + final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); final AbfsRestOperation op = new AbfsRestOperation( AbfsRestOperationType.CreateFileSystem, @@ -282,6 +281,9 @@ public AbfsRestOperation setFilesystemProperties(final String properties, Tracin final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); + // appending SAS token to query + appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); + final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); final AbfsRestOperation op = new AbfsRestOperation( AbfsRestOperationType.SetFileSystemProperties, @@ -324,6 +326,9 @@ public AbfsRestOperation getFilesystemProperties(TracingContext tracingContext) final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); + // appending SAS token to query + appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); + final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); final AbfsRestOperation op = new AbfsRestOperation( AbfsRestOperationType.GetFileSystemProperties, @@ -341,6 +346,9 @@ public AbfsRestOperation deleteFilesystem(TracingContext tracingContext) throws final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); + // appending SAS token to query + appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); + final URL url = createRequestUrl(abfsUriQueryBuilder.toString()); final AbfsRestOperation op = new AbfsRestOperation( AbfsRestOperationType.DeleteFileSystem, @@ -1096,6 +1104,14 @@ public static String getDirectoryQueryParameter(final String path) { return directory; } + private String chooseSASToken(String operation, String path) throws IOException { + if (sasTokenProvider == null) { + return abfsConfiguration.get(ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN); + } else { + return sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); + } + } + /** * If configured for SAS AuthType, appends SAS token to queryBuilder * @param path @@ -1127,7 +1143,8 @@ private String appendSASTokenToQuery(String path, try { LOG.trace("Fetch SAS token for {} on {}", operation, path); if (cachedSasToken == null) { - sasToken = sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); + sasToken = chooseSASToken(operation, path); + // sasToken = sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); if ((sasToken == null) || sasToken.isEmpty()) { throw new UnsupportedOperationException("SASToken received is empty or null"); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java new file mode 100644 index 00000000000000..77632ebf079887 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java @@ -0,0 +1,2 @@ +package org.apache.hadoop.fs.azurebfs.extensions;public class MockActualSASTokenProvider { +} From 95e0363247240fca86355d90c81575036d2aa8e2 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Mon, 28 Nov 2022 13:08:39 +0530 Subject: [PATCH 13/34] Deleted pid file --- hadoop-tools/hadoop-azure/.attach_pid103782 | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure/.attach_pid103782 diff --git a/hadoop-tools/hadoop-azure/.attach_pid103782 b/hadoop-tools/hadoop-azure/.attach_pid103782 deleted file mode 100644 index e69de29bb2d1d6..00000000000000 From c9152746651c52118e1254488da3ad31439e7d59 Mon Sep 17 00:00:00 2001 From: sreeb-msft <111426823+sreeb-msft@users.noreply.github.com> Date: Wed, 7 Dec 2022 18:12:44 +0530 Subject: [PATCH 14/34] Delete FixedSASTokenProvider --- .../extensions/FixedSASTokenProvider.java | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java deleted file mode 100644 index 786f466f644028..00000000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/FixedSASTokenProvider.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs.azurebfs.extensions; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; -import org.apache.hadoop.security.AccessControlException; - -import java.io.IOException; - -public class FixedSASTokenProvider implements SASTokenProvider{ - Configuration configuration; - String accountName; - - @Override - public void initialize(Configuration configuration, String accountName) throws IOException { - this.configuration = configuration; - this.accountName = accountName; - } - - @Override - public String getSASToken(String account, String fileSystem, String path, String operation) throws IOException { - String fixedToken = configuration.get(ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN, null); - if (fixedToken == null) - throw new InvalidConfigurationValueException("The fixed SAS Token configuration value is invalid."); - else - return fixedToken; - } -} From 8288cdc3261375b3b10b34076dc5977598750b5b Mon Sep 17 00:00:00 2001 From: sreeb-msft <111426823+sreeb-msft@users.noreply.github.com> Date: Wed, 7 Dec 2022 18:14:06 +0530 Subject: [PATCH 15/34] Delete MockActualSASTokenProvider --- .../fs/azurebfs/extensions/MockActualSASTokenProvider.java | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java deleted file mode 100644 index 77632ebf079887..00000000000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java +++ /dev/null @@ -1,2 +0,0 @@ -package org.apache.hadoop.fs.azurebfs.extensions;public class MockActualSASTokenProvider { -} From 5174b3cee381ddc30b8fe9ab688a34ae179284ef Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 7 Dec 2022 18:25:27 +0530 Subject: [PATCH 16/34] HADOOP-18516. Appending sas token to fs level operations --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 29 ++++++++++++++----- .../fs/azurebfs/services/AbfsClient.java | 12 +++----- .../MockActualSASTokenProvider.java | 2 -- .../azurebfs/utils/AccountSASGenerator.java | 2 ++ 4 files changed, 27 insertions(+), 18 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 70ac7188eb3954..a04c48096c7d9a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -886,6 +886,19 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio } } + /** + * @return sasTokenProvider object + * @throws AzureBlobFileSystemException + * The following method chooses between a configured fixed sas token, and a user implementation of the SASTokenProvider interface, + * depending on which one is available. In case a user SASTokenProvider implementation is not present, and a fixed token is configured, + * it simply returns null, to set the sasTokenProvider object for current configuration instance to null. + * The fixed token is read and used later. This is done to: + * 1. check for cases where both are not set, while initializing AbfsConfiguration, to not proceed further than thi stage itself when none of the options are available. + * 2. avoid using similar tokenProvider implementation to just read the configured fixed token, as this could create confusion. The configuration is introduced + * primarily to avoid using any tokenProvider class/interface. Also,implementing the SASTokenProvider requires relying on the raw configurations. + * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, and eliminate chances of dynamic modifications and spurious situations. + */ + public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); if (authType != AuthType.SAS) { @@ -893,22 +906,22 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio } try { - Class sasTokenProviderClass = + Class sasTokenProviderImplementation = getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, null, SASTokenProvider.class); - String fixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); + String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); - Preconditions.checkArgument(sasTokenProviderClass != null || fixedToken != null, + Preconditions.checkArgument(sasTokenProviderImplementation != null || configuredFixedToken != null, String.format("The configuration value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); - if (sasTokenProviderClass != null) { + if (sasTokenProviderImplementation != null) { LOG.trace("Using SASTokenProvider class instead of config although both are available for use"); - SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderClass, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderClass)); + SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderImplementation, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderImplementation)); - LOG.trace("Initializing {}", sasTokenProviderClass.getName()); + LOG.trace("Initializing {}", sasTokenProviderImplementation.getName()); sasTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", sasTokenProviderClass.getName()); + LOG.trace("{} init complete", sasTokenProviderImplementation.getName()); return sasTokenProvider; } else { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 34ea689577fae0..a4dcc39ac60b02 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -59,7 +59,6 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; -import org.apache.hadoop.fs.azurebfs.enums.Trilean; import org.apache.hadoop.fs.azurebfs.extensions.ExtensionHelper; import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider; import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; @@ -89,7 +88,8 @@ public class AbfsClient implements Closeable { private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; - private final String xMsVersion = "2019-12-12"; + // private final String xMsVersion = "2019-12-12"; + private final String xMsVersion = "2021-10-04"; private final ExponentialRetryPolicy retryPolicy; private final String filesystem; private final AbfsConfiguration abfsConfiguration; @@ -253,7 +253,6 @@ public AbfsRestOperation createFilesystem(TracingContext tracingContext) throws final AbfsUriQueryBuilder abfsUriQueryBuilder = new AbfsUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); - // appending SAS Token to query appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); @@ -280,7 +279,6 @@ public AbfsRestOperation setFilesystemProperties(final String properties, Tracin final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); - // appending SAS token to query appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); @@ -325,7 +323,6 @@ public AbfsRestOperation getFilesystemProperties(TracingContext tracingContext) final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); - // appending SAS token to query appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); @@ -345,7 +342,6 @@ public AbfsRestOperation deleteFilesystem(TracingContext tracingContext) throws final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); - // appending SAS token to query appendSASTokenToQuery(ROOT_PATH, "", abfsUriQueryBuilder); @@ -1105,6 +1101,7 @@ public static String getDirectoryQueryParameter(final String path) { } private String chooseSASToken(String operation, String path) throws IOException { + // chooses the SAS token provider class if it is configured, otherwise reads the configured fixed token if (sasTokenProvider == null) { return abfsConfiguration.get(ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN); } else { @@ -1144,7 +1141,6 @@ private String appendSASTokenToQuery(String path, LOG.trace("Fetch SAS token for {} on {}", operation, path); if (cachedSasToken == null) { sasToken = chooseSASToken(operation, path); - // sasToken = sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); if ((sasToken == null) || sasToken.isEmpty()) { throw new UnsupportedOperationException("SASToken received is empty or null"); } @@ -1178,7 +1174,7 @@ protected URL createRequestUrl(final String path, final String query) } catch (AzureBlobFileSystemException ex) { LOG.debug("Unexpected error.", ex); throw new InvalidUriException(path); - } + } final StringBuilder sb = new StringBuilder(); sb.append(base); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java deleted file mode 100644 index 77632ebf079887..00000000000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockActualSASTokenProvider.java +++ /dev/null @@ -1,2 +0,0 @@ -package org.apache.hadoop.fs.azurebfs.extensions;public class MockActualSASTokenProvider { -} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java new file mode 100644 index 00000000000000..c76ff2130ec5ec --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java @@ -0,0 +1,2 @@ +package org.apache.hadoop.fs.azurebfs.utils;public class AccountSASGenerator { +} From 40d334f86a12f4cceea24feb26b7d1698a5bae22 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 7 Dec 2022 18:27:28 +0530 Subject: [PATCH 17/34] HADOOP-18516. Tests for correct SAS token choice --- .../ITestAzureBlobFileSystemChooseSAS.java | 128 ++++++++++-------- .../azurebfs/utils/AccountSASGenerator.java | 89 +++++++++++- .../fs/azurebfs/utils/SASGenerator.java | 20 ++- 3 files changed, 177 insertions(+), 60 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 741d0e436d3748..c6aa185c08e17b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -17,105 +17,119 @@ */ package org.apache.hadoop.fs.azurebfs; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException; -import org.apache.hadoop.fs.azurebfs.extensions.FixedSASTokenProvider; -import org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider; -import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider; -import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider; import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.assertj.core.api.Assertions; +import org.apache.hadoop.fs.azurebfs.utils.AccountSASGenerator; +import org.apache.hadoop.fs.azurebfs.utils.Base64; +import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.junit.Assume; import org.junit.Test; import java.io.IOException; -import java.lang.reflect.Executable; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; public class ITestAzureBlobFileSystemChooseSAS extends AbstractAbfsIntegrationTest{ - MockSASTokenProvider fixedTokenProvider; + private String accountSAS; + public ITestAzureBlobFileSystemChooseSAS() throws Exception { - // These tests rely on specific settings in azure-auth-keys.xml: - Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_ID)); - Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET)); - Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID)); - Assume.assumeNotNull(getRawConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID)); // The test uses shared key to create a random filesystem and then creates another // instance of this filesystem using SAS authorization. Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } + private void generateAccountSAS() throws AzureBlobFileSystemException { + final String accountKey = getConfiguration().getStorageAccountKey(); + AccountSASGenerator configAccountSASGenerator = new AccountSASGenerator(Base64.decode(accountKey)); + accountSAS = configAccountSASGenerator.getAccountSAS(getAccountName()); + } + @Override public void setup() throws Exception { createFilesystemForSASTests(); - fixedTokenProvider = new MockSASTokenProvider(); - fixedTokenProvider.initialize(getRawConfiguration(), getAccountName()); super.setup(); + // obtaining an account SAS token from in-built generator to set as configuration for testing filesystem level operations + generateAccountSAS(); } + /** + * Tests the scenario where both the token provider class and a fixed token are configured: + * whether the correct choice is made (precedence given to token provider class), and the chosen SAS Token works as expected + * @throws Exception + */ @Test - public void bothProviderConfigSet() throws IOException { - Configuration testConfig = getRawConfiguration(); - AzureBlobFileSystem testFs = getFileSystem(); - testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, "org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider"); - // Not setting any operation as Service SAS generator does not use it - testConfig.set(FS_AZURE_SAS_FIXED_TOKEN, fixedTokenProvider.getSASToken(getAccountName(), testFs.toString(), "/", "")); - SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); - // the tokenProvider class should be chosen - assertEquals(MockDelegationSASTokenProvider.class, actualClass.getClass()); + public void testBothProviderFixedTokenConfigured() throws Exception { + AbfsConfiguration testAbfsConfig = getConfiguration(); - // attempting an operation using the selected SAS Token - // creating a new fs instance with the new configs - AzureBlobFileSystem newTestFs = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()); - Path testPath = new Path("/testCorrectSASToken"); + // configuring a SASTokenProvider class: this provides a user delegation SAS + // user delegation SAS Provider is set to easily distinguish between results of filesystem level and blob level operations to ensure correct SAS is chosen, + // when both a provider class and fixed token is configured. + testAbfsConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, "org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider"); - newTestFs.create(testPath).close(); - } + // configuring the fixed SAS token + testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); - @Test - public void onlyConfigSet() throws IOException { - Configuration testConfig = getRawConfiguration(); - AzureBlobFileSystem testFs = getFileSystem(); - testConfig.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE); - // Not setting any operation as Service SAS generator does not use it - testConfig.set(FS_AZURE_SAS_FIXED_TOKEN, fixedTokenProvider.getSASToken(getAccountName(), testFs.toString(), "/", "")); - SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); - // the tokenProvider class should be chosen - assertEquals(FixedSASTokenProvider.class, actualClass.getClass()); + // creating a new fs instance with the updated configs + AzureBlobFileSystem newTestFs = (AzureBlobFileSystem) FileSystem.newInstance(testAbfsConfig.getRawConfiguration()); - // attempting an operation using the selected SAS Token + // testing a file system level operation + TracingContext tracingContext = getTestTracingContext(newTestFs, true); + // expected to fail in the ideal case, as delegation SAS will be chosen, provider class is given preference when both are configured + intercept(SASTokenProviderException.class, + () -> { + newTestFs.getAbfsStore().getFilesystemProperties(tracingContext); + }); + + // testing blob level operation to ensure delegation SAS token is otherwise valid and above operation fails only because it is fs level Path testPath = new Path("/testCorrectSASToken"); - testFs.create(testPath).close(); + newTestFs.create(testPath).close(); } + /** + * Tests the scenario where only the fixed token is configured, and no token provider class is set: + * whether fixed token is read correctly from configs, and whether the chosen SAS Token works as expected + * @throws IOException + */ @Test - public void onlyProviderSet() throws IOException { - Configuration testConfig = getRawConfiguration(); - AzureBlobFileSystem testFs = getFileSystem(); - testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, "org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider"); - testConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); - SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); - // the tokenProvider class should be chosen - assertEquals(MockDelegationSASTokenProvider.class, actualClass.getClass()); + public void testOnlyFixedTokenConfigured() throws IOException { + AbfsConfiguration testAbfsConfig = getConfiguration(); + + // clearing any previously configured SAS Token Provider class + testAbfsConfig.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE); + + // setting an account SAS token in the fixed token field + testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); + + // creating a new FS with updated configs + AzureBlobFileSystem newTestFs = (AzureBlobFileSystem) FileSystem.newInstance(testAbfsConfig.getRawConfiguration()); // attempting an operation using the selected SAS Token + // as an account SAS is configured, both filesystem level operations (on root) and blob level operations should succeed + newTestFs.getFileStatus(new Path("/")); Path testPath = new Path("/testCorrectSASToken"); - testFs.create(testPath).close(); + newTestFs.create(testPath).close(); + newTestFs.delete(new Path("/"), true); } + /** + * Tests the scenario where both the token provider class and the fixed token are not configured: + * whether the code errors out at the initialization stage itself + * @throws IOException + */ @Test(expected = TokenAccessProviderException.class) - public void bothProviderConfigUnset() throws IOException { - Configuration testConfig = getRawConfiguration(); + public void testBothProviderFixedTokenUnset() throws IOException { + AbfsConfiguration testAbfsConfig = getConfiguration(); - testConfig.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE); - testConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); + testAbfsConfig.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE); + testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); - SASTokenProvider actualClass = getConfiguration().getSASTokenProvider(); + AzureBlobFileSystem newTestFs = (AzureBlobFileSystem) FileSystem.newInstance(testAbfsConfig.getRawConfiguration()); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java index c76ff2130ec5ec..9863b1d54b2c3d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java @@ -1,2 +1,89 @@ -package org.apache.hadoop.fs.azurebfs.utils;public class AccountSASGenerator { +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.utils; + +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; +import org.apache.hadoop.fs.azurebfs.services.AbfsUriQueryBuilder; + +import java.time.Instant; + +/** + * Account SAS Generator to be used by tests + */ + +public class AccountSASGenerator extends SASGenerator { + /** + * Creates Account SAS + * https://learn.microsoft.com/en-us/rest/api/storageservices/create-account-sas + * @param accountKey: the storage account key + */ + public AccountSASGenerator(byte[] accountKey) { super(accountKey); } + + public String getAccountSAS(String accountName) throws AzureBlobFileSystemException { + // retaining only the account name + accountName = getCanonicalAccountName(accountName); + String sp = "racwdl"; + String sv = "2021-06-08"; + String srt = "sco"; + + String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES)); + String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY)); + + String ss = "bf"; + String spr = "https"; + String signature = computeSignatureForSAS(sp, ss, srt, st, se, sv, accountName); + + AbfsUriQueryBuilder qb = new AbfsUriQueryBuilder(); + qb.addQuery("sp", sp); + qb.addQuery("ss", ss); + qb.addQuery("srt", srt); + qb.addQuery("st", st); + qb.addQuery("se", se); + qb.addQuery("sv", sv); + qb.addQuery("sig", signature); + return qb.toString().substring(1); + } + + private String computeSignatureForSAS(String signedPerm, String signedService, String signedResType, + String signedStart, String signedExp, String signedVersion, String accountName) { + + StringBuilder sb = new StringBuilder(); + sb.append(accountName); + sb.append("\n"); + sb.append(signedPerm); + sb.append("\n"); + sb.append(signedService); + sb.append("\n"); + sb.append(signedResType); + sb.append("\n"); + sb.append(signedStart); + sb.append("\n"); + sb.append(signedExp); + sb.append("\n"); + sb.append("\n"); // signedIP + sb.append("\n"); // signedProtocol + sb.append(signedVersion); + sb.append("\n"); + sb.append("\n"); //signed encryption scope + + String stringToSign = sb.toString(); + LOG.debug("Account SAS stringToSign: " + stringToSign.replace("\n", ".")); + return computeHmac256(stringToSign); + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index 2e9289d8d44c7a..bd65145e6093b8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -23,10 +23,15 @@ import java.time.Duration; import java.time.format.DateTimeFormatter; import java.time.ZoneId; +import java.util.Base64; import java.util.Locale; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; +import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; +import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** @@ -70,7 +75,7 @@ private SASGenerator() { * Called by subclasses to initialize the cryptographic SHA-256 HMAC provider. * @param key - a 256-bit secret key */ - protected SASGenerator(byte[] key) { + protected SASGenerator(byte[] key) { this.key = key; initializeMac(); } @@ -85,10 +90,21 @@ private void initializeMac() { } } + protected String getCanonicalAccountName(String accountName) throws InvalidConfigurationValueException { + // returns the account name without the endpoint + int dotIndex = accountName.indexOf(AbfsHttpConstants.DOT); + if (dotIndex <= 0) { + throw new InvalidConfigurationValueException("Account Name is not fully qualified"); + } + String truncAccountName = accountName.substring(0, dotIndex); + return truncAccountName; + } + protected String computeHmac256(final String stringToSign) { byte[] utf8Bytes; try { utf8Bytes = stringToSign.getBytes(StandardCharsets.UTF_8.toString()); + // utf8Bytes = stringToSign.getBytes("UTF-8"); } catch (final UnsupportedEncodingException e) { throw new IllegalArgumentException(e); } @@ -96,6 +112,6 @@ protected String computeHmac256(final String stringToSign) { synchronized (this) { hmac = hmacSha256.doFinal(utf8Bytes); } - return Base64.encode(hmac); + return org.apache.hadoop.fs.azurebfs.utils.Base64.encode(hmac); } } \ No newline at end of file From 0c20560b3bf9be1c5ba2ab07d9ff175bd1d93fd9 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 7 Dec 2022 18:28:11 +0530 Subject: [PATCH 18/34] HADOOP-18516. Canonicalizing account name for service SAS generator --- .../apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java index 24a1cea255b4aa..2cbed62646e7f8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java @@ -20,6 +20,7 @@ import java.time.Instant; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.fs.azurebfs.services.AbfsUriQueryBuilder; /** @@ -36,7 +37,8 @@ public ServiceSASGenerator(byte[] accountKey) { super(accountKey); } - public String getContainerSASWithFullControl(String accountName, String containerName) { + public String getContainerSASWithFullControl(String accountName, String containerName) throws InvalidConfigurationValueException { + accountName = getCanonicalAccountName(accountName); String sp = "rcwdl"; String sv = AuthenticationVersion.Feb20.toString(); String sr = "c"; From ea997618728833ea0df16ef36d688407b77f8298 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 7 Dec 2022 18:43:48 +0530 Subject: [PATCH 19/34] HADOOP-18516. Removed unnecessary import --- .../java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index a04c48096c7d9a..b25044f6254a29 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -22,7 +22,6 @@ import java.lang.reflect.Field; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.azurebfs.extensions.FixedSASTokenProvider; import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.StringUtils; From d523e503b580c648587b6f612ab4fd89ab3f623d Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 8 Dec 2022 11:56:16 +0530 Subject: [PATCH 20/34] HADOOP-18516. Ensuring account name is canonicalized --- .../azurebfs/extensions/MockSASTokenProvider.java | 15 ++++++++++++++- .../hadoop/fs/azurebfs/utils/SASGenerator.java | 11 ++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java index 50ac20970f45f2..1715ab418bbd18 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java @@ -21,11 +21,15 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsScaleTest; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.utils.Base64; import org.apache.hadoop.fs.azurebfs.utils.ServiceSASGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A mock SAS token provider implementation @@ -35,10 +39,19 @@ public class MockSASTokenProvider implements SASTokenProvider { private byte[] accountKey; private ServiceSASGenerator generator; private boolean skipAuthorizationForTestSetup = false; + protected static final Logger LOG = + LoggerFactory.getLogger(MockSASTokenProvider.class); // For testing we use a container SAS for all operations. private String generateSAS(byte[] accountKey, String accountName, String fileSystemName) { - return generator.getContainerSASWithFullControl(accountName, fileSystemName); + String containerSAS = ""; + try { + containerSAS = generator.getContainerSASWithFullControl(accountName, fileSystemName); + } catch (InvalidConfigurationValueException e) { + LOG.debug(e.getMessage()); + containerSAS = ""; + } + return containerSAS; } @Override diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index bd65145e6093b8..292037671ec76b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -93,11 +93,16 @@ private void initializeMac() { protected String getCanonicalAccountName(String accountName) throws InvalidConfigurationValueException { // returns the account name without the endpoint int dotIndex = accountName.indexOf(AbfsHttpConstants.DOT); - if (dotIndex <= 0) { + if (dotIndex == 0) { + // case when accountname starts with a ".": endpoint is present, accountName is null throw new InvalidConfigurationValueException("Account Name is not fully qualified"); } - String truncAccountName = accountName.substring(0, dotIndex); - return truncAccountName; + if (dotIndex > 0) + // case when endpoint is present with accountName + return accountName.substring(0, dotIndex); + else + // case when accountName is already canonicalized + return accountName; } protected String computeHmac256(final String stringToSign) { From 6fff60f860d13c261cfdb15dcc9e68381ae9febb Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 8 Dec 2022 12:13:12 +0530 Subject: [PATCH 21/34] HADOOP-18516. Style changes --- .../apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 11 +++++++---- .../azurebfs/ITestAzureBlobFileSystemChooseSAS.java | 3 ++- .../hadoop/fs/azurebfs/utils/AccountSASGenerator.java | 4 +++- .../apache/hadoop/fs/azurebfs/utils/SASGenerator.java | 3 --- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index b25044f6254a29..79302785eb84cf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -892,10 +892,13 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio * depending on which one is available. In case a user SASTokenProvider implementation is not present, and a fixed token is configured, * it simply returns null, to set the sasTokenProvider object for current configuration instance to null. * The fixed token is read and used later. This is done to: - * 1. check for cases where both are not set, while initializing AbfsConfiguration, to not proceed further than thi stage itself when none of the options are available. - * 2. avoid using similar tokenProvider implementation to just read the configured fixed token, as this could create confusion. The configuration is introduced + * 1. check for cases where both are not set, while initializing AbfsConfiguration, + * to not proceed further than thi stage itself when none of the options are available. + * 2. avoid using similar tokenProvider implementation to just read the configured fixed token, + * as this could create confusion. The configuration is introduced * primarily to avoid using any tokenProvider class/interface. Also,implementing the SASTokenProvider requires relying on the raw configurations. - * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, and eliminate chances of dynamic modifications and spurious situations. + * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, + * and eliminate chances of dynamic modifications and spurious situations. */ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { @@ -911,7 +914,7 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); Preconditions.checkArgument(sasTokenProviderImplementation != null || configuredFixedToken != null, - String.format("The configuration value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); + String.format("The value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); if (sasTokenProviderImplementation != null) { LOG.trace("Using SASTokenProvider class instead of config although both are available for use"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index c6aa185c08e17b..24466c206891a0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -69,7 +69,8 @@ public void testBothProviderFixedTokenConfigured() throws Exception { AbfsConfiguration testAbfsConfig = getConfiguration(); // configuring a SASTokenProvider class: this provides a user delegation SAS - // user delegation SAS Provider is set to easily distinguish between results of filesystem level and blob level operations to ensure correct SAS is chosen, + // user delegation SAS Provider is set + // This easily distinguishes between results of filesystem level and blob level operations to ensure correct SAS is chosen, // when both a provider class and fixed token is configured. testAbfsConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, "org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java index 9863b1d54b2c3d..5e7902adf118ab 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java @@ -33,7 +33,9 @@ public class AccountSASGenerator extends SASGenerator { * https://learn.microsoft.com/en-us/rest/api/storageservices/create-account-sas * @param accountKey: the storage account key */ - public AccountSASGenerator(byte[] accountKey) { super(accountKey); } + public AccountSASGenerator(byte[] accountKey) { + super(accountKey); + } public String getAccountSAS(String accountName) throws AzureBlobFileSystemException { // retaining only the account name diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index 292037671ec76b..86fb5a17cca4f4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -23,15 +23,12 @@ import java.time.Duration; import java.time.format.DateTimeFormatter; import java.time.ZoneId; -import java.util.Base64; import java.util.Locale; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException; -import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** From 7612036b5c96182cb016019efc8013dbf1b605ee Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 8 Dec 2022 17:04:06 +0530 Subject: [PATCH 22/34] HADOOP-18516. Style changes --- .../org/apache/hadoop/fs/azurebfs/services/AbfsClient.java | 1 - .../hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java | 1 - .../org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java | 5 +++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index a4dcc39ac60b02..a7c6207cef1342 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -88,7 +88,6 @@ public class AbfsClient implements Closeable { private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; - // private final String xMsVersion = "2019-12-12"; private final String xMsVersion = "2021-10-04"; private final ExponentialRetryPolicy retryPolicy; private final String filesystem; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java index 1715ab418bbd18..8965fd75f8ed8f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockSASTokenProvider.java @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.azurebfs.AbstractAbfsScaleTest; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.security.AccessControlException; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index 86fb5a17cca4f4..eeb7f29a48bb8c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -94,12 +94,13 @@ protected String getCanonicalAccountName(String accountName) throws InvalidConfi // case when accountname starts with a ".": endpoint is present, accountName is null throw new InvalidConfigurationValueException("Account Name is not fully qualified"); } - if (dotIndex > 0) + if (dotIndex > 0) { // case when endpoint is present with accountName return accountName.substring(0, dotIndex); - else + } else { // case when accountName is already canonicalized return accountName; + } } protected String computeHmac256(final String stringToSign) { From 573d4c59bd4c10067bb340fcbff4347bc2dfca8d Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 8 Dec 2022 17:47:42 +0530 Subject: [PATCH 23/34] HADOOP-18516. Updated docs --- .../hadoop-azure/src/site/markdown/abfs.md | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 35d360556047e4..5a4a95c0b090aa 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -315,6 +315,7 @@ driven by them. 1. Deployed in-Azure with the Azure VMs providing OAuth 2.0 tokens to the application, "Managed Instance". 1. Using Shared Access Signature (SAS) tokens provided by a custom implementation of the SASTokenProvider interface. +2. By directly configuring a fixed Shared Access Signature (SAS) token in the account configuration settings files. What can be changed is what secrets/credentials are used to authenticate the caller. @@ -625,6 +626,28 @@ tokens by implementing the SASTokenProvider interface. The declared class must implement `org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider`. +*Note:* When using a token provider implementation that provides a User Delegation SAS Token or Service SAS Token, some operations may be out of scope and may fail. + +### Fixed Shared Access Signature (SAS) Token + +A Shared Access Signature Token can be directly configured in the account settings file. +This should ideally be used for an Account SAS Token, that can be fixed as a constant for an account. + +```xml + + fs.azure.account.auth.type + SAS + + + fs.azure.sas.fixed.token + {SAS Token generated or obtained directly from public interfaces} + Fixed SAS Token directly configured + +``` +*Note:* When `fs.azure.sas.token.provider.type` and `fs.azure.fixed.sas.token` are both configured, precedence will be given to the custom token provider implementation. + + + ## Technical notes ### Proxy setup From cc045ac936ed97acf4d2c91df6cc52cb18e18c4f Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 8 Dec 2022 17:50:34 +0530 Subject: [PATCH 24/34] HADOOP-18516. Added EOF newline --- .../java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index eeb7f29a48bb8c..5520f5848b50e9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -117,4 +117,4 @@ protected String computeHmac256(final String stringToSign) { } return org.apache.hadoop.fs.azurebfs.utils.Base64.encode(hmac); } -} \ No newline at end of file +} From 49c2f703b2c5b470c0a6919296a982dc78e393f3 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Fri, 9 Dec 2022 11:20:19 +0530 Subject: [PATCH 25/34] HADOOP-18516. Whitespace fix in doc --- hadoop-tools/hadoop-azure/src/site/markdown/abfs.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 5a4a95c0b090aa..2395fd18590eda 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -630,9 +630,7 @@ The declared class must implement `org.apache.hadoop.fs.azurebfs.extensions.SAST ### Fixed Shared Access Signature (SAS) Token -A Shared Access Signature Token can be directly configured in the account settings file. -This should ideally be used for an Account SAS Token, that can be fixed as a constant for an account. - +A Shared Access Signature Token can be directly configured in the account settings file. This should ideally be used for an Account SAS Token, that can be fixed as a constant for an account. ```xml fs.azure.account.auth.type @@ -645,8 +643,6 @@ This should ideally be used for an Account SAS Token, that can be fixed as a con ``` *Note:* When `fs.azure.sas.token.provider.type` and `fs.azure.fixed.sas.token` are both configured, precedence will be given to the custom token provider implementation. - - ## Technical notes From 29a13e0e4280d32c15f8453cf2e9b076cf6ae337 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 14 Dec 2022 14:35:43 +0530 Subject: [PATCH 26/34] HADOOP-18516. Revert xms version update --- .../java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index a7c6207cef1342..7a866d062d2295 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -88,7 +88,7 @@ public class AbfsClient implements Closeable { private final URL baseUrl; private final SharedKeyCredentials sharedKeyCredentials; - private final String xMsVersion = "2021-10-04"; + private final String xMsVersion = "2019-12-12"; private final ExponentialRetryPolicy retryPolicy; private final String filesystem; private final AbfsConfiguration abfsConfiguration; From 7a3ea12b0096a88fabe4e2b6a476626801a41bdd Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 14 Dec 2022 14:54:11 +0530 Subject: [PATCH 27/34] Removed full import --- .../java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index 5520f5848b50e9..4fe0bc81789e5e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; +import org.apache.hadoop.fs.azurebfs.utils.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** @@ -115,6 +116,6 @@ protected String computeHmac256(final String stringToSign) { synchronized (this) { hmac = hmacSha256.doFinal(utf8Bytes); } - return org.apache.hadoop.fs.azurebfs.utils.Base64.encode(hmac); + return Base64.encode(hmac); } } From b603468f93a9af1c0d44c0527ad8abdfce011c1a Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 14 Dec 2022 14:55:27 +0530 Subject: [PATCH 28/34] HADOOP-18516. Updated log comment --- .../java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 79302785eb84cf..da131cb1b144bb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -917,7 +917,7 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio String.format("The value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); if (sasTokenProviderImplementation != null) { - LOG.trace("Using SASTokenProvider class instead of config although both are available for use"); + LOG.trace("Using SASTokenProvider class because it is given precedence when it is set"); SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderImplementation, rawConfig); Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderImplementation)); From 2a896e9fe4138edc20de1a80e2f8fd7070916c32 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 22 Dec 2022 13:23:16 +0530 Subject: [PATCH 29/34] HADOOP-18516. Minor changes --- .../ITestAzureBlobFileSystemChooseSAS.java | 22 +++++++++++++------ .../fs/azurebfs/utils/SASGenerator.java | 1 - 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 24466c206891a0..26efa5c45a388c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -113,10 +113,15 @@ public void testOnlyFixedTokenConfigured() throws IOException { // attempting an operation using the selected SAS Token // as an account SAS is configured, both filesystem level operations (on root) and blob level operations should succeed - newTestFs.getFileStatus(new Path("/")); - Path testPath = new Path("/testCorrectSASToken"); - newTestFs.create(testPath).close(); - newTestFs.delete(new Path("/"), true); + try { + newTestFs.getFileStatus(new Path("/")); + Path testPath = new Path("/testCorrectSASToken"); + newTestFs.create(testPath).close(); + newTestFs.delete(new Path("/"), true); + } catch (Exception e) { + fail("Exception has been thrown: "+e.getMessage()); + } + } /** @@ -124,13 +129,16 @@ public void testOnlyFixedTokenConfigured() throws IOException { * whether the code errors out at the initialization stage itself * @throws IOException */ - @Test(expected = TokenAccessProviderException.class) - public void testBothProviderFixedTokenUnset() throws IOException { + @Test + public void testBothProviderFixedTokenUnset() throws Exception { AbfsConfiguration testAbfsConfig = getConfiguration(); testAbfsConfig.unset(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE); testAbfsConfig.unset(FS_AZURE_SAS_FIXED_TOKEN); - AzureBlobFileSystem newTestFs = (AzureBlobFileSystem) FileSystem.newInstance(testAbfsConfig.getRawConfiguration()); + intercept(TokenAccessProviderException.class, + () -> { + AzureBlobFileSystem newTestFs = (AzureBlobFileSystem) FileSystem.newInstance(testAbfsConfig.getRawConfiguration()); + }); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index 4fe0bc81789e5e..e0f7c3f8be2ffd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -108,7 +108,6 @@ protected String computeHmac256(final String stringToSign) { byte[] utf8Bytes; try { utf8Bytes = stringToSign.getBytes(StandardCharsets.UTF_8.toString()); - // utf8Bytes = stringToSign.getBytes("UTF-8"); } catch (final UnsupportedEncodingException e) { throw new IllegalArgumentException(e); } From 0f6e222ef2edf98951a758592d66211710442752 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 28 Dec 2022 10:35:12 +0530 Subject: [PATCH 30/34] HADOOP-18516. Style changes --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 2131 ++++++++--------- .../fs/azurebfs/services/AbfsClient.java | 5 +- .../ITestAzureBlobFileSystemChooseSAS.java | 3 +- 3 files changed, 1069 insertions(+), 1070 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index beefd95e66a6bb..cdc069fd35d6a8 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -78,1075 +78,1074 @@ */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class AbfsConfiguration{ - - private final Configuration rawConfig; - private final String accountName; - private final boolean isSecure; - private static final Logger LOG = LoggerFactory.getLogger(AbfsConfiguration.class); - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_IS_HNS_ENABLED, - DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED) - private String isNamespaceEnabledAccount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_CONCURRENT_REQUESTS, - DefaultValue = -1) - private int writeMaxConcurrentRequestCount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_REQUESTS_TO_QUEUE, - DefaultValue = -1) - private int maxWriteRequestsToQueue; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_BUFFER_SIZE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_WRITE_BUFFER_SIZE) - private int writeBufferSize; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION, - DefaultValue = DEFAULT_AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION) - private boolean enableSmallWriteOptimization; - - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = AZURE_READ_SMALL_FILES_COMPLETELY, - DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY) - private boolean readSmallFilesCompletely; - - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, - DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ) - private boolean optimizeFooterRead; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, - DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED) - private boolean accountThrottlingEnabled; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_BUFFER_SIZE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_READ_BUFFER_SIZE) - private int readBufferSize; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_AHEAD_RANGE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_READ_AHEAD_RANGE) - private int readAheadRange; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MIN_BACKOFF_INTERVAL, - DefaultValue = DEFAULT_MIN_BACKOFF_INTERVAL) - private int minBackoffInterval; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_BACKOFF_INTERVAL, - DefaultValue = DEFAULT_MAX_BACKOFF_INTERVAL) - private int maxBackoffInterval; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BACKOFF_INTERVAL, - DefaultValue = DEFAULT_BACKOFF_INTERVAL) - private int backoffInterval; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_IO_RETRIES, - MinValue = 0, - DefaultValue = DEFAULT_MAX_RETRY_ATTEMPTS) - private int maxIoRetries; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, - MinValue = 0, - DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT) - private int customTokenFetchRetryCount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS) - private int oauthTokenFetchRetryCount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL) - private int oauthTokenFetchRetryMinBackoff; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL) - private int oauthTokenFetchRetryMaxBackoff; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF) - private int oauthTokenFetchRetryDeltaBackoff; - - @LongConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_SIZE_PROPERTY_NAME, - MinValue = 0, - MaxValue = MAX_AZURE_BLOCK_SIZE, - DefaultValue = MAX_AZURE_BLOCK_SIZE) - private long azureBlockSize; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, - DefaultValue = AZURE_BLOCK_LOCATION_HOST_DEFAULT) - private String azureBlockLocationHost; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_OUT, - MinValue = 1, - DefaultValue = MAX_CONCURRENT_WRITE_THREADS) - private int maxConcurrentWriteThreads; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_LIST_MAX_RESULTS, - MinValue = 1, - DefaultValue = DEFAULT_AZURE_LIST_MAX_RESULTS) - private int listMaxResults; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_IN, - MinValue = 1, - DefaultValue = MAX_CONCURRENT_READ_THREADS) - private int maxConcurrentReadThreads; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_TOLERATE_CONCURRENT_APPEND, - DefaultValue = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND) - private boolean tolerateOobAppends; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ATOMIC_RENAME_KEY, - DefaultValue = DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES) - private String azureAtomicDirs; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE, - DefaultValue = DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE) - private boolean enableConditionalCreateOverwrite; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = - FS_AZURE_ENABLE_MKDIR_OVERWRITE, DefaultValue = - DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE) - private boolean mkdirOverwrite; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_APPEND_BLOB_KEY, - DefaultValue = DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES) - private String azureAppendBlobDirs; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_INFINITE_LEASE_KEY, - DefaultValue = DEFAULT_FS_AZURE_INFINITE_LEASE_DIRECTORIES) - private String azureInfiniteLeaseDirs; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_LEASE_THREADS, - MinValue = MIN_LEASE_THREADS, - DefaultValue = DEFAULT_LEASE_THREADS) - private int numLeaseThreads; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - DefaultValue = DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION) - private boolean createRemoteFileSystemDuringInitialization; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION, - DefaultValue = DEFAULT_AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION) - private boolean skipUserGroupMetadataDuringInitialization; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_QUEUE_DEPTH, - DefaultValue = DEFAULT_READ_AHEAD_QUEUE_DEPTH) - private int readAheadQueueDepth; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_BLOCK_SIZE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_READ_AHEAD_BLOCK_SIZE) - private int readAheadBlockSize; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_READ_BUFFER_SIZE, - DefaultValue = DEFAULT_ALWAYS_READ_BUFFER_SIZE) - private boolean alwaysReadBufferSize; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_FLUSH, - DefaultValue = DEFAULT_ENABLE_FLUSH) - private boolean enableFlush; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH, - DefaultValue = DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH) - private boolean disableOutputStreamFlush; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_AUTOTHROTTLING, - DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING) - private boolean enableAutoThrottling; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_OPERATION_IDLE_TIMEOUT, - DefaultValue = DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS) - private int accountOperationIdleTimeout; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ANALYSIS_PERIOD, - DefaultValue = DEFAULT_ANALYSIS_PERIOD_MS) - private int analysisPeriod; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_IO_RATE_LIMIT, - MinValue = 0, - DefaultValue = RATE_LIMIT_DEFAULT) - private int rateLimit; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY, - DefaultValue = DEFAULT_FS_AZURE_USER_AGENT_PREFIX) - private String userAgentId; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_NAME, - DefaultValue = DEFAULT_VALUE_UNKNOWN) - private String clusterName; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_TYPE, - DefaultValue = DEFAULT_VALUE_UNKNOWN) - private String clusterType; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLIENT_CORRELATIONID, - DefaultValue = EMPTY_STRING) - private String clientCorrelationId; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_DELEGATION_TOKEN, - DefaultValue = DEFAULT_ENABLE_DELEGATION_TOKEN) - private boolean enableDelegationToken; - - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_USE_HTTPS, - DefaultValue = DEFAULT_ENABLE_HTTPS) - private boolean alwaysUseHttps; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USE_UPN, - DefaultValue = DEFAULT_USE_UPN) - private boolean useUpn; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = - FS_AZURE_ENABLE_CHECK_ACCESS, DefaultValue = DEFAULT_ENABLE_CHECK_ACCESS) - private boolean isCheckAccessEnabled; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_LATENCY_TRACK, - DefaultValue = DEFAULT_ABFS_LATENCY_TRACK) - private boolean trackLatency; - - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = FS_AZURE_ENABLE_READAHEAD, - DefaultValue = DEFAULT_ENABLE_READAHEAD) - private boolean enabledReadAhead; - - @LongConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS, - MinValue = 0, - DefaultValue = DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS) - private long sasTokenRenewPeriodForStreamsInSeconds; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = - FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR) - private boolean enableAbfsListIterator; - - public AbfsConfiguration(final Configuration rawConfig, String accountName) - throws IllegalAccessException, InvalidConfigurationValueException, IOException { - this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( - rawConfig, AzureBlobFileSystem.class); - this.accountName = accountName; - this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); - - Field[] fields = this.getClass().getDeclaredFields(); - for (Field field : fields) { - field.setAccessible(true); - if (field.isAnnotationPresent(IntegerConfigurationValidatorAnnotation.class)) { - field.set(this, validateInt(field)); - } else if (field.isAnnotationPresent(IntegerWithOutlierConfigurationValidatorAnnotation.class)) { - field.set(this, validateIntWithOutlier(field)); - } else if (field.isAnnotationPresent(LongConfigurationValidatorAnnotation.class)) { - field.set(this, validateLong(field)); - } else if (field.isAnnotationPresent(StringConfigurationValidatorAnnotation.class)) { - field.set(this, validateString(field)); - } else if (field.isAnnotationPresent(Base64StringConfigurationValidatorAnnotation.class)) { - field.set(this, validateBase64String(field)); - } else if (field.isAnnotationPresent(BooleanConfigurationValidatorAnnotation.class)) { - field.set(this, validateBoolean(field)); - } - } - } - - public Trilean getIsNamespaceEnabledAccount() { - return Trilean.getTrilean(isNamespaceEnabledAccount); - } - - /** - * Gets the Azure Storage account name corresponding to this instance of configuration. - * @return the Azure Storage account name - */ - public String getAccountName() { - return accountName; - } - - /** - * Gets client correlation ID provided in config. - * @return Client Correlation ID config - */ - public String getClientCorrelationId() { - return clientCorrelationId; - } - - /** - * Appends an account name to a configuration key yielding the - * account-specific form. - * @param key Account-agnostic configuration key - * @return Account-specific configuration key - */ - public String accountConf(String key) { - return key + "." + accountName; - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value. - * @param key Account-agnostic configuration key - * @return value if one exists, else null - */ - public String get(String key) { - return rawConfig.get(accountConf(key), rawConfig.get(key)); - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value. - * @param key Account-agnostic configuration key - * @return value if one exists, else the default value - */ - public String getString(String key, String defaultValue) { - return rawConfig.get(accountConf(key), rawConfig.get(key, defaultValue)); - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value, and finally tries the default value. - * @param key Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @return value if one exists, else the default value - */ - public boolean getBoolean(String key, boolean defaultValue) { - return rawConfig.getBoolean(accountConf(key), rawConfig.getBoolean(key, defaultValue)); - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value, and finally tries the default value. - * @param key Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @return value if one exists, else the default value - */ - public long getLong(String key, long defaultValue) { - return rawConfig.getLong(accountConf(key), rawConfig.getLong(key, defaultValue)); - } - - /** - * Returns the account-specific password in string form if it exists, then - * looks for an account-agnostic value. - * @param key Account-agnostic configuration key - * @return value in String form if one exists, else null - * @throws IOException - */ - public String getPasswordString(String key) throws IOException { - char[] passchars = rawConfig.getPassword(accountConf(key)); - if (passchars == null) { - passchars = rawConfig.getPassword(key); - } - if (passchars != null) { - return new String(passchars); - } - return null; - } - - /** - * Returns a value for the key if the value exists and is not null. - * Otherwise, throws {@link ConfigurationPropertyNotFoundException} with - * key name. - * - * @param key Account-agnostic configuration key - * @return value if exists - * @throws IOException if error in fetching password or - * ConfigurationPropertyNotFoundException for missing key - */ - private String getMandatoryPasswordString(String key) throws IOException { - String value = getPasswordString(key); - if (value == null) { - throw new ConfigurationPropertyNotFoundException(key); - } - return value; - } - - /** - * Returns account-specific token provider class if it exists, else checks if - * an account-agnostic setting is present for token provider class if AuthType - * matches with authType passed. - * @param authType AuthType effective on the account - * @param name Account-agnostic configuration key - * @param defaultValue Class returned if none is configured - * @param xface Interface shared by all possible values - * @param Interface class type - * @return Highest-precedence Class object that was found - */ - public Class getTokenProviderClass(AuthType authType, - String name, - Class defaultValue, - Class xface) { - Class tokenProviderClass = getAccountSpecificClass(name, defaultValue, - xface); - - // If there is none set specific for account - // fall back to generic setting if Auth Type matches - if ((tokenProviderClass == null) - && (authType == getAccountAgnosticEnum( - FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey))) { - tokenProviderClass = getAccountAgnosticClass(name, defaultValue, xface); - } - - return (tokenProviderClass == null) - ? null - : tokenProviderClass.asSubclass(xface); - } - - /** - * Returns the account-specific class if it exists, else returns default value. - * @param name Account-agnostic configuration key - * @param defaultValue Class returned if none is configured - * @param xface Interface shared by all possible values - * @param Interface class type - * @return Account specific Class object that was found - */ - public Class getAccountSpecificClass(String name, - Class defaultValue, - Class xface) { - return rawConfig.getClass(accountConf(name), - defaultValue, - xface); - } - - /** - * Returns account-agnostic Class if it exists, else returns the default value. - * @param name Account-agnostic configuration key - * @param defaultValue Class returned if none is configured - * @param xface Interface shared by all possible values - * @param Interface class type - * @return Account-Agnostic Class object that was found - */ - public Class getAccountAgnosticClass(String name, - Class defaultValue, - Class xface) { - return rawConfig.getClass(name, defaultValue, xface); - } - - /** - * Returns the account-specific enum value if it exists, then - * looks for an account-agnostic value. - * @param name Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @param Enum type - * @return enum value if one exists, else null - */ - public > T getEnum(String name, T defaultValue) { - return rawConfig.getEnum(accountConf(name), - rawConfig.getEnum(name, defaultValue)); - } - - /** - * Returns the account-agnostic enum value if it exists, else - * return default. - * @param name Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @param Enum type - * @return enum value if one exists, else null - */ - public > T getAccountAgnosticEnum(String name, T defaultValue) { - return rawConfig.getEnum(name, defaultValue); - } - - /** - * Unsets parameter in the underlying Configuration object. - * Provided only as a convenience; does not add any account logic. - * @param key Configuration key - */ - public void unset(String key) { - rawConfig.unset(key); - } - - /** - * Sets String in the underlying Configuration object. - * Provided only as a convenience; does not add any account logic. - * @param key Configuration key - * @param value Configuration value - */ - public void set(String key, String value) { - rawConfig.set(key, value); - } - - /** - * Sets boolean in the underlying Configuration object. - * Provided only as a convenience; does not add any account logic. - * @param key Configuration key - * @param value Configuration value - */ - public void setBoolean(String key, boolean value) { - rawConfig.setBoolean(key, value); - } - - public boolean isSecureMode() { - return isSecure; - } - - public String getStorageAccountKey() throws AzureBlobFileSystemException { - String key; - String keyProviderClass = get(AZURE_KEY_ACCOUNT_KEYPROVIDER); - KeyProvider keyProvider; - - if (keyProviderClass == null) { - // No key provider was provided so use the provided key as is. - keyProvider = new SimpleKeyProvider(); - } else { - // create an instance of the key provider class and verify it - // implements KeyProvider - Object keyProviderObject; - try { - Class clazz = rawConfig.getClassByName(keyProviderClass); - keyProviderObject = clazz.newInstance(); - } catch (Exception e) { - throw new KeyProviderException("Unable to load key provider class.", e); - } - if (!(keyProviderObject instanceof KeyProvider)) { - throw new KeyProviderException(keyProviderClass - + " specified in config is not a valid KeyProvider class."); - } - keyProvider = (KeyProvider) keyProviderObject; - } - key = keyProvider.getStorageAccountKey(accountName, rawConfig); - - if (key == null) { - throw new ConfigurationPropertyNotFoundException(accountName); - } - - return key; - } - - public Configuration getRawConfiguration() { - return this.rawConfig; - } - - public int getWriteBufferSize() { - return this.writeBufferSize; - } - - public boolean isSmallWriteOptimizationEnabled() { - return this.enableSmallWriteOptimization; - } - - public boolean readSmallFilesCompletely() { - return this.readSmallFilesCompletely; - } - - public boolean optimizeFooterRead() { - return this.optimizeFooterRead; - } - - public int getReadBufferSize() { - return this.readBufferSize; - } - - public int getMinBackoffIntervalMilliseconds() { - return this.minBackoffInterval; - } - - public int getMaxBackoffIntervalMilliseconds() { - return this.maxBackoffInterval; - } - - public int getBackoffIntervalMilliseconds() { - return this.backoffInterval; - } - - public int getMaxIoRetries() { - return this.maxIoRetries; - } - - public int getCustomTokenFetchRetryCount() { - return this.customTokenFetchRetryCount; - } - - public long getAzureBlockSize() { - return this.azureBlockSize; - } - - public boolean isCheckAccessEnabled() { - return this.isCheckAccessEnabled; - } - - public long getSasTokenRenewPeriodForStreamsInSeconds() { - return this.sasTokenRenewPeriodForStreamsInSeconds; - } - - public String getAzureBlockLocationHost() { - return this.azureBlockLocationHost; - } - - public int getMaxConcurrentWriteThreads() { - return this.maxConcurrentWriteThreads; - } - - public int getMaxConcurrentReadThreads() { - return this.maxConcurrentReadThreads; - } - - public int getListMaxResults() { - return this.listMaxResults; - } - - public boolean getTolerateOobAppends() { - return this.tolerateOobAppends; - } - - public String getAzureAtomicRenameDirs() { - return this.azureAtomicDirs; - } - - public boolean isConditionalCreateOverwriteEnabled() { - return this.enableConditionalCreateOverwrite; - } - - public boolean isEnabledMkdirOverwrite() { - return mkdirOverwrite; - } +public class AbfsConfiguration { + + private final Configuration rawConfig; + private final String accountName; + private final boolean isSecure; + private static final Logger LOG = LoggerFactory.getLogger(AbfsConfiguration.class); + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_IS_HNS_ENABLED, + DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED) + private String isNamespaceEnabledAccount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_CONCURRENT_REQUESTS, + DefaultValue = -1) + private int writeMaxConcurrentRequestCount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_REQUESTS_TO_QUEUE, + DefaultValue = -1) + private int maxWriteRequestsToQueue; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_BUFFER_SIZE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_WRITE_BUFFER_SIZE) + private int writeBufferSize; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION, + DefaultValue = DEFAULT_AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION) + private boolean enableSmallWriteOptimization; + + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = AZURE_READ_SMALL_FILES_COMPLETELY, + DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY) + private boolean readSmallFilesCompletely; + + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, + DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ) + private boolean optimizeFooterRead; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, + DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED) + private boolean accountThrottlingEnabled; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_BUFFER_SIZE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_READ_BUFFER_SIZE) + private int readBufferSize; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_AHEAD_RANGE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_READ_AHEAD_RANGE) + private int readAheadRange; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MIN_BACKOFF_INTERVAL, + DefaultValue = DEFAULT_MIN_BACKOFF_INTERVAL) + private int minBackoffInterval; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_BACKOFF_INTERVAL, + DefaultValue = DEFAULT_MAX_BACKOFF_INTERVAL) + private int maxBackoffInterval; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BACKOFF_INTERVAL, + DefaultValue = DEFAULT_BACKOFF_INTERVAL) + private int backoffInterval; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_IO_RETRIES, + MinValue = 0, + DefaultValue = DEFAULT_MAX_RETRY_ATTEMPTS) + private int maxIoRetries; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, + MinValue = 0, + DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT) + private int customTokenFetchRetryCount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS) + private int oauthTokenFetchRetryCount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL) + private int oauthTokenFetchRetryMinBackoff; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL) + private int oauthTokenFetchRetryMaxBackoff; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF) + private int oauthTokenFetchRetryDeltaBackoff; + + @LongConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_SIZE_PROPERTY_NAME, + MinValue = 0, + MaxValue = MAX_AZURE_BLOCK_SIZE, + DefaultValue = MAX_AZURE_BLOCK_SIZE) + private long azureBlockSize; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, + DefaultValue = AZURE_BLOCK_LOCATION_HOST_DEFAULT) + private String azureBlockLocationHost; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_OUT, + MinValue = 1, + DefaultValue = MAX_CONCURRENT_WRITE_THREADS) + private int maxConcurrentWriteThreads; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_LIST_MAX_RESULTS, + MinValue = 1, + DefaultValue = DEFAULT_AZURE_LIST_MAX_RESULTS) + private int listMaxResults; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_IN, + MinValue = 1, + DefaultValue = MAX_CONCURRENT_READ_THREADS) + private int maxConcurrentReadThreads; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_TOLERATE_CONCURRENT_APPEND, + DefaultValue = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND) + private boolean tolerateOobAppends; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ATOMIC_RENAME_KEY, + DefaultValue = DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES) + private String azureAtomicDirs; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE, + DefaultValue = DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE) + private boolean enableConditionalCreateOverwrite; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_MKDIR_OVERWRITE, DefaultValue = + DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE) + private boolean mkdirOverwrite; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_APPEND_BLOB_KEY, + DefaultValue = DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES) + private String azureAppendBlobDirs; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_INFINITE_LEASE_KEY, + DefaultValue = DEFAULT_FS_AZURE_INFINITE_LEASE_DIRECTORIES) + private String azureInfiniteLeaseDirs; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_LEASE_THREADS, + MinValue = MIN_LEASE_THREADS, + DefaultValue = DEFAULT_LEASE_THREADS) + private int numLeaseThreads; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + DefaultValue = DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION) + private boolean createRemoteFileSystemDuringInitialization; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION, + DefaultValue = DEFAULT_AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION) + private boolean skipUserGroupMetadataDuringInitialization; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_QUEUE_DEPTH, + DefaultValue = DEFAULT_READ_AHEAD_QUEUE_DEPTH) + private int readAheadQueueDepth; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_BLOCK_SIZE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_READ_AHEAD_BLOCK_SIZE) + private int readAheadBlockSize; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_READ_BUFFER_SIZE, + DefaultValue = DEFAULT_ALWAYS_READ_BUFFER_SIZE) + private boolean alwaysReadBufferSize; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_FLUSH, + DefaultValue = DEFAULT_ENABLE_FLUSH) + private boolean enableFlush; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH, + DefaultValue = DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH) + private boolean disableOutputStreamFlush; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_AUTOTHROTTLING, + DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING) + private boolean enableAutoThrottling; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_OPERATION_IDLE_TIMEOUT, + DefaultValue = DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS) + private int accountOperationIdleTimeout; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ANALYSIS_PERIOD, + DefaultValue = DEFAULT_ANALYSIS_PERIOD_MS) + private int analysisPeriod; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_IO_RATE_LIMIT, + MinValue = 0, + DefaultValue = RATE_LIMIT_DEFAULT) + private int rateLimit; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY, + DefaultValue = DEFAULT_FS_AZURE_USER_AGENT_PREFIX) + private String userAgentId; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_NAME, + DefaultValue = DEFAULT_VALUE_UNKNOWN) + private String clusterName; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_TYPE, + DefaultValue = DEFAULT_VALUE_UNKNOWN) + private String clusterType; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLIENT_CORRELATIONID, + DefaultValue = EMPTY_STRING) + private String clientCorrelationId; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_DELEGATION_TOKEN, + DefaultValue = DEFAULT_ENABLE_DELEGATION_TOKEN) + private boolean enableDelegationToken; + + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_USE_HTTPS, + DefaultValue = DEFAULT_ENABLE_HTTPS) + private boolean alwaysUseHttps; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USE_UPN, + DefaultValue = DEFAULT_USE_UPN) + private boolean useUpn; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_CHECK_ACCESS, DefaultValue = DEFAULT_ENABLE_CHECK_ACCESS) + private boolean isCheckAccessEnabled; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_LATENCY_TRACK, + DefaultValue = DEFAULT_ABFS_LATENCY_TRACK) + private boolean trackLatency; + + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = FS_AZURE_ENABLE_READAHEAD, + DefaultValue = DEFAULT_ENABLE_READAHEAD) + private boolean enabledReadAhead; + + @LongConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS, + MinValue = 0, + DefaultValue = DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS) + private long sasTokenRenewPeriodForStreamsInSeconds; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR) + private boolean enableAbfsListIterator; + + public AbfsConfiguration(final Configuration rawConfig, String accountName) + throws IllegalAccessException, InvalidConfigurationValueException, IOException { + this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( + rawConfig, AzureBlobFileSystem.class); + this.accountName = accountName; + this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); + + Field[] fields = this.getClass().getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + if (field.isAnnotationPresent(IntegerConfigurationValidatorAnnotation.class)) { + field.set(this, validateInt(field)); + } else if (field.isAnnotationPresent(IntegerWithOutlierConfigurationValidatorAnnotation.class)) { + field.set(this, validateIntWithOutlier(field)); + } else if (field.isAnnotationPresent(LongConfigurationValidatorAnnotation.class)) { + field.set(this, validateLong(field)); + } else if (field.isAnnotationPresent(StringConfigurationValidatorAnnotation.class)) { + field.set(this, validateString(field)); + } else if (field.isAnnotationPresent(Base64StringConfigurationValidatorAnnotation.class)) { + field.set(this, validateBase64String(field)); + } else if (field.isAnnotationPresent(BooleanConfigurationValidatorAnnotation.class)) { + field.set(this, validateBoolean(field)); + } + } + } + + public Trilean getIsNamespaceEnabledAccount() { + return Trilean.getTrilean(isNamespaceEnabledAccount); + } + + /** + * Gets the Azure Storage account name corresponding to this instance of configuration. + * @return the Azure Storage account name + */ + public String getAccountName() { + return accountName; + } + + /** + * Gets client correlation ID provided in config. + * @return Client Correlation ID config + */ + public String getClientCorrelationId() { + return clientCorrelationId; + } + + /** + * Appends an account name to a configuration key yielding the + * account-specific form. + * @param key Account-agnostic configuration key + * @return Account-specific configuration key + */ + public String accountConf(String key) { + return key + "." + accountName; + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value. + * @param key Account-agnostic configuration key + * @return value if one exists, else null + */ + public String get(String key) { + return rawConfig.get(accountConf(key), rawConfig.get(key)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value. + * @param key Account-agnostic configuration key + * @return value if one exists, else the default value + */ + public String getString(String key, String defaultValue) { + return rawConfig.get(accountConf(key), rawConfig.get(key, defaultValue)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value, and finally tries the default value. + * @param key Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @return value if one exists, else the default value + */ + public boolean getBoolean(String key, boolean defaultValue) { + return rawConfig.getBoolean(accountConf(key), rawConfig.getBoolean(key, defaultValue)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value, and finally tries the default value. + * @param key Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @return value if one exists, else the default value + */ + public long getLong(String key, long defaultValue) { + return rawConfig.getLong(accountConf(key), rawConfig.getLong(key, defaultValue)); + } + + /** + * Returns the account-specific password in string form if it exists, then + * looks for an account-agnostic value. + * @param key Account-agnostic configuration key + * @return value in String form if one exists, else null + * @throws IOException + */ + public String getPasswordString(String key) throws IOException { + char[] passchars = rawConfig.getPassword(accountConf(key)); + if (passchars == null) { + passchars = rawConfig.getPassword(key); + } + if (passchars != null) { + return new String(passchars); + } + return null; + } + + /** + * Returns a value for the key if the value exists and is not null. + * Otherwise, throws {@link ConfigurationPropertyNotFoundException} with + * key name. + * + * @param key Account-agnostic configuration key + * @return value if exists + * @throws IOException if error in fetching password or + * ConfigurationPropertyNotFoundException for missing key + */ + private String getMandatoryPasswordString(String key) throws IOException { + String value = getPasswordString(key); + if (value == null) { + throw new ConfigurationPropertyNotFoundException(key); + } + return value; + } + + /** + * Returns account-specific token provider class if it exists, else checks if + * an account-agnostic setting is present for token provider class if AuthType + * matches with authType passed. + * @param authType AuthType effective on the account + * @param name Account-agnostic configuration key + * @param defaultValue Class returned if none is configured + * @param xface Interface shared by all possible values + * @param Interface class type + * @return Highest-precedence Class object that was found + */ + public Class getTokenProviderClass(AuthType authType, + String name, + Class defaultValue, + Class xface) { + Class tokenProviderClass = getAccountSpecificClass(name, defaultValue, + xface); + + // If there is none set specific for account + // fall back to generic setting if Auth Type matches + if ((tokenProviderClass == null) + && (authType == getAccountAgnosticEnum( + FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey))) { + tokenProviderClass = getAccountAgnosticClass(name, defaultValue, xface); + } + + return (tokenProviderClass == null) + ? null + : tokenProviderClass.asSubclass(xface); + } + + /** + * Returns the account-specific class if it exists, else returns default value. + * @param name Account-agnostic configuration key + * @param defaultValue Class returned if none is configured + * @param xface Interface shared by all possible values + * @param Interface class type + * @return Account specific Class object that was found + */ + public Class getAccountSpecificClass(String name, + Class defaultValue, + Class xface) { + return rawConfig.getClass(accountConf(name), + defaultValue, + xface); + } + + /** + * Returns account-agnostic Class if it exists, else returns the default value. + * @param name Account-agnostic configuration key + * @param defaultValue Class returned if none is configured + * @param xface Interface shared by all possible values + * @param Interface class type + * @return Account-Agnostic Class object that was found + */ + public Class getAccountAgnosticClass(String name, + Class defaultValue, + Class xface) { + return rawConfig.getClass(name, defaultValue, xface); + } - public String getAppendBlobDirs() { - return this.azureAppendBlobDirs; - } + /** + * Returns the account-specific enum value if it exists, then + * looks for an account-agnostic value. + * @param name Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @param Enum type + * @return enum value if one exists, else null + */ + public > T getEnum(String name, T defaultValue) { + return rawConfig.getEnum(accountConf(name), + rawConfig.getEnum(name, defaultValue)); + } + + /** + * Returns the account-agnostic enum value if it exists, else + * return default. + * @param name Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @param Enum type + * @return enum value if one exists, else null + */ + public > T getAccountAgnosticEnum(String name, T defaultValue) { + return rawConfig.getEnum(name, defaultValue); + } - public boolean accountThrottlingEnabled() { - return accountThrottlingEnabled; - } + /** + * Unsets parameter in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + */ + public void unset(String key) { + rawConfig.unset(key); + } - public String getAzureInfiniteLeaseDirs() { - return this.azureInfiniteLeaseDirs; - } - - public int getNumLeaseThreads() { - return this.numLeaseThreads; - } - - public boolean getCreateRemoteFileSystemDuringInitialization() { - // we do not support creating the filesystem when AuthType is SAS - return this.createRemoteFileSystemDuringInitialization - && this.getAuthType(this.accountName) != AuthType.SAS; - } - - public boolean getSkipUserGroupMetadataDuringInitialization() { - return this.skipUserGroupMetadataDuringInitialization; - } - - public int getReadAheadQueueDepth() { - return this.readAheadQueueDepth; - } - - public int getReadAheadBlockSize() { - return this.readAheadBlockSize; - } - - public boolean shouldReadBufferSizeAlways() { - return this.alwaysReadBufferSize; - } - - public boolean isFlushEnabled() { - return this.enableFlush; - } - - public boolean isOutputStreamFlushDisabled() { - return this.disableOutputStreamFlush; - } - - public boolean isAutoThrottlingEnabled() { - return this.enableAutoThrottling; - } - - public int getAccountOperationIdleTimeout() { - return accountOperationIdleTimeout; - } - - public int getAnalysisPeriod() { - return analysisPeriod; - } - - public int getRateLimit() { - return rateLimit; - } - - public String getCustomUserAgentPrefix() { - return this.userAgentId; - } - - public String getClusterName() { - return this.clusterName; - } - - public String getClusterType() { - return this.clusterType; - } - - public DelegatingSSLSocketFactory.SSLChannelMode getPreferredSSLFactoryOption() { - return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE); - } - - /** - * Enum config to allow user to pick format of x-ms-client-request-id header - * @return tracingContextFormat config if valid, else default ALL_ID_FORMAT - */ - public TracingHeaderFormat getTracingHeaderFormat() { - return getEnum(FS_AZURE_TRACINGHEADER_FORMAT, TracingHeaderFormat.ALL_ID_FORMAT); - } - - public AuthType getAuthType(String accountName) { - return getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); - } - - public boolean isDelegationTokenManagerEnabled() { - return enableDelegationToken; - } - - public AbfsDelegationTokenManager getDelegationTokenManager() throws IOException { - return new AbfsDelegationTokenManager(getRawConfiguration()); - } - - public boolean isHttpsAlwaysUsed() { - return this.alwaysUseHttps; - } - - public boolean isUpnUsed() { - return this.useUpn; - } - - /** - * Whether {@code AbfsClient} should track and send latency info back to storage servers. - * - * @return a boolean indicating whether latency should be tracked. - */ - public boolean shouldTrackLatency() { - return this.trackLatency; - } - - public AccessTokenProvider getTokenProvider() throws TokenAccessProviderException { - AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); - if (authType == AuthType.OAuth) { - try { - Class tokenProviderClass = - getTokenProviderClass(authType, - FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null, - AccessTokenProvider.class); - - AccessTokenProvider tokenProvider; - if (tokenProviderClass == ClientCredsTokenProvider.class) { - String authEndpoint = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); - String clientId = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); - String clientSecret = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET); - tokenProvider = new ClientCredsTokenProvider(authEndpoint, clientId, clientSecret); - LOG.trace("ClientCredsTokenProvider initialized"); - } else if (tokenProviderClass == UserPasswordTokenProvider.class) { - String authEndpoint = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); - String username = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME); - String password = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD); - tokenProvider = new UserPasswordTokenProvider(authEndpoint, username, password); - LOG.trace("UserPasswordTokenProvider initialized"); - } else if (tokenProviderClass == MsiTokenProvider.class) { - String authEndpoint = getTrimmedPasswordString( - FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, - AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); - String tenantGuid = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT); - String clientId = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); - String authority = getTrimmedPasswordString( - FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, - AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); - authority = appendSlashIfNeeded(authority); - tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid, - clientId, authority); - LOG.trace("MsiTokenProvider initialized"); - } else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class) { - String authEndpoint = getTrimmedPasswordString( - FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT, - AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT); - String refreshToken = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN); - String clientId = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); - tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint, - clientId, refreshToken); - LOG.trace("RefreshTokenBasedTokenProvider initialized"); + /** + * Sets String in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + * @param value Configuration value + */ + public void set(String key, String value) { + rawConfig.set(key, value); + } + + /** + * Sets boolean in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + * @param value Configuration value + */ + public void setBoolean(String key, boolean value) { + rawConfig.setBoolean(key, value); + } + + public boolean isSecureMode() { + return isSecure; + } + + public String getStorageAccountKey() throws AzureBlobFileSystemException { + String key; + String keyProviderClass = get(AZURE_KEY_ACCOUNT_KEYPROVIDER); + KeyProvider keyProvider; + + if (keyProviderClass == null) { + // No key provider was provided so use the provided key as is. + keyProvider = new SimpleKeyProvider(); } else { - throw new IllegalArgumentException("Failed to initialize " + tokenProviderClass); + // create an instance of the key provider class and verify it + // implements KeyProvider + Object keyProviderObject; + try { + Class clazz = rawConfig.getClassByName(keyProviderClass); + keyProviderObject = clazz.newInstance(); + } catch (Exception e) { + throw new KeyProviderException("Unable to load key provider class.", e); + } + if (!(keyProviderObject instanceof KeyProvider)) { + throw new KeyProviderException(keyProviderClass + + " specified in config is not a valid KeyProvider class."); + } + keyProvider = (KeyProvider) keyProviderObject; } - return tokenProvider; - } catch(IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new TokenAccessProviderException("Unable to load OAuth token provider class.", e); - } - - } else if (authType == AuthType.Custom) { - try { - String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; - - Class customTokenProviderClass - = getTokenProviderClass(authType, configKey, null, - CustomTokenProviderAdaptee.class); - - if (customTokenProviderClass == null) { - throw new IllegalArgumentException( - String.format("The configuration value for \"%s\" is invalid.", configKey)); + key = keyProvider.getStorageAccountKey(accountName, rawConfig); + + if (key == null) { + throw new ConfigurationPropertyNotFoundException(accountName); } - CustomTokenProviderAdaptee azureTokenProvider = ReflectionUtils - .newInstance(customTokenProviderClass, rawConfig); - if (azureTokenProvider == null) { - throw new IllegalArgumentException("Failed to initialize " + customTokenProviderClass); + + return key; + } + + public Configuration getRawConfiguration() { + return this.rawConfig; + } + + public int getWriteBufferSize() { + return this.writeBufferSize; + } + + public boolean isSmallWriteOptimizationEnabled() { + return this.enableSmallWriteOptimization; + } + + public boolean readSmallFilesCompletely() { + return this.readSmallFilesCompletely; + } + + public boolean optimizeFooterRead() { + return this.optimizeFooterRead; + } + + public int getReadBufferSize() { + return this.readBufferSize; + } + + public int getMinBackoffIntervalMilliseconds() { + return this.minBackoffInterval; + } + + public int getMaxBackoffIntervalMilliseconds() { + return this.maxBackoffInterval; + } + + public int getBackoffIntervalMilliseconds() { + return this.backoffInterval; + } + + public int getMaxIoRetries() { + return this.maxIoRetries; + } + + public int getCustomTokenFetchRetryCount() { + return this.customTokenFetchRetryCount; + } + + public long getAzureBlockSize() { + return this.azureBlockSize; + } + + public boolean isCheckAccessEnabled() { + return this.isCheckAccessEnabled; + } + + public long getSasTokenRenewPeriodForStreamsInSeconds() { + return this.sasTokenRenewPeriodForStreamsInSeconds; + } + + public String getAzureBlockLocationHost() { + return this.azureBlockLocationHost; + } + + public int getMaxConcurrentWriteThreads() { + return this.maxConcurrentWriteThreads; + } + + public int getMaxConcurrentReadThreads() { + return this.maxConcurrentReadThreads; + } + + public int getListMaxResults() { + return this.listMaxResults; + } + + public boolean getTolerateOobAppends() { + return this.tolerateOobAppends; + } + + public String getAzureAtomicRenameDirs() { + return this.azureAtomicDirs; + } + + public boolean isConditionalCreateOverwriteEnabled() { + return this.enableConditionalCreateOverwrite; + } + + public boolean isEnabledMkdirOverwrite() { + return mkdirOverwrite; + } + + public String getAppendBlobDirs() { + return this.azureAppendBlobDirs; + } + + public boolean accountThrottlingEnabled() { + return accountThrottlingEnabled; + } + + public String getAzureInfiniteLeaseDirs() { + return this.azureInfiniteLeaseDirs; + } + + public int getNumLeaseThreads() { + return this.numLeaseThreads; + } + + public boolean getCreateRemoteFileSystemDuringInitialization() { + // we do not support creating the filesystem when AuthType is SAS + return this.createRemoteFileSystemDuringInitialization + && this.getAuthType(this.accountName) != AuthType.SAS; + } + + public boolean getSkipUserGroupMetadataDuringInitialization() { + return this.skipUserGroupMetadataDuringInitialization; + } + + public int getReadAheadQueueDepth() { + return this.readAheadQueueDepth; + } + + public int getReadAheadBlockSize() { + return this.readAheadBlockSize; + } + + public boolean shouldReadBufferSizeAlways() { + return this.alwaysReadBufferSize; + } + + public boolean isFlushEnabled() { + return this.enableFlush; + } + + public boolean isOutputStreamFlushDisabled() { + return this.disableOutputStreamFlush; + } + + public boolean isAutoThrottlingEnabled() { + return this.enableAutoThrottling; + } + + public int getAccountOperationIdleTimeout() { + return accountOperationIdleTimeout; + } + + public int getAnalysisPeriod() { + return analysisPeriod; + } + + public int getRateLimit() { + return rateLimit; + } + + public String getCustomUserAgentPrefix() { + return this.userAgentId; + } + + public String getClusterName() { + return this.clusterName; + } + + public String getClusterType() { + return this.clusterType; + } + + public DelegatingSSLSocketFactory.SSLChannelMode getPreferredSSLFactoryOption() { + return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE); + } + + /** + * Enum config to allow user to pick format of x-ms-client-request-id header + * @return tracingContextFormat config if valid, else default ALL_ID_FORMAT + */ + public TracingHeaderFormat getTracingHeaderFormat() { + return getEnum(FS_AZURE_TRACINGHEADER_FORMAT, TracingHeaderFormat.ALL_ID_FORMAT); + } + + public AuthType getAuthType(String accountName) { + return getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); + } + + public boolean isDelegationTokenManagerEnabled() { + return enableDelegationToken; + } + + public AbfsDelegationTokenManager getDelegationTokenManager() throws IOException { + return new AbfsDelegationTokenManager(getRawConfiguration()); + } + + public boolean isHttpsAlwaysUsed() { + return this.alwaysUseHttps; + } + + public boolean isUpnUsed() { + return this.useUpn; + } + + /** + * Whether {@code AbfsClient} should track and send latency info back to storage servers. + * + * @return a boolean indicating whether latency should be tracked. + */ + public boolean shouldTrackLatency() { + return this.trackLatency; + } + + public AccessTokenProvider getTokenProvider() throws TokenAccessProviderException { + AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); + if (authType == AuthType.OAuth) { + try { + Class tokenProviderClass = + getTokenProviderClass(authType, + FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null, + AccessTokenProvider.class); + + AccessTokenProvider tokenProvider; + if (tokenProviderClass == ClientCredsTokenProvider.class) { + String authEndpoint = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); + String clientId = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + String clientSecret = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET); + tokenProvider = new ClientCredsTokenProvider(authEndpoint, clientId, clientSecret); + LOG.trace("ClientCredsTokenProvider initialized"); + } else if (tokenProviderClass == UserPasswordTokenProvider.class) { + String authEndpoint = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); + String username = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME); + String password = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD); + tokenProvider = new UserPasswordTokenProvider(authEndpoint, username, password); + LOG.trace("UserPasswordTokenProvider initialized"); + } else if (tokenProviderClass == MsiTokenProvider.class) { + String authEndpoint = getTrimmedPasswordString( + FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, + AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); + String tenantGuid = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT); + String clientId = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + String authority = getTrimmedPasswordString( + FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, + AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); + authority = appendSlashIfNeeded(authority); + tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid, + clientId, authority); + LOG.trace("MsiTokenProvider initialized"); + } else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class) { + String authEndpoint = getTrimmedPasswordString( + FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT, + AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT); + String refreshToken = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN); + String clientId = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint, + clientId, refreshToken); + LOG.trace("RefreshTokenBasedTokenProvider initialized"); + } else { + throw new IllegalArgumentException("Failed to initialize " + tokenProviderClass); + } + return tokenProvider; + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new TokenAccessProviderException("Unable to load OAuth token provider class.", e); + } + + } else if (authType == AuthType.Custom) { + try { + String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; + + Class customTokenProviderClass + = getTokenProviderClass(authType, configKey, null, + CustomTokenProviderAdaptee.class); + + if (customTokenProviderClass == null) { + throw new IllegalArgumentException( + String.format("The configuration value for \"%s\" is invalid.", configKey)); + } + CustomTokenProviderAdaptee azureTokenProvider = ReflectionUtils + .newInstance(customTokenProviderClass, rawConfig); + if (azureTokenProvider == null) { + throw new IllegalArgumentException("Failed to initialize " + customTokenProviderClass); + } + LOG.trace("Initializing {}", customTokenProviderClass.getName()); + azureTokenProvider.initialize(rawConfig, accountName); + LOG.trace("{} init complete", customTokenProviderClass.getName()); + return new CustomTokenProviderAdapter(azureTokenProvider, getCustomTokenFetchRetryCount()); + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new TokenAccessProviderException("Unable to load custom token provider class: " + e, e); + } + + } else { + throw new TokenAccessProviderException(String.format( + "Invalid auth type: %s is being used, expecting OAuth", authType)); } - LOG.trace("Initializing {}", customTokenProviderClass.getName()); - azureTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", customTokenProviderClass.getName()); - return new CustomTokenProviderAdapter(azureTokenProvider, getCustomTokenFetchRetryCount()); - } catch(IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new TokenAccessProviderException("Unable to load custom token provider class: " + e, e); - } - - } else { - throw new TokenAccessProviderException(String.format( - "Invalid auth type: %s is being used, expecting OAuth", authType)); - } - } - - /** - * @return sasTokenProvider object - * @throws AzureBlobFileSystemException - * The following method chooses between a configured fixed sas token, and a user implementation of the SASTokenProvider interface, - * depending on which one is available. In case a user SASTokenProvider implementation is not present, and a fixed token is configured, - * it simply returns null, to set the sasTokenProvider object for current configuration instance to null. - * The fixed token is read and used later. This is done to: - * 1. check for cases where both are not set, while initializing AbfsConfiguration, - * to not proceed further than thi stage itself when none of the options are available. - * 2. avoid using similar tokenProvider implementation to just read the configured fixed token, - * as this could create confusion. The configuration is introduced - * primarily to avoid using any tokenProvider class/interface. Also,implementing the SASTokenProvider requires relying on the raw configurations. - * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, - * and eliminate chances of dynamic modifications and spurious situations. - */ - - public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { - AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); - if (authType != AuthType.SAS) { - throw new SASTokenProviderException(String.format("Invalid auth type: %s is being used, expecting SAS", authType)); - } - - try { - Class sasTokenProviderImplementation = - getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, null, - SASTokenProvider.class); - String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); - - Preconditions.checkArgument(sasTokenProviderImplementation != null || configuredFixedToken != null, - String.format("The value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); - - if (sasTokenProviderImplementation != null) { - LOG.trace("Using SASTokenProvider class because it is given precedence when it is set"); - SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderImplementation, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderImplementation)); - - LOG.trace("Initializing {}", sasTokenProviderImplementation.getName()); - sasTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", sasTokenProviderImplementation.getName()); - return sasTokenProvider; - } - else { - return null; - } - } catch (Exception e) { - throw new TokenAccessProviderException("Unable to load SAS token provider class: " + e, e); - } - } - - public boolean isReadAheadEnabled() { - return this.enabledReadAhead; - } - - @VisibleForTesting - void setReadAheadEnabled(final boolean enabledReadAhead) { - this.enabledReadAhead = enabledReadAhead; - } - - public int getReadAheadRange() { - return this.readAheadRange; - } - - int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class); - String value = get(validator.ConfigurationKey()); - - // validate - return new IntegerConfigurationBasicValidator( - validator.MinValue(), - validator.MaxValue(), - validator.DefaultValue(), - validator.ConfigurationKey(), - validator.ThrowIfInvalid()).validate(value); - } - - int validateIntWithOutlier(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - IntegerWithOutlierConfigurationValidatorAnnotation validator = - field.getAnnotation(IntegerWithOutlierConfigurationValidatorAnnotation.class); - String value = get(validator.ConfigurationKey()); - - // validate - return new IntegerConfigurationBasicValidator( - validator.OutlierValue(), - validator.MinValue(), - validator.MaxValue(), - validator.DefaultValue(), - validator.ConfigurationKey(), - validator.ThrowIfInvalid()).validate(value); - } - - long validateLong(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - LongConfigurationValidatorAnnotation validator = field.getAnnotation(LongConfigurationValidatorAnnotation.class); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new LongConfigurationBasicValidator( - validator.MinValue(), - validator.MaxValue(), - validator.DefaultValue(), - validator.ConfigurationKey(), - validator.ThrowIfInvalid()).validate(value); - } - - String validateString(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - StringConfigurationValidatorAnnotation validator = field.getAnnotation(StringConfigurationValidatorAnnotation.class); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new StringConfigurationBasicValidator( - validator.ConfigurationKey(), - validator.DefaultValue(), - validator.ThrowIfInvalid()).validate(value); - } - - String validateBase64String(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - Base64StringConfigurationValidatorAnnotation validator = field.getAnnotation((Base64StringConfigurationValidatorAnnotation.class)); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new Base64StringConfigurationBasicValidator( - validator.ConfigurationKey(), - validator.DefaultValue(), - validator.ThrowIfInvalid()).validate(value); - } - - boolean validateBoolean(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - BooleanConfigurationValidatorAnnotation validator = field.getAnnotation(BooleanConfigurationValidatorAnnotation.class); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new BooleanConfigurationBasicValidator( - validator.ConfigurationKey(), - validator.DefaultValue(), - validator.ThrowIfInvalid()).validate(value); - } - - public ExponentialRetryPolicy getOauthTokenFetchRetryPolicy() { - return new ExponentialRetryPolicy(oauthTokenFetchRetryCount, - oauthTokenFetchRetryMinBackoff, oauthTokenFetchRetryMaxBackoff, - oauthTokenFetchRetryDeltaBackoff); - } - - public int getWriteMaxConcurrentRequestCount() { - if (this.writeMaxConcurrentRequestCount < 1) { - return 4 * Runtime.getRuntime().availableProcessors(); - } - return this.writeMaxConcurrentRequestCount; - } - - public int getMaxWriteRequestsToQueue() { - if (this.maxWriteRequestsToQueue < 1) { - return 2 * getWriteMaxConcurrentRequestCount(); - } - return this.maxWriteRequestsToQueue; - } - - public boolean enableAbfsListIterator() { - return this.enableAbfsListIterator; - } - - public String getClientProvidedEncryptionKey() { - String accSpecEncKey = accountConf(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY); - return rawConfig.get(accSpecEncKey, null); - } - - @VisibleForTesting - void setReadBufferSize(int bufferSize) { - this.readBufferSize = bufferSize; - } - - @VisibleForTesting - void setWriteBufferSize(int bufferSize) { - this.writeBufferSize = bufferSize; - } - - @VisibleForTesting - void setEnableFlush(boolean enableFlush) { - this.enableFlush = enableFlush; - } - - @VisibleForTesting - void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) { - this.disableOutputStreamFlush = disableOutputStreamFlush; - } - - @VisibleForTesting - void setListMaxResults(int listMaxResults) { - this.listMaxResults = listMaxResults; - } - - @VisibleForTesting - public void setMaxIoRetries(int maxIoRetries) { - this.maxIoRetries = maxIoRetries; - } - - @VisibleForTesting - void setMaxBackoffIntervalMilliseconds(int maxBackoffInterval) { - this.maxBackoffInterval = maxBackoffInterval; - } - - @VisibleForTesting - void setIsNamespaceEnabledAccount(String isNamespaceEnabledAccount) { - this.isNamespaceEnabledAccount = isNamespaceEnabledAccount; - } - - private String getTrimmedPasswordString(String key, String defaultValue) throws IOException { - String value = getPasswordString(key); - if (StringUtils.isBlank(value)) { - value = defaultValue; - } - return value.trim(); - } - - private String appendSlashIfNeeded(String authority) { - if (!authority.endsWith(AbfsHttpConstants.FORWARD_SLASH)) { - authority = authority + AbfsHttpConstants.FORWARD_SLASH; - } - return authority; - } - - @VisibleForTesting - public void setReadSmallFilesCompletely(boolean readSmallFilesCompletely) { - this.readSmallFilesCompletely = readSmallFilesCompletely; - } - - @VisibleForTesting - public void setOptimizeFooterRead(boolean optimizeFooterRead) { - this.optimizeFooterRead = optimizeFooterRead; - } - - @VisibleForTesting - public void setEnableAbfsListIterator(boolean enableAbfsListIterator) { - this.enableAbfsListIterator = enableAbfsListIterator; - } + } + + /** + * The following method chooses between a configured fixed sas token, and a user implementation of the SASTokenProvider interface, + * depending on which one is available. In case a user SASTokenProvider implementation is not present, and a fixed token is configured, + * it simply returns null, to set the sasTokenProvider object for current configuration instance to null. + * The fixed token is read and used later. This is done to: + * 1. check for cases where both are not set, while initializing AbfsConfiguration, + * to not proceed further than thi stage itself when none of the options are available. + * 2. avoid using similar tokenProvider implementation to just read the configured fixed token, + * as this could create confusion. The configuration is introduced + * primarily to avoid using any tokenProvider class/interface. Also,implementing the SASTokenProvider requires relying on the raw configurations. + * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, + * and eliminate chances of dynamic modifications and spurious situations. + * @return sasTokenProvider object + * @throws AzureBlobFileSystemException + */ + + public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { + AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); + if (authType != AuthType.SAS) { + throw new SASTokenProviderException(String.format("Invalid auth type: %s is being used, expecting SAS", authType)); + } + + try { + Class sasTokenProviderImplementation = + getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, null, + SASTokenProvider.class); + String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); + + Preconditions.checkArgument(!(sasTokenProviderImplementation == null && configuredFixedToken == null), + String.format("The value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); + + if (sasTokenProviderImplementation != null) { + LOG.trace("Using SASTokenProvider class because it is given precedence when it is set"); + SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderImplementation, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderImplementation)); + + LOG.trace("Initializing {}", sasTokenProviderImplementation.getName()); + sasTokenProvider.initialize(rawConfig, accountName); + LOG.trace("{} init complete", sasTokenProviderImplementation.getName()); + return sasTokenProvider; + } else { + return null; + } + } catch (Exception e) { + throw new TokenAccessProviderException("Unable to load SAS token provider class: " + e, e); + } + } + + public boolean isReadAheadEnabled() { + return this.enabledReadAhead; + } + + @VisibleForTesting + void setReadAheadEnabled(final boolean enabledReadAhead) { + this.enabledReadAhead = enabledReadAhead; + } + + public int getReadAheadRange() { + return this.readAheadRange; + } + + int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class); + String value = get(validator.ConfigurationKey()); + + // validate + return new IntegerConfigurationBasicValidator( + validator.MinValue(), + validator.MaxValue(), + validator.DefaultValue(), + validator.ConfigurationKey(), + validator.ThrowIfInvalid()).validate(value); + } + + int validateIntWithOutlier(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + IntegerWithOutlierConfigurationValidatorAnnotation validator = + field.getAnnotation(IntegerWithOutlierConfigurationValidatorAnnotation.class); + String value = get(validator.ConfigurationKey()); + + // validate + return new IntegerConfigurationBasicValidator( + validator.OutlierValue(), + validator.MinValue(), + validator.MaxValue(), + validator.DefaultValue(), + validator.ConfigurationKey(), + validator.ThrowIfInvalid()).validate(value); + } + + long validateLong(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + LongConfigurationValidatorAnnotation validator = field.getAnnotation(LongConfigurationValidatorAnnotation.class); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new LongConfigurationBasicValidator( + validator.MinValue(), + validator.MaxValue(), + validator.DefaultValue(), + validator.ConfigurationKey(), + validator.ThrowIfInvalid()).validate(value); + } + + String validateString(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + StringConfigurationValidatorAnnotation validator = field.getAnnotation(StringConfigurationValidatorAnnotation.class); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new StringConfigurationBasicValidator( + validator.ConfigurationKey(), + validator.DefaultValue(), + validator.ThrowIfInvalid()).validate(value); + } + + String validateBase64String(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + Base64StringConfigurationValidatorAnnotation validator = field.getAnnotation((Base64StringConfigurationValidatorAnnotation.class)); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new Base64StringConfigurationBasicValidator( + validator.ConfigurationKey(), + validator.DefaultValue(), + validator.ThrowIfInvalid()).validate(value); + } + + boolean validateBoolean(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + BooleanConfigurationValidatorAnnotation validator = field.getAnnotation(BooleanConfigurationValidatorAnnotation.class); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new BooleanConfigurationBasicValidator( + validator.ConfigurationKey(), + validator.DefaultValue(), + validator.ThrowIfInvalid()).validate(value); + } + + public ExponentialRetryPolicy getOauthTokenFetchRetryPolicy() { + return new ExponentialRetryPolicy(oauthTokenFetchRetryCount, + oauthTokenFetchRetryMinBackoff, oauthTokenFetchRetryMaxBackoff, + oauthTokenFetchRetryDeltaBackoff); + } + + public int getWriteMaxConcurrentRequestCount() { + if (this.writeMaxConcurrentRequestCount < 1) { + return 4 * Runtime.getRuntime().availableProcessors(); + } + return this.writeMaxConcurrentRequestCount; + } + + public int getMaxWriteRequestsToQueue() { + if (this.maxWriteRequestsToQueue < 1) { + return 2 * getWriteMaxConcurrentRequestCount(); + } + return this.maxWriteRequestsToQueue; + } + + public boolean enableAbfsListIterator() { + return this.enableAbfsListIterator; + } + + public String getClientProvidedEncryptionKey() { + String accSpecEncKey = accountConf(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY); + return rawConfig.get(accSpecEncKey, null); + } + + @VisibleForTesting + void setReadBufferSize(int bufferSize) { + this.readBufferSize = bufferSize; + } + + @VisibleForTesting + void setWriteBufferSize(int bufferSize) { + this.writeBufferSize = bufferSize; + } + + @VisibleForTesting + void setEnableFlush(boolean enableFlush) { + this.enableFlush = enableFlush; + } + + @VisibleForTesting + void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) { + this.disableOutputStreamFlush = disableOutputStreamFlush; + } + + @VisibleForTesting + void setListMaxResults(int listMaxResults) { + this.listMaxResults = listMaxResults; + } + + @VisibleForTesting + public void setMaxIoRetries(int maxIoRetries) { + this.maxIoRetries = maxIoRetries; + } + + @VisibleForTesting + void setMaxBackoffIntervalMilliseconds(int maxBackoffInterval) { + this.maxBackoffInterval = maxBackoffInterval; + } + + @VisibleForTesting + void setIsNamespaceEnabledAccount(String isNamespaceEnabledAccount) { + this.isNamespaceEnabledAccount = isNamespaceEnabledAccount; + } + + private String getTrimmedPasswordString(String key, String defaultValue) throws IOException { + String value = getPasswordString(key); + if (StringUtils.isBlank(value)) { + value = defaultValue; + } + return value.trim(); + } + + private String appendSlashIfNeeded(String authority) { + if (!authority.endsWith(AbfsHttpConstants.FORWARD_SLASH)) { + authority = authority + AbfsHttpConstants.FORWARD_SLASH; + } + return authority; + } + + @VisibleForTesting + public void setReadSmallFilesCompletely(boolean readSmallFilesCompletely) { + this.readSmallFilesCompletely = readSmallFilesCompletely; + } + + @VisibleForTesting + public void setOptimizeFooterRead(boolean optimizeFooterRead) { + this.optimizeFooterRead = optimizeFooterRead; + } + + @VisibleForTesting + public void setEnableAbfsListIterator(boolean enableAbfsListIterator) { + this.enableAbfsListIterator = enableAbfsListIterator; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 54ede54d471d37..e452cebbb67670 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1109,9 +1109,8 @@ private String chooseSASToken(String operation, String path) throws IOException // chooses the SAS token provider class if it is configured, otherwise reads the configured fixed token if (sasTokenProvider == null) { return abfsConfiguration.get(ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN); - } else { - return sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); } + return sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); } /** @@ -1183,7 +1182,7 @@ protected URL createRequestUrl(final String path, final String query) } catch (AzureBlobFileSystemException ex) { LOG.debug("Unexpected error.", ex); throw new InvalidUriException(path); - } + } final StringBuilder sb = new StringBuilder(); sb.append(base); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 26efa5c45a388c..80a057ab127a23 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -82,7 +82,8 @@ public void testBothProviderFixedTokenConfigured() throws Exception { // testing a file system level operation TracingContext tracingContext = getTestTracingContext(newTestFs, true); - // expected to fail in the ideal case, as delegation SAS will be chosen, provider class is given preference when both are configured + // Expected to fail in the ideal case, as Delegation SAS will be chosen, provider class is given preference when both are configured. + // This is because filesystem level operations are beyond the scope of a Delegation SAS token. intercept(SASTokenProviderException.class, () -> { newTestFs.getAbfsStore().getFilesystemProperties(tracingContext); From 7536b9eecc02a22d73f702fbe5094aabe09ae5c6 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 28 Dec 2022 11:44:25 +0530 Subject: [PATCH 31/34] Revert "HADOOP-18516. Style changes" This reverts commit 0f6e222ef2edf98951a758592d66211710442752. --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 2131 +++++++++-------- .../fs/azurebfs/services/AbfsClient.java | 5 +- .../ITestAzureBlobFileSystemChooseSAS.java | 3 +- 3 files changed, 1070 insertions(+), 1069 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index cdc069fd35d6a8..beefd95e66a6bb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -78,1074 +78,1075 @@ */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class AbfsConfiguration { - - private final Configuration rawConfig; - private final String accountName; - private final boolean isSecure; - private static final Logger LOG = LoggerFactory.getLogger(AbfsConfiguration.class); - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_IS_HNS_ENABLED, - DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED) - private String isNamespaceEnabledAccount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_CONCURRENT_REQUESTS, - DefaultValue = -1) - private int writeMaxConcurrentRequestCount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_REQUESTS_TO_QUEUE, - DefaultValue = -1) - private int maxWriteRequestsToQueue; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_BUFFER_SIZE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_WRITE_BUFFER_SIZE) - private int writeBufferSize; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION, - DefaultValue = DEFAULT_AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION) - private boolean enableSmallWriteOptimization; - - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = AZURE_READ_SMALL_FILES_COMPLETELY, - DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY) - private boolean readSmallFilesCompletely; - - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, - DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ) - private boolean optimizeFooterRead; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, - DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED) - private boolean accountThrottlingEnabled; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_BUFFER_SIZE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_READ_BUFFER_SIZE) - private int readBufferSize; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_AHEAD_RANGE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_READ_AHEAD_RANGE) - private int readAheadRange; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MIN_BACKOFF_INTERVAL, - DefaultValue = DEFAULT_MIN_BACKOFF_INTERVAL) - private int minBackoffInterval; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_BACKOFF_INTERVAL, - DefaultValue = DEFAULT_MAX_BACKOFF_INTERVAL) - private int maxBackoffInterval; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BACKOFF_INTERVAL, - DefaultValue = DEFAULT_BACKOFF_INTERVAL) - private int backoffInterval; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_IO_RETRIES, - MinValue = 0, - DefaultValue = DEFAULT_MAX_RETRY_ATTEMPTS) - private int maxIoRetries; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, - MinValue = 0, - DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT) - private int customTokenFetchRetryCount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS) - private int oauthTokenFetchRetryCount; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL) - private int oauthTokenFetchRetryMinBackoff; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL) - private int oauthTokenFetchRetryMaxBackoff; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF, - MinValue = 0, - DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF) - private int oauthTokenFetchRetryDeltaBackoff; - - @LongConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_SIZE_PROPERTY_NAME, - MinValue = 0, - MaxValue = MAX_AZURE_BLOCK_SIZE, - DefaultValue = MAX_AZURE_BLOCK_SIZE) - private long azureBlockSize; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, - DefaultValue = AZURE_BLOCK_LOCATION_HOST_DEFAULT) - private String azureBlockLocationHost; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_OUT, - MinValue = 1, - DefaultValue = MAX_CONCURRENT_WRITE_THREADS) - private int maxConcurrentWriteThreads; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_LIST_MAX_RESULTS, - MinValue = 1, - DefaultValue = DEFAULT_AZURE_LIST_MAX_RESULTS) - private int listMaxResults; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_IN, - MinValue = 1, - DefaultValue = MAX_CONCURRENT_READ_THREADS) - private int maxConcurrentReadThreads; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_TOLERATE_CONCURRENT_APPEND, - DefaultValue = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND) - private boolean tolerateOobAppends; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ATOMIC_RENAME_KEY, - DefaultValue = DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES) - private String azureAtomicDirs; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE, - DefaultValue = DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE) - private boolean enableConditionalCreateOverwrite; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = - FS_AZURE_ENABLE_MKDIR_OVERWRITE, DefaultValue = - DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE) - private boolean mkdirOverwrite; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_APPEND_BLOB_KEY, - DefaultValue = DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES) - private String azureAppendBlobDirs; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_INFINITE_LEASE_KEY, - DefaultValue = DEFAULT_FS_AZURE_INFINITE_LEASE_DIRECTORIES) - private String azureInfiniteLeaseDirs; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_LEASE_THREADS, - MinValue = MIN_LEASE_THREADS, - DefaultValue = DEFAULT_LEASE_THREADS) - private int numLeaseThreads; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - DefaultValue = DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION) - private boolean createRemoteFileSystemDuringInitialization; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION, - DefaultValue = DEFAULT_AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION) - private boolean skipUserGroupMetadataDuringInitialization; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_QUEUE_DEPTH, - DefaultValue = DEFAULT_READ_AHEAD_QUEUE_DEPTH) - private int readAheadQueueDepth; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_BLOCK_SIZE, - MinValue = MIN_BUFFER_SIZE, - MaxValue = MAX_BUFFER_SIZE, - DefaultValue = DEFAULT_READ_AHEAD_BLOCK_SIZE) - private int readAheadBlockSize; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_READ_BUFFER_SIZE, - DefaultValue = DEFAULT_ALWAYS_READ_BUFFER_SIZE) - private boolean alwaysReadBufferSize; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_FLUSH, - DefaultValue = DEFAULT_ENABLE_FLUSH) - private boolean enableFlush; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH, - DefaultValue = DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH) - private boolean disableOutputStreamFlush; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_AUTOTHROTTLING, - DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING) - private boolean enableAutoThrottling; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_OPERATION_IDLE_TIMEOUT, - DefaultValue = DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS) - private int accountOperationIdleTimeout; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ANALYSIS_PERIOD, - DefaultValue = DEFAULT_ANALYSIS_PERIOD_MS) - private int analysisPeriod; - - @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_IO_RATE_LIMIT, - MinValue = 0, - DefaultValue = RATE_LIMIT_DEFAULT) - private int rateLimit; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY, - DefaultValue = DEFAULT_FS_AZURE_USER_AGENT_PREFIX) - private String userAgentId; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_NAME, - DefaultValue = DEFAULT_VALUE_UNKNOWN) - private String clusterName; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_TYPE, - DefaultValue = DEFAULT_VALUE_UNKNOWN) - private String clusterType; - - @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLIENT_CORRELATIONID, - DefaultValue = EMPTY_STRING) - private String clientCorrelationId; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_DELEGATION_TOKEN, - DefaultValue = DEFAULT_ENABLE_DELEGATION_TOKEN) - private boolean enableDelegationToken; - - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_USE_HTTPS, - DefaultValue = DEFAULT_ENABLE_HTTPS) - private boolean alwaysUseHttps; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USE_UPN, - DefaultValue = DEFAULT_USE_UPN) - private boolean useUpn; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = - FS_AZURE_ENABLE_CHECK_ACCESS, DefaultValue = DEFAULT_ENABLE_CHECK_ACCESS) - private boolean isCheckAccessEnabled; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_LATENCY_TRACK, - DefaultValue = DEFAULT_ABFS_LATENCY_TRACK) - private boolean trackLatency; - - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = FS_AZURE_ENABLE_READAHEAD, - DefaultValue = DEFAULT_ENABLE_READAHEAD) - private boolean enabledReadAhead; - - @LongConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS, - MinValue = 0, - DefaultValue = DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS) - private long sasTokenRenewPeriodForStreamsInSeconds; - - @BooleanConfigurationValidatorAnnotation(ConfigurationKey = - FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR) - private boolean enableAbfsListIterator; - - public AbfsConfiguration(final Configuration rawConfig, String accountName) - throws IllegalAccessException, InvalidConfigurationValueException, IOException { - this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( - rawConfig, AzureBlobFileSystem.class); - this.accountName = accountName; - this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); - - Field[] fields = this.getClass().getDeclaredFields(); - for (Field field : fields) { - field.setAccessible(true); - if (field.isAnnotationPresent(IntegerConfigurationValidatorAnnotation.class)) { - field.set(this, validateInt(field)); - } else if (field.isAnnotationPresent(IntegerWithOutlierConfigurationValidatorAnnotation.class)) { - field.set(this, validateIntWithOutlier(field)); - } else if (field.isAnnotationPresent(LongConfigurationValidatorAnnotation.class)) { - field.set(this, validateLong(field)); - } else if (field.isAnnotationPresent(StringConfigurationValidatorAnnotation.class)) { - field.set(this, validateString(field)); - } else if (field.isAnnotationPresent(Base64StringConfigurationValidatorAnnotation.class)) { - field.set(this, validateBase64String(field)); - } else if (field.isAnnotationPresent(BooleanConfigurationValidatorAnnotation.class)) { - field.set(this, validateBoolean(field)); - } - } - } - - public Trilean getIsNamespaceEnabledAccount() { - return Trilean.getTrilean(isNamespaceEnabledAccount); - } - - /** - * Gets the Azure Storage account name corresponding to this instance of configuration. - * @return the Azure Storage account name - */ - public String getAccountName() { - return accountName; - } - - /** - * Gets client correlation ID provided in config. - * @return Client Correlation ID config - */ - public String getClientCorrelationId() { - return clientCorrelationId; - } - - /** - * Appends an account name to a configuration key yielding the - * account-specific form. - * @param key Account-agnostic configuration key - * @return Account-specific configuration key - */ - public String accountConf(String key) { - return key + "." + accountName; - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value. - * @param key Account-agnostic configuration key - * @return value if one exists, else null - */ - public String get(String key) { - return rawConfig.get(accountConf(key), rawConfig.get(key)); - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value. - * @param key Account-agnostic configuration key - * @return value if one exists, else the default value - */ - public String getString(String key, String defaultValue) { - return rawConfig.get(accountConf(key), rawConfig.get(key, defaultValue)); - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value, and finally tries the default value. - * @param key Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @return value if one exists, else the default value - */ - public boolean getBoolean(String key, boolean defaultValue) { - return rawConfig.getBoolean(accountConf(key), rawConfig.getBoolean(key, defaultValue)); - } - - /** - * Returns the account-specific value if it exists, then looks for an - * account-agnostic value, and finally tries the default value. - * @param key Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @return value if one exists, else the default value - */ - public long getLong(String key, long defaultValue) { - return rawConfig.getLong(accountConf(key), rawConfig.getLong(key, defaultValue)); - } - - /** - * Returns the account-specific password in string form if it exists, then - * looks for an account-agnostic value. - * @param key Account-agnostic configuration key - * @return value in String form if one exists, else null - * @throws IOException - */ - public String getPasswordString(String key) throws IOException { - char[] passchars = rawConfig.getPassword(accountConf(key)); - if (passchars == null) { - passchars = rawConfig.getPassword(key); - } - if (passchars != null) { - return new String(passchars); - } - return null; - } - - /** - * Returns a value for the key if the value exists and is not null. - * Otherwise, throws {@link ConfigurationPropertyNotFoundException} with - * key name. - * - * @param key Account-agnostic configuration key - * @return value if exists - * @throws IOException if error in fetching password or - * ConfigurationPropertyNotFoundException for missing key - */ - private String getMandatoryPasswordString(String key) throws IOException { - String value = getPasswordString(key); - if (value == null) { - throw new ConfigurationPropertyNotFoundException(key); - } - return value; - } - - /** - * Returns account-specific token provider class if it exists, else checks if - * an account-agnostic setting is present for token provider class if AuthType - * matches with authType passed. - * @param authType AuthType effective on the account - * @param name Account-agnostic configuration key - * @param defaultValue Class returned if none is configured - * @param xface Interface shared by all possible values - * @param Interface class type - * @return Highest-precedence Class object that was found - */ - public Class getTokenProviderClass(AuthType authType, - String name, - Class defaultValue, - Class xface) { - Class tokenProviderClass = getAccountSpecificClass(name, defaultValue, - xface); - - // If there is none set specific for account - // fall back to generic setting if Auth Type matches - if ((tokenProviderClass == null) - && (authType == getAccountAgnosticEnum( - FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey))) { - tokenProviderClass = getAccountAgnosticClass(name, defaultValue, xface); - } - - return (tokenProviderClass == null) - ? null - : tokenProviderClass.asSubclass(xface); - } - - /** - * Returns the account-specific class if it exists, else returns default value. - * @param name Account-agnostic configuration key - * @param defaultValue Class returned if none is configured - * @param xface Interface shared by all possible values - * @param Interface class type - * @return Account specific Class object that was found - */ - public Class getAccountSpecificClass(String name, - Class defaultValue, - Class xface) { - return rawConfig.getClass(accountConf(name), - defaultValue, - xface); - } - - /** - * Returns account-agnostic Class if it exists, else returns the default value. - * @param name Account-agnostic configuration key - * @param defaultValue Class returned if none is configured - * @param xface Interface shared by all possible values - * @param Interface class type - * @return Account-Agnostic Class object that was found - */ - public Class getAccountAgnosticClass(String name, - Class defaultValue, - Class xface) { - return rawConfig.getClass(name, defaultValue, xface); - } +public class AbfsConfiguration{ + + private final Configuration rawConfig; + private final String accountName; + private final boolean isSecure; + private static final Logger LOG = LoggerFactory.getLogger(AbfsConfiguration.class); + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_IS_HNS_ENABLED, + DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED) + private String isNamespaceEnabledAccount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_CONCURRENT_REQUESTS, + DefaultValue = -1) + private int writeMaxConcurrentRequestCount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_MAX_REQUESTS_TO_QUEUE, + DefaultValue = -1) + private int maxWriteRequestsToQueue; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_WRITE_BUFFER_SIZE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_WRITE_BUFFER_SIZE) + private int writeBufferSize; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION, + DefaultValue = DEFAULT_AZURE_ENABLE_SMALL_WRITE_OPTIMIZATION) + private boolean enableSmallWriteOptimization; + + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = AZURE_READ_SMALL_FILES_COMPLETELY, + DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY) + private boolean readSmallFilesCompletely; + + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, + DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ) + private boolean optimizeFooterRead; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, + DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED) + private boolean accountThrottlingEnabled; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_BUFFER_SIZE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_READ_BUFFER_SIZE) + private int readBufferSize; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_AHEAD_RANGE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_READ_AHEAD_RANGE) + private int readAheadRange; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MIN_BACKOFF_INTERVAL, + DefaultValue = DEFAULT_MIN_BACKOFF_INTERVAL) + private int minBackoffInterval; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_BACKOFF_INTERVAL, + DefaultValue = DEFAULT_MAX_BACKOFF_INTERVAL) + private int maxBackoffInterval; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BACKOFF_INTERVAL, + DefaultValue = DEFAULT_BACKOFF_INTERVAL) + private int backoffInterval; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_MAX_IO_RETRIES, + MinValue = 0, + DefaultValue = DEFAULT_MAX_RETRY_ATTEMPTS) + private int maxIoRetries; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, + MinValue = 0, + DefaultValue = DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT) + private int customTokenFetchRetryCount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_COUNT, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS) + private int oauthTokenFetchRetryCount; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL) + private int oauthTokenFetchRetryMinBackoff; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL) + private int oauthTokenFetchRetryMaxBackoff; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF, + MinValue = 0, + DefaultValue = DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF) + private int oauthTokenFetchRetryDeltaBackoff; + + @LongConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_SIZE_PROPERTY_NAME, + MinValue = 0, + MaxValue = MAX_AZURE_BLOCK_SIZE, + DefaultValue = MAX_AZURE_BLOCK_SIZE) + private long azureBlockSize; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, + DefaultValue = AZURE_BLOCK_LOCATION_HOST_DEFAULT) + private String azureBlockLocationHost; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_OUT, + MinValue = 1, + DefaultValue = MAX_CONCURRENT_WRITE_THREADS) + private int maxConcurrentWriteThreads; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_LIST_MAX_RESULTS, + MinValue = 1, + DefaultValue = DEFAULT_AZURE_LIST_MAX_RESULTS) + private int listMaxResults; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CONCURRENT_CONNECTION_VALUE_IN, + MinValue = 1, + DefaultValue = MAX_CONCURRENT_READ_THREADS) + private int maxConcurrentReadThreads; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_TOLERATE_CONCURRENT_APPEND, + DefaultValue = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND) + private boolean tolerateOobAppends; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ATOMIC_RENAME_KEY, + DefaultValue = DEFAULT_FS_AZURE_ATOMIC_RENAME_DIRECTORIES) + private String azureAtomicDirs; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE, + DefaultValue = DEFAULT_FS_AZURE_ENABLE_CONDITIONAL_CREATE_OVERWRITE) + private boolean enableConditionalCreateOverwrite; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_MKDIR_OVERWRITE, DefaultValue = + DEFAULT_FS_AZURE_ENABLE_MKDIR_OVERWRITE) + private boolean mkdirOverwrite; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_APPEND_BLOB_KEY, + DefaultValue = DEFAULT_FS_AZURE_APPEND_BLOB_DIRECTORIES) + private String azureAppendBlobDirs; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_INFINITE_LEASE_KEY, + DefaultValue = DEFAULT_FS_AZURE_INFINITE_LEASE_DIRECTORIES) + private String azureInfiniteLeaseDirs; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_LEASE_THREADS, + MinValue = MIN_LEASE_THREADS, + DefaultValue = DEFAULT_LEASE_THREADS) + private int numLeaseThreads; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + DefaultValue = DEFAULT_AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION) + private boolean createRemoteFileSystemDuringInitialization; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION, + DefaultValue = DEFAULT_AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION) + private boolean skipUserGroupMetadataDuringInitialization; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_QUEUE_DEPTH, + DefaultValue = DEFAULT_READ_AHEAD_QUEUE_DEPTH) + private int readAheadQueueDepth; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_READ_AHEAD_BLOCK_SIZE, + MinValue = MIN_BUFFER_SIZE, + MaxValue = MAX_BUFFER_SIZE, + DefaultValue = DEFAULT_READ_AHEAD_BLOCK_SIZE) + private int readAheadBlockSize; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_READ_BUFFER_SIZE, + DefaultValue = DEFAULT_ALWAYS_READ_BUFFER_SIZE) + private boolean alwaysReadBufferSize; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_FLUSH, + DefaultValue = DEFAULT_ENABLE_FLUSH) + private boolean enableFlush; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_DISABLE_OUTPUTSTREAM_FLUSH, + DefaultValue = DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH) + private boolean disableOutputStreamFlush; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_AUTOTHROTTLING, + DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING) + private boolean enableAutoThrottling; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_OPERATION_IDLE_TIMEOUT, + DefaultValue = DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS) + private int accountOperationIdleTimeout; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ANALYSIS_PERIOD, + DefaultValue = DEFAULT_ANALYSIS_PERIOD_MS) + private int analysisPeriod; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_IO_RATE_LIMIT, + MinValue = 0, + DefaultValue = RATE_LIMIT_DEFAULT) + private int rateLimit; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY, + DefaultValue = DEFAULT_FS_AZURE_USER_AGENT_PREFIX) + private String userAgentId; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_NAME, + DefaultValue = DEFAULT_VALUE_UNKNOWN) + private String clusterName; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLUSTER_TYPE, + DefaultValue = DEFAULT_VALUE_UNKNOWN) + private String clusterType; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_CLIENT_CORRELATIONID, + DefaultValue = EMPTY_STRING) + private String clientCorrelationId; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ENABLE_DELEGATION_TOKEN, + DefaultValue = DEFAULT_ENABLE_DELEGATION_TOKEN) + private boolean enableDelegationToken; + + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ALWAYS_USE_HTTPS, + DefaultValue = DEFAULT_ENABLE_HTTPS) + private boolean alwaysUseHttps; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USE_UPN, + DefaultValue = DEFAULT_USE_UPN) + private boolean useUpn; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_CHECK_ACCESS, DefaultValue = DEFAULT_ENABLE_CHECK_ACCESS) + private boolean isCheckAccessEnabled; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_LATENCY_TRACK, + DefaultValue = DEFAULT_ABFS_LATENCY_TRACK) + private boolean trackLatency; + + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = FS_AZURE_ENABLE_READAHEAD, + DefaultValue = DEFAULT_ENABLE_READAHEAD) + private boolean enabledReadAhead; + + @LongConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS, + MinValue = 0, + DefaultValue = DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS) + private long sasTokenRenewPeriodForStreamsInSeconds; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = + FS_AZURE_ENABLE_ABFS_LIST_ITERATOR, DefaultValue = DEFAULT_ENABLE_ABFS_LIST_ITERATOR) + private boolean enableAbfsListIterator; + + public AbfsConfiguration(final Configuration rawConfig, String accountName) + throws IllegalAccessException, InvalidConfigurationValueException, IOException { + this.rawConfig = ProviderUtils.excludeIncompatibleCredentialProviders( + rawConfig, AzureBlobFileSystem.class); + this.accountName = accountName; + this.isSecure = getBoolean(FS_AZURE_SECURE_MODE, false); + + Field[] fields = this.getClass().getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + if (field.isAnnotationPresent(IntegerConfigurationValidatorAnnotation.class)) { + field.set(this, validateInt(field)); + } else if (field.isAnnotationPresent(IntegerWithOutlierConfigurationValidatorAnnotation.class)) { + field.set(this, validateIntWithOutlier(field)); + } else if (field.isAnnotationPresent(LongConfigurationValidatorAnnotation.class)) { + field.set(this, validateLong(field)); + } else if (field.isAnnotationPresent(StringConfigurationValidatorAnnotation.class)) { + field.set(this, validateString(field)); + } else if (field.isAnnotationPresent(Base64StringConfigurationValidatorAnnotation.class)) { + field.set(this, validateBase64String(field)); + } else if (field.isAnnotationPresent(BooleanConfigurationValidatorAnnotation.class)) { + field.set(this, validateBoolean(field)); + } + } + } + + public Trilean getIsNamespaceEnabledAccount() { + return Trilean.getTrilean(isNamespaceEnabledAccount); + } + + /** + * Gets the Azure Storage account name corresponding to this instance of configuration. + * @return the Azure Storage account name + */ + public String getAccountName() { + return accountName; + } + + /** + * Gets client correlation ID provided in config. + * @return Client Correlation ID config + */ + public String getClientCorrelationId() { + return clientCorrelationId; + } + + /** + * Appends an account name to a configuration key yielding the + * account-specific form. + * @param key Account-agnostic configuration key + * @return Account-specific configuration key + */ + public String accountConf(String key) { + return key + "." + accountName; + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value. + * @param key Account-agnostic configuration key + * @return value if one exists, else null + */ + public String get(String key) { + return rawConfig.get(accountConf(key), rawConfig.get(key)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value. + * @param key Account-agnostic configuration key + * @return value if one exists, else the default value + */ + public String getString(String key, String defaultValue) { + return rawConfig.get(accountConf(key), rawConfig.get(key, defaultValue)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value, and finally tries the default value. + * @param key Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @return value if one exists, else the default value + */ + public boolean getBoolean(String key, boolean defaultValue) { + return rawConfig.getBoolean(accountConf(key), rawConfig.getBoolean(key, defaultValue)); + } + + /** + * Returns the account-specific value if it exists, then looks for an + * account-agnostic value, and finally tries the default value. + * @param key Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @return value if one exists, else the default value + */ + public long getLong(String key, long defaultValue) { + return rawConfig.getLong(accountConf(key), rawConfig.getLong(key, defaultValue)); + } + + /** + * Returns the account-specific password in string form if it exists, then + * looks for an account-agnostic value. + * @param key Account-agnostic configuration key + * @return value in String form if one exists, else null + * @throws IOException + */ + public String getPasswordString(String key) throws IOException { + char[] passchars = rawConfig.getPassword(accountConf(key)); + if (passchars == null) { + passchars = rawConfig.getPassword(key); + } + if (passchars != null) { + return new String(passchars); + } + return null; + } + + /** + * Returns a value for the key if the value exists and is not null. + * Otherwise, throws {@link ConfigurationPropertyNotFoundException} with + * key name. + * + * @param key Account-agnostic configuration key + * @return value if exists + * @throws IOException if error in fetching password or + * ConfigurationPropertyNotFoundException for missing key + */ + private String getMandatoryPasswordString(String key) throws IOException { + String value = getPasswordString(key); + if (value == null) { + throw new ConfigurationPropertyNotFoundException(key); + } + return value; + } + + /** + * Returns account-specific token provider class if it exists, else checks if + * an account-agnostic setting is present for token provider class if AuthType + * matches with authType passed. + * @param authType AuthType effective on the account + * @param name Account-agnostic configuration key + * @param defaultValue Class returned if none is configured + * @param xface Interface shared by all possible values + * @param Interface class type + * @return Highest-precedence Class object that was found + */ + public Class getTokenProviderClass(AuthType authType, + String name, + Class defaultValue, + Class xface) { + Class tokenProviderClass = getAccountSpecificClass(name, defaultValue, + xface); + + // If there is none set specific for account + // fall back to generic setting if Auth Type matches + if ((tokenProviderClass == null) + && (authType == getAccountAgnosticEnum( + FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey))) { + tokenProviderClass = getAccountAgnosticClass(name, defaultValue, xface); + } + + return (tokenProviderClass == null) + ? null + : tokenProviderClass.asSubclass(xface); + } + + /** + * Returns the account-specific class if it exists, else returns default value. + * @param name Account-agnostic configuration key + * @param defaultValue Class returned if none is configured + * @param xface Interface shared by all possible values + * @param Interface class type + * @return Account specific Class object that was found + */ + public Class getAccountSpecificClass(String name, + Class defaultValue, + Class xface) { + return rawConfig.getClass(accountConf(name), + defaultValue, + xface); + } + + /** + * Returns account-agnostic Class if it exists, else returns the default value. + * @param name Account-agnostic configuration key + * @param defaultValue Class returned if none is configured + * @param xface Interface shared by all possible values + * @param Interface class type + * @return Account-Agnostic Class object that was found + */ + public Class getAccountAgnosticClass(String name, + Class defaultValue, + Class xface) { + return rawConfig.getClass(name, defaultValue, xface); + } + + /** + * Returns the account-specific enum value if it exists, then + * looks for an account-agnostic value. + * @param name Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @param Enum type + * @return enum value if one exists, else null + */ + public > T getEnum(String name, T defaultValue) { + return rawConfig.getEnum(accountConf(name), + rawConfig.getEnum(name, defaultValue)); + } + + /** + * Returns the account-agnostic enum value if it exists, else + * return default. + * @param name Account-agnostic configuration key + * @param defaultValue Value returned if none is configured + * @param Enum type + * @return enum value if one exists, else null + */ + public > T getAccountAgnosticEnum(String name, T defaultValue) { + return rawConfig.getEnum(name, defaultValue); + } + + /** + * Unsets parameter in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + */ + public void unset(String key) { + rawConfig.unset(key); + } + + /** + * Sets String in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + * @param value Configuration value + */ + public void set(String key, String value) { + rawConfig.set(key, value); + } + + /** + * Sets boolean in the underlying Configuration object. + * Provided only as a convenience; does not add any account logic. + * @param key Configuration key + * @param value Configuration value + */ + public void setBoolean(String key, boolean value) { + rawConfig.setBoolean(key, value); + } + + public boolean isSecureMode() { + return isSecure; + } + + public String getStorageAccountKey() throws AzureBlobFileSystemException { + String key; + String keyProviderClass = get(AZURE_KEY_ACCOUNT_KEYPROVIDER); + KeyProvider keyProvider; + + if (keyProviderClass == null) { + // No key provider was provided so use the provided key as is. + keyProvider = new SimpleKeyProvider(); + } else { + // create an instance of the key provider class and verify it + // implements KeyProvider + Object keyProviderObject; + try { + Class clazz = rawConfig.getClassByName(keyProviderClass); + keyProviderObject = clazz.newInstance(); + } catch (Exception e) { + throw new KeyProviderException("Unable to load key provider class.", e); + } + if (!(keyProviderObject instanceof KeyProvider)) { + throw new KeyProviderException(keyProviderClass + + " specified in config is not a valid KeyProvider class."); + } + keyProvider = (KeyProvider) keyProviderObject; + } + key = keyProvider.getStorageAccountKey(accountName, rawConfig); + + if (key == null) { + throw new ConfigurationPropertyNotFoundException(accountName); + } + + return key; + } + + public Configuration getRawConfiguration() { + return this.rawConfig; + } + + public int getWriteBufferSize() { + return this.writeBufferSize; + } + + public boolean isSmallWriteOptimizationEnabled() { + return this.enableSmallWriteOptimization; + } + + public boolean readSmallFilesCompletely() { + return this.readSmallFilesCompletely; + } + + public boolean optimizeFooterRead() { + return this.optimizeFooterRead; + } + + public int getReadBufferSize() { + return this.readBufferSize; + } + + public int getMinBackoffIntervalMilliseconds() { + return this.minBackoffInterval; + } + + public int getMaxBackoffIntervalMilliseconds() { + return this.maxBackoffInterval; + } + + public int getBackoffIntervalMilliseconds() { + return this.backoffInterval; + } + + public int getMaxIoRetries() { + return this.maxIoRetries; + } + + public int getCustomTokenFetchRetryCount() { + return this.customTokenFetchRetryCount; + } + + public long getAzureBlockSize() { + return this.azureBlockSize; + } + + public boolean isCheckAccessEnabled() { + return this.isCheckAccessEnabled; + } + + public long getSasTokenRenewPeriodForStreamsInSeconds() { + return this.sasTokenRenewPeriodForStreamsInSeconds; + } + + public String getAzureBlockLocationHost() { + return this.azureBlockLocationHost; + } + + public int getMaxConcurrentWriteThreads() { + return this.maxConcurrentWriteThreads; + } + + public int getMaxConcurrentReadThreads() { + return this.maxConcurrentReadThreads; + } + + public int getListMaxResults() { + return this.listMaxResults; + } + + public boolean getTolerateOobAppends() { + return this.tolerateOobAppends; + } + + public String getAzureAtomicRenameDirs() { + return this.azureAtomicDirs; + } + + public boolean isConditionalCreateOverwriteEnabled() { + return this.enableConditionalCreateOverwrite; + } + + public boolean isEnabledMkdirOverwrite() { + return mkdirOverwrite; + } - /** - * Returns the account-specific enum value if it exists, then - * looks for an account-agnostic value. - * @param name Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @param Enum type - * @return enum value if one exists, else null - */ - public > T getEnum(String name, T defaultValue) { - return rawConfig.getEnum(accountConf(name), - rawConfig.getEnum(name, defaultValue)); - } - - /** - * Returns the account-agnostic enum value if it exists, else - * return default. - * @param name Account-agnostic configuration key - * @param defaultValue Value returned if none is configured - * @param Enum type - * @return enum value if one exists, else null - */ - public > T getAccountAgnosticEnum(String name, T defaultValue) { - return rawConfig.getEnum(name, defaultValue); - } + public String getAppendBlobDirs() { + return this.azureAppendBlobDirs; + } - /** - * Unsets parameter in the underlying Configuration object. - * Provided only as a convenience; does not add any account logic. - * @param key Configuration key - */ - public void unset(String key) { - rawConfig.unset(key); - } + public boolean accountThrottlingEnabled() { + return accountThrottlingEnabled; + } - /** - * Sets String in the underlying Configuration object. - * Provided only as a convenience; does not add any account logic. - * @param key Configuration key - * @param value Configuration value - */ - public void set(String key, String value) { - rawConfig.set(key, value); - } - - /** - * Sets boolean in the underlying Configuration object. - * Provided only as a convenience; does not add any account logic. - * @param key Configuration key - * @param value Configuration value - */ - public void setBoolean(String key, boolean value) { - rawConfig.setBoolean(key, value); - } - - public boolean isSecureMode() { - return isSecure; - } - - public String getStorageAccountKey() throws AzureBlobFileSystemException { - String key; - String keyProviderClass = get(AZURE_KEY_ACCOUNT_KEYPROVIDER); - KeyProvider keyProvider; - - if (keyProviderClass == null) { - // No key provider was provided so use the provided key as is. - keyProvider = new SimpleKeyProvider(); + public String getAzureInfiniteLeaseDirs() { + return this.azureInfiniteLeaseDirs; + } + + public int getNumLeaseThreads() { + return this.numLeaseThreads; + } + + public boolean getCreateRemoteFileSystemDuringInitialization() { + // we do not support creating the filesystem when AuthType is SAS + return this.createRemoteFileSystemDuringInitialization + && this.getAuthType(this.accountName) != AuthType.SAS; + } + + public boolean getSkipUserGroupMetadataDuringInitialization() { + return this.skipUserGroupMetadataDuringInitialization; + } + + public int getReadAheadQueueDepth() { + return this.readAheadQueueDepth; + } + + public int getReadAheadBlockSize() { + return this.readAheadBlockSize; + } + + public boolean shouldReadBufferSizeAlways() { + return this.alwaysReadBufferSize; + } + + public boolean isFlushEnabled() { + return this.enableFlush; + } + + public boolean isOutputStreamFlushDisabled() { + return this.disableOutputStreamFlush; + } + + public boolean isAutoThrottlingEnabled() { + return this.enableAutoThrottling; + } + + public int getAccountOperationIdleTimeout() { + return accountOperationIdleTimeout; + } + + public int getAnalysisPeriod() { + return analysisPeriod; + } + + public int getRateLimit() { + return rateLimit; + } + + public String getCustomUserAgentPrefix() { + return this.userAgentId; + } + + public String getClusterName() { + return this.clusterName; + } + + public String getClusterType() { + return this.clusterType; + } + + public DelegatingSSLSocketFactory.SSLChannelMode getPreferredSSLFactoryOption() { + return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE); + } + + /** + * Enum config to allow user to pick format of x-ms-client-request-id header + * @return tracingContextFormat config if valid, else default ALL_ID_FORMAT + */ + public TracingHeaderFormat getTracingHeaderFormat() { + return getEnum(FS_AZURE_TRACINGHEADER_FORMAT, TracingHeaderFormat.ALL_ID_FORMAT); + } + + public AuthType getAuthType(String accountName) { + return getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); + } + + public boolean isDelegationTokenManagerEnabled() { + return enableDelegationToken; + } + + public AbfsDelegationTokenManager getDelegationTokenManager() throws IOException { + return new AbfsDelegationTokenManager(getRawConfiguration()); + } + + public boolean isHttpsAlwaysUsed() { + return this.alwaysUseHttps; + } + + public boolean isUpnUsed() { + return this.useUpn; + } + + /** + * Whether {@code AbfsClient} should track and send latency info back to storage servers. + * + * @return a boolean indicating whether latency should be tracked. + */ + public boolean shouldTrackLatency() { + return this.trackLatency; + } + + public AccessTokenProvider getTokenProvider() throws TokenAccessProviderException { + AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); + if (authType == AuthType.OAuth) { + try { + Class tokenProviderClass = + getTokenProviderClass(authType, + FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null, + AccessTokenProvider.class); + + AccessTokenProvider tokenProvider; + if (tokenProviderClass == ClientCredsTokenProvider.class) { + String authEndpoint = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); + String clientId = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + String clientSecret = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET); + tokenProvider = new ClientCredsTokenProvider(authEndpoint, clientId, clientSecret); + LOG.trace("ClientCredsTokenProvider initialized"); + } else if (tokenProviderClass == UserPasswordTokenProvider.class) { + String authEndpoint = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); + String username = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME); + String password = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD); + tokenProvider = new UserPasswordTokenProvider(authEndpoint, username, password); + LOG.trace("UserPasswordTokenProvider initialized"); + } else if (tokenProviderClass == MsiTokenProvider.class) { + String authEndpoint = getTrimmedPasswordString( + FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, + AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); + String tenantGuid = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT); + String clientId = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + String authority = getTrimmedPasswordString( + FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, + AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); + authority = appendSlashIfNeeded(authority); + tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid, + clientId, authority); + LOG.trace("MsiTokenProvider initialized"); + } else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class) { + String authEndpoint = getTrimmedPasswordString( + FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT, + AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT); + String refreshToken = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN); + String clientId = + getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint, + clientId, refreshToken); + LOG.trace("RefreshTokenBasedTokenProvider initialized"); } else { - // create an instance of the key provider class and verify it - // implements KeyProvider - Object keyProviderObject; - try { - Class clazz = rawConfig.getClassByName(keyProviderClass); - keyProviderObject = clazz.newInstance(); - } catch (Exception e) { - throw new KeyProviderException("Unable to load key provider class.", e); - } - if (!(keyProviderObject instanceof KeyProvider)) { - throw new KeyProviderException(keyProviderClass - + " specified in config is not a valid KeyProvider class."); - } - keyProvider = (KeyProvider) keyProviderObject; + throw new IllegalArgumentException("Failed to initialize " + tokenProviderClass); } - key = keyProvider.getStorageAccountKey(accountName, rawConfig); - - if (key == null) { - throw new ConfigurationPropertyNotFoundException(accountName); + return tokenProvider; + } catch(IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new TokenAccessProviderException("Unable to load OAuth token provider class.", e); + } + + } else if (authType == AuthType.Custom) { + try { + String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; + + Class customTokenProviderClass + = getTokenProviderClass(authType, configKey, null, + CustomTokenProviderAdaptee.class); + + if (customTokenProviderClass == null) { + throw new IllegalArgumentException( + String.format("The configuration value for \"%s\" is invalid.", configKey)); } - - return key; - } - - public Configuration getRawConfiguration() { - return this.rawConfig; - } - - public int getWriteBufferSize() { - return this.writeBufferSize; - } - - public boolean isSmallWriteOptimizationEnabled() { - return this.enableSmallWriteOptimization; - } - - public boolean readSmallFilesCompletely() { - return this.readSmallFilesCompletely; - } - - public boolean optimizeFooterRead() { - return this.optimizeFooterRead; - } - - public int getReadBufferSize() { - return this.readBufferSize; - } - - public int getMinBackoffIntervalMilliseconds() { - return this.minBackoffInterval; - } - - public int getMaxBackoffIntervalMilliseconds() { - return this.maxBackoffInterval; - } - - public int getBackoffIntervalMilliseconds() { - return this.backoffInterval; - } - - public int getMaxIoRetries() { - return this.maxIoRetries; - } - - public int getCustomTokenFetchRetryCount() { - return this.customTokenFetchRetryCount; - } - - public long getAzureBlockSize() { - return this.azureBlockSize; - } - - public boolean isCheckAccessEnabled() { - return this.isCheckAccessEnabled; - } - - public long getSasTokenRenewPeriodForStreamsInSeconds() { - return this.sasTokenRenewPeriodForStreamsInSeconds; - } - - public String getAzureBlockLocationHost() { - return this.azureBlockLocationHost; - } - - public int getMaxConcurrentWriteThreads() { - return this.maxConcurrentWriteThreads; - } - - public int getMaxConcurrentReadThreads() { - return this.maxConcurrentReadThreads; - } - - public int getListMaxResults() { - return this.listMaxResults; - } - - public boolean getTolerateOobAppends() { - return this.tolerateOobAppends; - } - - public String getAzureAtomicRenameDirs() { - return this.azureAtomicDirs; - } - - public boolean isConditionalCreateOverwriteEnabled() { - return this.enableConditionalCreateOverwrite; - } - - public boolean isEnabledMkdirOverwrite() { - return mkdirOverwrite; - } - - public String getAppendBlobDirs() { - return this.azureAppendBlobDirs; - } - - public boolean accountThrottlingEnabled() { - return accountThrottlingEnabled; - } - - public String getAzureInfiniteLeaseDirs() { - return this.azureInfiniteLeaseDirs; - } - - public int getNumLeaseThreads() { - return this.numLeaseThreads; - } - - public boolean getCreateRemoteFileSystemDuringInitialization() { - // we do not support creating the filesystem when AuthType is SAS - return this.createRemoteFileSystemDuringInitialization - && this.getAuthType(this.accountName) != AuthType.SAS; - } - - public boolean getSkipUserGroupMetadataDuringInitialization() { - return this.skipUserGroupMetadataDuringInitialization; - } - - public int getReadAheadQueueDepth() { - return this.readAheadQueueDepth; - } - - public int getReadAheadBlockSize() { - return this.readAheadBlockSize; - } - - public boolean shouldReadBufferSizeAlways() { - return this.alwaysReadBufferSize; - } - - public boolean isFlushEnabled() { - return this.enableFlush; - } - - public boolean isOutputStreamFlushDisabled() { - return this.disableOutputStreamFlush; - } - - public boolean isAutoThrottlingEnabled() { - return this.enableAutoThrottling; - } - - public int getAccountOperationIdleTimeout() { - return accountOperationIdleTimeout; - } - - public int getAnalysisPeriod() { - return analysisPeriod; - } - - public int getRateLimit() { - return rateLimit; - } - - public String getCustomUserAgentPrefix() { - return this.userAgentId; - } - - public String getClusterName() { - return this.clusterName; - } - - public String getClusterType() { - return this.clusterType; - } - - public DelegatingSSLSocketFactory.SSLChannelMode getPreferredSSLFactoryOption() { - return getEnum(FS_AZURE_SSL_CHANNEL_MODE_KEY, DEFAULT_FS_AZURE_SSL_CHANNEL_MODE); - } - - /** - * Enum config to allow user to pick format of x-ms-client-request-id header - * @return tracingContextFormat config if valid, else default ALL_ID_FORMAT - */ - public TracingHeaderFormat getTracingHeaderFormat() { - return getEnum(FS_AZURE_TRACINGHEADER_FORMAT, TracingHeaderFormat.ALL_ID_FORMAT); - } - - public AuthType getAuthType(String accountName) { - return getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); - } - - public boolean isDelegationTokenManagerEnabled() { - return enableDelegationToken; - } - - public AbfsDelegationTokenManager getDelegationTokenManager() throws IOException { - return new AbfsDelegationTokenManager(getRawConfiguration()); - } - - public boolean isHttpsAlwaysUsed() { - return this.alwaysUseHttps; - } - - public boolean isUpnUsed() { - return this.useUpn; - } - - /** - * Whether {@code AbfsClient} should track and send latency info back to storage servers. - * - * @return a boolean indicating whether latency should be tracked. - */ - public boolean shouldTrackLatency() { - return this.trackLatency; - } - - public AccessTokenProvider getTokenProvider() throws TokenAccessProviderException { - AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); - if (authType == AuthType.OAuth) { - try { - Class tokenProviderClass = - getTokenProviderClass(authType, - FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, null, - AccessTokenProvider.class); - - AccessTokenProvider tokenProvider; - if (tokenProviderClass == ClientCredsTokenProvider.class) { - String authEndpoint = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); - String clientId = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); - String clientSecret = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET); - tokenProvider = new ClientCredsTokenProvider(authEndpoint, clientId, clientSecret); - LOG.trace("ClientCredsTokenProvider initialized"); - } else if (tokenProviderClass == UserPasswordTokenProvider.class) { - String authEndpoint = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ENDPOINT); - String username = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_NAME); - String password = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD); - tokenProvider = new UserPasswordTokenProvider(authEndpoint, username, password); - LOG.trace("UserPasswordTokenProvider initialized"); - } else if (tokenProviderClass == MsiTokenProvider.class) { - String authEndpoint = getTrimmedPasswordString( - FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, - AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); - String tenantGuid = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT); - String clientId = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); - String authority = getTrimmedPasswordString( - FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, - AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); - authority = appendSlashIfNeeded(authority); - tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid, - clientId, authority); - LOG.trace("MsiTokenProvider initialized"); - } else if (tokenProviderClass == RefreshTokenBasedTokenProvider.class) { - String authEndpoint = getTrimmedPasswordString( - FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT, - AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN_ENDPOINT); - String refreshToken = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN); - String clientId = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); - tokenProvider = new RefreshTokenBasedTokenProvider(authEndpoint, - clientId, refreshToken); - LOG.trace("RefreshTokenBasedTokenProvider initialized"); - } else { - throw new IllegalArgumentException("Failed to initialize " + tokenProviderClass); - } - return tokenProvider; - } catch (IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new TokenAccessProviderException("Unable to load OAuth token provider class.", e); - } - - } else if (authType == AuthType.Custom) { - try { - String configKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; - - Class customTokenProviderClass - = getTokenProviderClass(authType, configKey, null, - CustomTokenProviderAdaptee.class); - - if (customTokenProviderClass == null) { - throw new IllegalArgumentException( - String.format("The configuration value for \"%s\" is invalid.", configKey)); - } - CustomTokenProviderAdaptee azureTokenProvider = ReflectionUtils - .newInstance(customTokenProviderClass, rawConfig); - if (azureTokenProvider == null) { - throw new IllegalArgumentException("Failed to initialize " + customTokenProviderClass); - } - LOG.trace("Initializing {}", customTokenProviderClass.getName()); - azureTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", customTokenProviderClass.getName()); - return new CustomTokenProviderAdapter(azureTokenProvider, getCustomTokenFetchRetryCount()); - } catch (IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new TokenAccessProviderException("Unable to load custom token provider class: " + e, e); - } - - } else { - throw new TokenAccessProviderException(String.format( - "Invalid auth type: %s is being used, expecting OAuth", authType)); + CustomTokenProviderAdaptee azureTokenProvider = ReflectionUtils + .newInstance(customTokenProviderClass, rawConfig); + if (azureTokenProvider == null) { + throw new IllegalArgumentException("Failed to initialize " + customTokenProviderClass); } - } - - /** - * The following method chooses between a configured fixed sas token, and a user implementation of the SASTokenProvider interface, - * depending on which one is available. In case a user SASTokenProvider implementation is not present, and a fixed token is configured, - * it simply returns null, to set the sasTokenProvider object for current configuration instance to null. - * The fixed token is read and used later. This is done to: - * 1. check for cases where both are not set, while initializing AbfsConfiguration, - * to not proceed further than thi stage itself when none of the options are available. - * 2. avoid using similar tokenProvider implementation to just read the configured fixed token, - * as this could create confusion. The configuration is introduced - * primarily to avoid using any tokenProvider class/interface. Also,implementing the SASTokenProvider requires relying on the raw configurations. - * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, - * and eliminate chances of dynamic modifications and spurious situations. - * @return sasTokenProvider object - * @throws AzureBlobFileSystemException - */ - - public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { - AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); - if (authType != AuthType.SAS) { - throw new SASTokenProviderException(String.format("Invalid auth type: %s is being used, expecting SAS", authType)); - } - - try { - Class sasTokenProviderImplementation = - getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, null, - SASTokenProvider.class); - String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); - - Preconditions.checkArgument(!(sasTokenProviderImplementation == null && configuredFixedToken == null), - String.format("The value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); - - if (sasTokenProviderImplementation != null) { - LOG.trace("Using SASTokenProvider class because it is given precedence when it is set"); - SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderImplementation, rawConfig); - Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderImplementation)); - - LOG.trace("Initializing {}", sasTokenProviderImplementation.getName()); - sasTokenProvider.initialize(rawConfig, accountName); - LOG.trace("{} init complete", sasTokenProviderImplementation.getName()); - return sasTokenProvider; - } else { - return null; - } - } catch (Exception e) { - throw new TokenAccessProviderException("Unable to load SAS token provider class: " + e, e); - } - } - - public boolean isReadAheadEnabled() { - return this.enabledReadAhead; - } - - @VisibleForTesting - void setReadAheadEnabled(final boolean enabledReadAhead) { - this.enabledReadAhead = enabledReadAhead; - } - - public int getReadAheadRange() { - return this.readAheadRange; - } - - int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class); - String value = get(validator.ConfigurationKey()); - - // validate - return new IntegerConfigurationBasicValidator( - validator.MinValue(), - validator.MaxValue(), - validator.DefaultValue(), - validator.ConfigurationKey(), - validator.ThrowIfInvalid()).validate(value); - } - - int validateIntWithOutlier(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - IntegerWithOutlierConfigurationValidatorAnnotation validator = - field.getAnnotation(IntegerWithOutlierConfigurationValidatorAnnotation.class); - String value = get(validator.ConfigurationKey()); - - // validate - return new IntegerConfigurationBasicValidator( - validator.OutlierValue(), - validator.MinValue(), - validator.MaxValue(), - validator.DefaultValue(), - validator.ConfigurationKey(), - validator.ThrowIfInvalid()).validate(value); - } - - long validateLong(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - LongConfigurationValidatorAnnotation validator = field.getAnnotation(LongConfigurationValidatorAnnotation.class); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new LongConfigurationBasicValidator( - validator.MinValue(), - validator.MaxValue(), - validator.DefaultValue(), - validator.ConfigurationKey(), - validator.ThrowIfInvalid()).validate(value); - } - - String validateString(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - StringConfigurationValidatorAnnotation validator = field.getAnnotation(StringConfigurationValidatorAnnotation.class); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new StringConfigurationBasicValidator( - validator.ConfigurationKey(), - validator.DefaultValue(), - validator.ThrowIfInvalid()).validate(value); - } - - String validateBase64String(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - Base64StringConfigurationValidatorAnnotation validator = field.getAnnotation((Base64StringConfigurationValidatorAnnotation.class)); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new Base64StringConfigurationBasicValidator( - validator.ConfigurationKey(), - validator.DefaultValue(), - validator.ThrowIfInvalid()).validate(value); - } - - boolean validateBoolean(Field field) throws IllegalAccessException, InvalidConfigurationValueException { - BooleanConfigurationValidatorAnnotation validator = field.getAnnotation(BooleanConfigurationValidatorAnnotation.class); - String value = rawConfig.get(validator.ConfigurationKey()); - - // validate - return new BooleanConfigurationBasicValidator( - validator.ConfigurationKey(), - validator.DefaultValue(), - validator.ThrowIfInvalid()).validate(value); - } - - public ExponentialRetryPolicy getOauthTokenFetchRetryPolicy() { - return new ExponentialRetryPolicy(oauthTokenFetchRetryCount, - oauthTokenFetchRetryMinBackoff, oauthTokenFetchRetryMaxBackoff, - oauthTokenFetchRetryDeltaBackoff); - } - - public int getWriteMaxConcurrentRequestCount() { - if (this.writeMaxConcurrentRequestCount < 1) { - return 4 * Runtime.getRuntime().availableProcessors(); - } - return this.writeMaxConcurrentRequestCount; - } - - public int getMaxWriteRequestsToQueue() { - if (this.maxWriteRequestsToQueue < 1) { - return 2 * getWriteMaxConcurrentRequestCount(); - } - return this.maxWriteRequestsToQueue; - } - - public boolean enableAbfsListIterator() { - return this.enableAbfsListIterator; - } - - public String getClientProvidedEncryptionKey() { - String accSpecEncKey = accountConf(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY); - return rawConfig.get(accSpecEncKey, null); - } - - @VisibleForTesting - void setReadBufferSize(int bufferSize) { - this.readBufferSize = bufferSize; - } - - @VisibleForTesting - void setWriteBufferSize(int bufferSize) { - this.writeBufferSize = bufferSize; - } - - @VisibleForTesting - void setEnableFlush(boolean enableFlush) { - this.enableFlush = enableFlush; - } - - @VisibleForTesting - void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) { - this.disableOutputStreamFlush = disableOutputStreamFlush; - } - - @VisibleForTesting - void setListMaxResults(int listMaxResults) { - this.listMaxResults = listMaxResults; - } - - @VisibleForTesting - public void setMaxIoRetries(int maxIoRetries) { - this.maxIoRetries = maxIoRetries; - } - - @VisibleForTesting - void setMaxBackoffIntervalMilliseconds(int maxBackoffInterval) { - this.maxBackoffInterval = maxBackoffInterval; - } - - @VisibleForTesting - void setIsNamespaceEnabledAccount(String isNamespaceEnabledAccount) { - this.isNamespaceEnabledAccount = isNamespaceEnabledAccount; - } - - private String getTrimmedPasswordString(String key, String defaultValue) throws IOException { - String value = getPasswordString(key); - if (StringUtils.isBlank(value)) { - value = defaultValue; - } - return value.trim(); - } - - private String appendSlashIfNeeded(String authority) { - if (!authority.endsWith(AbfsHttpConstants.FORWARD_SLASH)) { - authority = authority + AbfsHttpConstants.FORWARD_SLASH; - } - return authority; - } - - @VisibleForTesting - public void setReadSmallFilesCompletely(boolean readSmallFilesCompletely) { - this.readSmallFilesCompletely = readSmallFilesCompletely; - } - - @VisibleForTesting - public void setOptimizeFooterRead(boolean optimizeFooterRead) { - this.optimizeFooterRead = optimizeFooterRead; - } - - @VisibleForTesting - public void setEnableAbfsListIterator(boolean enableAbfsListIterator) { - this.enableAbfsListIterator = enableAbfsListIterator; - } + LOG.trace("Initializing {}", customTokenProviderClass.getName()); + azureTokenProvider.initialize(rawConfig, accountName); + LOG.trace("{} init complete", customTokenProviderClass.getName()); + return new CustomTokenProviderAdapter(azureTokenProvider, getCustomTokenFetchRetryCount()); + } catch(IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new TokenAccessProviderException("Unable to load custom token provider class: " + e, e); + } + + } else { + throw new TokenAccessProviderException(String.format( + "Invalid auth type: %s is being used, expecting OAuth", authType)); + } + } + + /** + * @return sasTokenProvider object + * @throws AzureBlobFileSystemException + * The following method chooses between a configured fixed sas token, and a user implementation of the SASTokenProvider interface, + * depending on which one is available. In case a user SASTokenProvider implementation is not present, and a fixed token is configured, + * it simply returns null, to set the sasTokenProvider object for current configuration instance to null. + * The fixed token is read and used later. This is done to: + * 1. check for cases where both are not set, while initializing AbfsConfiguration, + * to not proceed further than thi stage itself when none of the options are available. + * 2. avoid using similar tokenProvider implementation to just read the configured fixed token, + * as this could create confusion. The configuration is introduced + * primarily to avoid using any tokenProvider class/interface. Also,implementing the SASTokenProvider requires relying on the raw configurations. + * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, + * and eliminate chances of dynamic modifications and spurious situations. + */ + + public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { + AuthType authType = getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); + if (authType != AuthType.SAS) { + throw new SASTokenProviderException(String.format("Invalid auth type: %s is being used, expecting SAS", authType)); + } + + try { + Class sasTokenProviderImplementation = + getTokenProviderClass(authType, FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, null, + SASTokenProvider.class); + String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); + + Preconditions.checkArgument(sasTokenProviderImplementation != null || configuredFixedToken != null, + String.format("The value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); + + if (sasTokenProviderImplementation != null) { + LOG.trace("Using SASTokenProvider class because it is given precedence when it is set"); + SASTokenProvider sasTokenProvider = ReflectionUtils.newInstance(sasTokenProviderImplementation, rawConfig); + Preconditions.checkArgument(sasTokenProvider != null, String.format("Failed to initialize %s", sasTokenProviderImplementation)); + + LOG.trace("Initializing {}", sasTokenProviderImplementation.getName()); + sasTokenProvider.initialize(rawConfig, accountName); + LOG.trace("{} init complete", sasTokenProviderImplementation.getName()); + return sasTokenProvider; + } + else { + return null; + } + } catch (Exception e) { + throw new TokenAccessProviderException("Unable to load SAS token provider class: " + e, e); + } + } + + public boolean isReadAheadEnabled() { + return this.enabledReadAhead; + } + + @VisibleForTesting + void setReadAheadEnabled(final boolean enabledReadAhead) { + this.enabledReadAhead = enabledReadAhead; + } + + public int getReadAheadRange() { + return this.readAheadRange; + } + + int validateInt(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + IntegerConfigurationValidatorAnnotation validator = field.getAnnotation(IntegerConfigurationValidatorAnnotation.class); + String value = get(validator.ConfigurationKey()); + + // validate + return new IntegerConfigurationBasicValidator( + validator.MinValue(), + validator.MaxValue(), + validator.DefaultValue(), + validator.ConfigurationKey(), + validator.ThrowIfInvalid()).validate(value); + } + + int validateIntWithOutlier(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + IntegerWithOutlierConfigurationValidatorAnnotation validator = + field.getAnnotation(IntegerWithOutlierConfigurationValidatorAnnotation.class); + String value = get(validator.ConfigurationKey()); + + // validate + return new IntegerConfigurationBasicValidator( + validator.OutlierValue(), + validator.MinValue(), + validator.MaxValue(), + validator.DefaultValue(), + validator.ConfigurationKey(), + validator.ThrowIfInvalid()).validate(value); + } + + long validateLong(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + LongConfigurationValidatorAnnotation validator = field.getAnnotation(LongConfigurationValidatorAnnotation.class); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new LongConfigurationBasicValidator( + validator.MinValue(), + validator.MaxValue(), + validator.DefaultValue(), + validator.ConfigurationKey(), + validator.ThrowIfInvalid()).validate(value); + } + + String validateString(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + StringConfigurationValidatorAnnotation validator = field.getAnnotation(StringConfigurationValidatorAnnotation.class); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new StringConfigurationBasicValidator( + validator.ConfigurationKey(), + validator.DefaultValue(), + validator.ThrowIfInvalid()).validate(value); + } + + String validateBase64String(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + Base64StringConfigurationValidatorAnnotation validator = field.getAnnotation((Base64StringConfigurationValidatorAnnotation.class)); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new Base64StringConfigurationBasicValidator( + validator.ConfigurationKey(), + validator.DefaultValue(), + validator.ThrowIfInvalid()).validate(value); + } + + boolean validateBoolean(Field field) throws IllegalAccessException, InvalidConfigurationValueException { + BooleanConfigurationValidatorAnnotation validator = field.getAnnotation(BooleanConfigurationValidatorAnnotation.class); + String value = rawConfig.get(validator.ConfigurationKey()); + + // validate + return new BooleanConfigurationBasicValidator( + validator.ConfigurationKey(), + validator.DefaultValue(), + validator.ThrowIfInvalid()).validate(value); + } + + public ExponentialRetryPolicy getOauthTokenFetchRetryPolicy() { + return new ExponentialRetryPolicy(oauthTokenFetchRetryCount, + oauthTokenFetchRetryMinBackoff, oauthTokenFetchRetryMaxBackoff, + oauthTokenFetchRetryDeltaBackoff); + } + + public int getWriteMaxConcurrentRequestCount() { + if (this.writeMaxConcurrentRequestCount < 1) { + return 4 * Runtime.getRuntime().availableProcessors(); + } + return this.writeMaxConcurrentRequestCount; + } + + public int getMaxWriteRequestsToQueue() { + if (this.maxWriteRequestsToQueue < 1) { + return 2 * getWriteMaxConcurrentRequestCount(); + } + return this.maxWriteRequestsToQueue; + } + + public boolean enableAbfsListIterator() { + return this.enableAbfsListIterator; + } + + public String getClientProvidedEncryptionKey() { + String accSpecEncKey = accountConf(FS_AZURE_CLIENT_PROVIDED_ENCRYPTION_KEY); + return rawConfig.get(accSpecEncKey, null); + } + + @VisibleForTesting + void setReadBufferSize(int bufferSize) { + this.readBufferSize = bufferSize; + } + + @VisibleForTesting + void setWriteBufferSize(int bufferSize) { + this.writeBufferSize = bufferSize; + } + + @VisibleForTesting + void setEnableFlush(boolean enableFlush) { + this.enableFlush = enableFlush; + } + + @VisibleForTesting + void setDisableOutputStreamFlush(boolean disableOutputStreamFlush) { + this.disableOutputStreamFlush = disableOutputStreamFlush; + } + + @VisibleForTesting + void setListMaxResults(int listMaxResults) { + this.listMaxResults = listMaxResults; + } + + @VisibleForTesting + public void setMaxIoRetries(int maxIoRetries) { + this.maxIoRetries = maxIoRetries; + } + + @VisibleForTesting + void setMaxBackoffIntervalMilliseconds(int maxBackoffInterval) { + this.maxBackoffInterval = maxBackoffInterval; + } + + @VisibleForTesting + void setIsNamespaceEnabledAccount(String isNamespaceEnabledAccount) { + this.isNamespaceEnabledAccount = isNamespaceEnabledAccount; + } + + private String getTrimmedPasswordString(String key, String defaultValue) throws IOException { + String value = getPasswordString(key); + if (StringUtils.isBlank(value)) { + value = defaultValue; + } + return value.trim(); + } + + private String appendSlashIfNeeded(String authority) { + if (!authority.endsWith(AbfsHttpConstants.FORWARD_SLASH)) { + authority = authority + AbfsHttpConstants.FORWARD_SLASH; + } + return authority; + } + + @VisibleForTesting + public void setReadSmallFilesCompletely(boolean readSmallFilesCompletely) { + this.readSmallFilesCompletely = readSmallFilesCompletely; + } + + @VisibleForTesting + public void setOptimizeFooterRead(boolean optimizeFooterRead) { + this.optimizeFooterRead = optimizeFooterRead; + } + + @VisibleForTesting + public void setEnableAbfsListIterator(boolean enableAbfsListIterator) { + this.enableAbfsListIterator = enableAbfsListIterator; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index e452cebbb67670..54ede54d471d37 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1109,8 +1109,9 @@ private String chooseSASToken(String operation, String path) throws IOException // chooses the SAS token provider class if it is configured, otherwise reads the configured fixed token if (sasTokenProvider == null) { return abfsConfiguration.get(ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN); + } else { + return sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); } - return sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); } /** @@ -1182,7 +1183,7 @@ protected URL createRequestUrl(final String path, final String query) } catch (AzureBlobFileSystemException ex) { LOG.debug("Unexpected error.", ex); throw new InvalidUriException(path); - } + } final StringBuilder sb = new StringBuilder(); sb.append(base); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 80a057ab127a23..26efa5c45a388c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -82,8 +82,7 @@ public void testBothProviderFixedTokenConfigured() throws Exception { // testing a file system level operation TracingContext tracingContext = getTestTracingContext(newTestFs, true); - // Expected to fail in the ideal case, as Delegation SAS will be chosen, provider class is given preference when both are configured. - // This is because filesystem level operations are beyond the scope of a Delegation SAS token. + // expected to fail in the ideal case, as delegation SAS will be chosen, provider class is given preference when both are configured intercept(SASTokenProviderException.class, () -> { newTestFs.getAbfsStore().getFilesystemProperties(tracingContext); From 3109c683e902098a641e9f6f602472555af27a87 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 28 Dec 2022 12:18:32 +0530 Subject: [PATCH 32/34] HADOOP-18516. Style changes --- .../org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 3 +-- .../org/apache/hadoop/fs/azurebfs/services/AbfsClient.java | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index be476858e88994..e7a5ddc09f36ed 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -202,8 +202,7 @@ public void initialize(URI uri, Configuration configuration) this.setWorkingDirectory(this.getHomeDirectory()); if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { - TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); + TracingContext tracingContext = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, listener); if (this.tryGetFileStatus(new Path(AbfsHttpConstants.ROOT_PATH), tracingContext) == null) { try { this.createFileSystem(tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 54ede54d471d37..e452cebbb67670 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1109,9 +1109,8 @@ private String chooseSASToken(String operation, String path) throws IOException // chooses the SAS token provider class if it is configured, otherwise reads the configured fixed token if (sasTokenProvider == null) { return abfsConfiguration.get(ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN); - } else { - return sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); } + return sasTokenProvider.getSASToken(this.accountName, this.filesystem, path, operation); } /** @@ -1183,7 +1182,7 @@ protected URL createRequestUrl(final String path, final String query) } catch (AzureBlobFileSystemException ex) { LOG.debug("Unexpected error.", ex); throw new InvalidUriException(path); - } + } final StringBuilder sb = new StringBuilder(); sb.append(base); From 6f7559621a24c810f7d0ecc66e4e88ce696393ab Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 28 Dec 2022 12:28:56 +0530 Subject: [PATCH 33/34] HADOOP-18516. Readability changes --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java | 1 + .../java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 26efa5c45a388c..2a74e6f86bd5bd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -83,6 +83,7 @@ public void testBothProviderFixedTokenConfigured() throws Exception { // testing a file system level operation TracingContext tracingContext = getTestTracingContext(newTestFs, true); // expected to fail in the ideal case, as delegation SAS will be chosen, provider class is given preference when both are configured + // this expectation is because filesystem level operations are beyond the scope of Delegation SAS Token intercept(SASTokenProviderException.class, () -> { newTestFs.getAbfsStore().getFilesystemProperties(tracingContext); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index e0f7c3f8be2ffd..5c9057ff00b2f2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -90,9 +90,12 @@ private void initializeMac() { protected String getCanonicalAccountName(String accountName) throws InvalidConfigurationValueException { // returns the account name without the endpoint + // given accountnames with endpoint have the format accountname.endpoint + // For example, input of xyz.dfs.core.windows.net should return "xyz" only int dotIndex = accountName.indexOf(AbfsHttpConstants.DOT); if (dotIndex == 0) { // case when accountname starts with a ".": endpoint is present, accountName is null + // for example .dfs.azure.com, which is invalid throw new InvalidConfigurationValueException("Account Name is not fully qualified"); } if (dotIndex > 0) { From 07c1ca63bd8c05e7bc522b37c2979ee388a2d711 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 28 Dec 2022 12:54:32 +0530 Subject: [PATCH 34/34] HADOOP-18516. Javadoc + if condition changed in AbfsConfiguration --- .../org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index beefd95e66a6bb..fd06df595a2ed3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -915,8 +915,6 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio } /** - * @return sasTokenProvider object - * @throws AzureBlobFileSystemException * The following method chooses between a configured fixed sas token, and a user implementation of the SASTokenProvider interface, * depending on which one is available. In case a user SASTokenProvider implementation is not present, and a fixed token is configured, * it simply returns null, to set the sasTokenProvider object for current configuration instance to null. @@ -928,6 +926,8 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio * primarily to avoid using any tokenProvider class/interface. Also,implementing the SASTokenProvider requires relying on the raw configurations. * It is more stable to depend on the AbfsConfiguration with which a filesystem is initialized, * and eliminate chances of dynamic modifications and spurious situations. + * @return sasTokenProvider object + * @throws AzureBlobFileSystemException */ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemException { @@ -942,7 +942,7 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio SASTokenProvider.class); String configuredFixedToken = this.rawConfig.get(FS_AZURE_SAS_FIXED_TOKEN, null); - Preconditions.checkArgument(sasTokenProviderImplementation != null || configuredFixedToken != null, + Preconditions.checkArgument(!(sasTokenProviderImplementation == null && configuredFixedToken == null), String.format("The value for both \"%s\" and \"%s\" cannot be invalid.", FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, FS_AZURE_SAS_FIXED_TOKEN)); if (sasTokenProviderImplementation != null) {