From d967c6f9a51196c3e59ed625123cd465f3f36234 Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Sun, 11 Jun 2023 03:43:46 +0800 Subject: [PATCH 1/4] HDFS-17043. HttpFS implementation for getAllErasureCodingPolicies --- .../fs/http/client/HttpFSFileSystem.java | 13 +++++++ .../hadoop/fs/http/server/FSOperations.java | 29 ++++++++++++++ .../http/server/HttpFSParametersProvider.java | 1 + .../hadoop/fs/http/server/HttpFSServer.java | 8 ++++ .../server/metrics/HttpFSServerMetrics.java | 5 +++ .../fs/http/client/BaseTestHttpFSWith.java | 39 ++++++++++++++++++- 6 files changed, 94 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 5d95df8f51649e..2260f923bca5ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -23,6 +23,7 @@ import java.util.EnumSet; import java.util.List; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.MapType; @@ -284,6 +285,7 @@ public enum Operation { HTTP_POST), SATISFYSTORAGEPOLICY(HTTP_PUT), GETSNAPSHOTDIFFLISTING(HTTP_GET), GETFILELINKSTATUS(HTTP_GET), GETSTATUS(HTTP_GET), + GETECPOLICIES(HTTP_GET), GET_BLOCK_LOCATIONS(HTTP_GET); private String httpMethod; @@ -1773,6 +1775,17 @@ public FsStatus getStatus(final Path path) throws IOException { return JsonUtilClient.toFsStatus(json); } + public Collection getAllErasureCodingPolicies() throws IOException { + Map params = new HashMap<>(); + params.put(OP_PARAM, Operation.GETECPOLICIES.toString()); + HttpURLConnection conn = + getConnection(Operation.GETECPOLICIES.getMethod(), params, new Path(getUri() + .toString(), "/"), true); + HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); + return JsonUtilClient.getAllErasureCodingPolicies(json); + } + @VisibleForTesting static BlockLocation[] toBlockLocations(JSONObject json) throws IOException { ObjectMapper mapper = new ObjectMapper(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index d32c19ec9e161a..4ce96a28d2295f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -44,6 +44,8 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -64,6 +66,7 @@ import java.io.OutputStream; import java.util.Collection; import java.util.EnumSet; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -2342,4 +2345,30 @@ public Map execute(FileSystem fs) throws IOException { return toJson(fsStatus); } } + + /** + * Executor that performs a FSGetErasureCodingPolicies operation. + */ + @InterfaceAudience.Private + public static class FSGetErasureCodingPolicies + implements FileSystemAccess.FileSystemExecutor { + + public FSGetErasureCodingPolicies() { + } + + @Override + public String execute(FileSystem fs) throws IOException { + Collection ecPolicyInfos = null; + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem dfs = (DistributedFileSystem) fs; + ecPolicyInfos = dfs.getAllErasureCodingPolicies(); + } else { + throw new UnsupportedOperationException("getErasureCodingPolicies is " + + "not supported for HttpFs on " + fs.getClass() + + ". Please check your fs.defaultFS configuration"); + } + HttpFSServerWebApp.get().getMetrics().incrOpsAllECPolicies(); + return JsonUtil.toJsonString(ecPolicyInfos.stream().toArray(ErasureCodingPolicyInfo[]::new)); + } + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java index 1d319516c2ec84..3477a6fef6e0a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java @@ -130,6 +130,7 @@ public class HttpFSParametersProvider extends ParametersProvider { PARAMS_DEF.put(Operation.SATISFYSTORAGEPOLICY, new Class[] {}); PARAMS_DEF.put(Operation.GETFILELINKSTATUS, new Class[]{}); PARAMS_DEF.put(Operation.GETSTATUS, new Class[]{}); + PARAMS_DEF.put(Operation.GETECPOLICIES, new Class[]{}); PARAMS_DEF.put(Operation.GET_BLOCK_LOCATIONS, new Class[] {OffsetParam.class, LenParam.class}); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java index 8d5921411ee74b..196dc44ec5f9a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java @@ -526,6 +526,14 @@ public InputStream run() throws Exception { response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } + case GETECPOLICIES: { + FSOperations.FSGetErasureCodingPolicies command = + new FSOperations.FSGetErasureCodingPolicies(); + String js = fsExecute(user, command); + AUDIT_LOG.info("[{}]", path); + response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + break; + } case GET_BLOCK_LOCATIONS: { long offset = 0; long len = Long.MAX_VALUE; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java index 6f2c484addb0d6..d65208fdbb8d00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/metrics/HttpFSServerMetrics.java @@ -65,6 +65,7 @@ public class HttpFSServerMetrics { private @Metric MutableCounterLong opsStat; private @Metric MutableCounterLong opsCheckAccess; private @Metric MutableCounterLong opsStatus; + private @Metric MutableCounterLong opsAllECPolicies; private final MetricsRegistry registry = new MetricsRegistry("httpfsserver"); private final String name; @@ -165,4 +166,8 @@ public long getOpsStat() { public void incrOpsStatus() { opsStatus.incr(); } + + public void incrOpsAllECPolicies() { + opsAllECPolicies.incr(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index e61431250b96ca..413c1697cf445c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; @@ -1217,7 +1218,7 @@ protected enum Operation { FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST, GET_SNAPSHOT_LIST, GET_SERVERDEFAULTS, CHECKACCESS, SETECPOLICY, SATISFYSTORAGEPOLICY, GET_SNAPSHOT_DIFF_LISTING, GETFILEBLOCKLOCATIONS, - GETFILELINKSTATUS, GETSTATUS + GETFILELINKSTATUS, GETSTATUS, GETECPOLICIES } private void operation(Operation op) throws Exception { @@ -1366,6 +1367,9 @@ private void operation(Operation op) throws Exception { case GETSTATUS: testGetStatus(); break; + case GETECPOLICIES: + testGetAllEEPolicies(); + break; } } @@ -2111,6 +2115,39 @@ private void testGetStatus() throws Exception { } } + private void testGetAllEEPolicies() throws Exception { + if (isLocalFS()) { + // do not test the getAllEEPolicies for local FS. + return; + } + final Path path = new Path("/foo"); + FileSystem fs = FileSystem.get(path.toUri(), this.getProxiedFSConf()); + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem dfs = + (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf()); + FileSystem httpFs = this.getHttpFSFileSystem(); + + Collection dfsAllErasureCodingPolicies = dfs.getAllErasureCodingPolicies(); + Collection diffErasureCodingPolicies = null; + + if (httpFs instanceof HttpFSFileSystem) { + HttpFSFileSystem httpFS = (HttpFSFileSystem) httpFs; + diffErasureCodingPolicies = httpFS.getAllErasureCodingPolicies(); + } else if (httpFs instanceof WebHdfsFileSystem) { + WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) httpFs; + diffErasureCodingPolicies = webHdfsFileSystem.getAllErasureCodingPolicies(); + } else { + Assert.fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff"); + } + + //Validate erasureCodingPolicyInfos are the same as DistributedFileSystem + assertEquals(dfsAllErasureCodingPolicies.size(), diffErasureCodingPolicies.size()); + assertTrue(dfsAllErasureCodingPolicies.containsAll(diffErasureCodingPolicies)); + } else { + Assert.fail(fs.getClass().getSimpleName() + " is not of type DistributedFileSystem."); + } + } + private void assertHttpFsReportListingWithDfsClient(SnapshotDiffReportListing diffReportListing, SnapshotDiffReportListing dfsDiffReportListing) { Assert.assertEquals(diffReportListing.getCreateList().size(), From 424ada9a342f9c26825153819cdf045c14dbc3ae Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Sun, 11 Jun 2023 18:28:18 +0800 Subject: [PATCH 2/4] fix checkstyle. --- .../java/org/apache/hadoop/fs/http/server/FSOperations.java | 2 -- .../org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 4ce96a28d2295f..f495a85a23e285 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -66,7 +65,6 @@ import java.io.OutputStream; import java.util.Collection; import java.util.EnumSet; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index 413c1697cf445c..c67d18df56a1ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -2124,10 +2124,11 @@ private void testGetAllEEPolicies() throws Exception { FileSystem fs = FileSystem.get(path.toUri(), this.getProxiedFSConf()); if (fs instanceof DistributedFileSystem) { DistributedFileSystem dfs = - (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf()); + (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf()); FileSystem httpFs = this.getHttpFSFileSystem(); - Collection dfsAllErasureCodingPolicies = dfs.getAllErasureCodingPolicies(); + Collection dfsAllErasureCodingPolicies = + dfs.getAllErasureCodingPolicies(); Collection diffErasureCodingPolicies = null; if (httpFs instanceof HttpFSFileSystem) { From 8f29ad16f9793ba83837e7db9b95b2d7f406ddc7 Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Wed, 14 Jun 2023 18:46:21 +0800 Subject: [PATCH 3/4] fix checkstyle. --- .../org/apache/hadoop/fs/http/client/HttpFSFileSystem.java | 4 ++-- .../org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 2260f923bca5ab..e5c59117738bc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -1778,9 +1778,9 @@ public FsStatus getStatus(final Path path) throws IOException { public Collection getAllErasureCodingPolicies() throws IOException { Map params = new HashMap<>(); params.put(OP_PARAM, Operation.GETECPOLICIES.toString()); + Path path = new Path(getUri().toString(), "/"); HttpURLConnection conn = - getConnection(Operation.GETECPOLICIES.getMethod(), params, new Path(getUri() - .toString(), "/"), true); + getConnection(Operation.GETECPOLICIES.getMethod(), params, path, true); HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); return JsonUtilClient.getAllErasureCodingPolicies(json); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index c67d18df56a1ea..8fc728a619a875 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -1371,7 +1371,6 @@ private void operation(Operation op) throws Exception { testGetAllEEPolicies(); break; } - } @Parameterized.Parameters From 721da6656fec58485c56e7a6166153631bf90dc7 Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Thu, 15 Jun 2023 22:12:49 +0800 Subject: [PATCH 4/4] fix checkstyle. --- .../org/apache/hadoop/fs/http/client/HttpFSFileSystem.java | 2 +- .../org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index e5c59117738bc7..557f3352075d81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -1780,7 +1780,7 @@ public Collection getAllErasureCodingPolicies() throws params.put(OP_PARAM, Operation.GETECPOLICIES.toString()); Path path = new Path(getUri().toString(), "/"); HttpURLConnection conn = - getConnection(Operation.GETECPOLICIES.getMethod(), params, path, true); + getConnection(Operation.GETECPOLICIES.getMethod(), params, path, false); HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); return JsonUtilClient.getAllErasureCodingPolicies(json); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index 8fc728a619a875..da3faf1066192f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -2137,7 +2137,8 @@ private void testGetAllEEPolicies() throws Exception { WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) httpFs; diffErasureCodingPolicies = webHdfsFileSystem.getAllErasureCodingPolicies(); } else { - Assert.fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff"); + Assert.fail(fs.getClass().getSimpleName() + + " is not of type HttpFSFileSystem or WebHdfsFileSystem"); } //Validate erasureCodingPolicyInfos are the same as DistributedFileSystem