diff --git a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java index fddae97125f5..231bc57087aa 100644 --- a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java +++ b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java @@ -382,6 +382,21 @@ default Table registerTable( throw new UnsupportedOperationException("Registering tables with overwrite is not supported"); } + /** + * Unregister a table without deleting its data or metadata files. + * + *

The returned table is fixed at the last metadata file registered with the catalog and cannot + * be modified. Its metadata file location can be used to {@link #registerTable(TableIdentifier, + * String) register} the table again. + * + * @param identifier a table identifier + * @return a read-only table fixed at the metadata current when it was unregistered + * @throws NoSuchTableException if the table does not exist + */ + default Table unregisterTable(TableIdentifier identifier) { + throw new UnsupportedOperationException("Unregistering tables is not supported"); + } + /** * Instantiate a builder to either create a table or start a create/replace transaction. * diff --git a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java index a3e269613591..a9132e1d6136 100644 --- a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java +++ b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java @@ -196,6 +196,21 @@ default Table registerTable( throw new UnsupportedOperationException("Registering tables with overwrite is not supported"); } + /** + * Unregister a table without deleting its data or metadata files. + * + *

The returned table is fixed at the last metadata file registered with the catalog and cannot + * be modified. + * + * @param context session context + * @param ident a table identifier + * @return a read-only table fixed at the metadata current when it was unregistered + * @throws NoSuchTableException if the table does not exist + */ + default Table unregisterTable(SessionContext context, TableIdentifier ident) { + throw new UnsupportedOperationException("Unregistering tables is not supported"); + } + /** * Check whether table exists. * diff --git a/core/src/main/java/org/apache/iceberg/CachingCatalog.java b/core/src/main/java/org/apache/iceberg/CachingCatalog.java index 700cfc6cb582..4ee6d61c0910 100644 --- a/core/src/main/java/org/apache/iceberg/CachingCatalog.java +++ b/core/src/main/java/org/apache/iceberg/CachingCatalog.java @@ -204,6 +204,13 @@ public Table registerTable( return table; } + @Override + public Table unregisterTable(TableIdentifier identifier) { + Table table = catalog.unregisterTable(identifier); + invalidateTable(identifier); + return table; + } + private Iterable metadataTableIdentifiers(TableIdentifier ident) { ImmutableList.Builder builder = ImmutableList.builder(); diff --git a/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java b/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java index 69e0990cbf3b..bf8bce4e4da2 100644 --- a/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/catalog/BaseSessionCatalog.java @@ -95,6 +95,11 @@ public Table registerTable( return BaseSessionCatalog.this.registerTable(context, ident, metadataFileLocation, overwrite); } + @Override + public Table unregisterTable(TableIdentifier ident) { + return BaseSessionCatalog.this.unregisterTable(context, ident); + } + @Override public boolean tableExists(TableIdentifier ident) { return BaseSessionCatalog.this.tableExists(context, ident); diff --git a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java index 80297310e82f..18b205ec249a 100644 --- a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java +++ b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java @@ -29,8 +29,11 @@ import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; import org.apache.iceberg.BaseMetastoreTableOperations; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.Namespace; @@ -125,6 +128,27 @@ private String defaultNamespaceLocation(Namespace namespace) { } } + @Override + public Table unregisterTable(TableIdentifier tableIdentifier) { + TableOperations ops = newTableOps(tableIdentifier); + TableMetadata metadata; + + synchronized (this) { + metadata = ops.current(); + if (metadata == null) { + throw new NoSuchTableException("Table does not exist: %s", tableIdentifier); + } + + if (tables.remove(tableIdentifier) == null) { + throw new NoSuchTableException("Table does not exist: %s", tableIdentifier); + } + } + + StaticTableOperations staticOps = + new StaticTableOperations(metadata, ops.io(), ops.locationProvider()); + return new BaseTable(staticOps, tableIdentifier.name(), metricsReporter()); + } + @Override public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) { TableOperations ops = newTableOps(tableIdentifier); diff --git a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java index 2d24e5598ac7..24d7c56f857f 100644 --- a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java +++ b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java @@ -18,6 +18,15 @@ */ package org.apache.iceberg.jdbc; +import static org.apache.iceberg.TableProperties.COMMIT_MAX_RETRY_WAIT_MS; +import static org.apache.iceberg.TableProperties.COMMIT_MAX_RETRY_WAIT_MS_DEFAULT; +import static org.apache.iceberg.TableProperties.COMMIT_MIN_RETRY_WAIT_MS; +import static org.apache.iceberg.TableProperties.COMMIT_MIN_RETRY_WAIT_MS_DEFAULT; +import static org.apache.iceberg.TableProperties.COMMIT_NUM_RETRIES; +import static org.apache.iceberg.TableProperties.COMMIT_NUM_RETRIES_DEFAULT; +import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS; +import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT; + import java.io.IOException; import java.io.UncheckedIOException; import java.sql.Connection; @@ -34,14 +43,18 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.Schema; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; import org.apache.iceberg.Transaction; @@ -49,6 +62,7 @@ import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; @@ -66,6 +80,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.LocationUtil; import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.Tasks; import org.apache.iceberg.view.BaseMetastoreViewCatalog; import org.apache.iceberg.view.ViewMetadata; import org.apache.iceberg.view.ViewOperations; @@ -297,6 +312,60 @@ protected String defaultWarehouseLocation(TableIdentifier table) { return SLASH.join(defaultNamespaceLocation(table.namespace()), tableLocation); } + @Override + public Table unregisterTable(TableIdentifier identifier) { + Preconditions.checkArgument( + identifier != null && isValidIdentifier(identifier), "Invalid identifier: %s", identifier); + + TableMetadata initialMetadata = newTableOps(identifier).current(); + if (initialMetadata == null) { + throw new NoSuchTableException("Table does not exist: %s", identifier); + } + + AtomicReference unregistered = new AtomicReference<>(); + Tasks.foreach(identifier) + .retry(initialMetadata.propertyAsInt(COMMIT_NUM_RETRIES, COMMIT_NUM_RETRIES_DEFAULT)) + .exponentialBackoff( + initialMetadata.propertyAsInt( + COMMIT_MIN_RETRY_WAIT_MS, COMMIT_MIN_RETRY_WAIT_MS_DEFAULT), + initialMetadata.propertyAsInt( + COMMIT_MAX_RETRY_WAIT_MS, COMMIT_MAX_RETRY_WAIT_MS_DEFAULT), + initialMetadata.propertyAsInt( + COMMIT_TOTAL_RETRY_TIME_MS, COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT), + 2.0 /* exponential */) + .onlyRetryOn(CommitFailedException.class) + .run(tableIdentifier -> unregistered.set(unregisterTableOnce(tableIdentifier))); + return unregistered.get(); + } + + private Table unregisterTableOnce(TableIdentifier identifier) { + TableOperations ops = newTableOps(identifier); + TableMetadata metadata = ops.current(); + if (metadata == null) { + throw new NoSuchTableException("Table does not exist: %s", identifier); + } + + if (dropTableIfMetadataMatches(identifier, metadata.metadataFileLocation()) == 0) { + throw new CommitFailedException( + "Cannot unregister table %s: metadata location changed concurrently", identifier); + } + + StaticTableOperations staticOps = + new StaticTableOperations(metadata, ops.io(), ops.locationProvider()); + return new BaseTable(staticOps, identifier.name(), metricsReporter()); + } + + int dropTableIfMetadataMatches(TableIdentifier identifier, String metadataLocation) { + return execute( + (schemaVersion == JdbcUtil.SchemaVersion.V1) + ? JdbcUtil.V1_UNREGISTER_TABLE_SQL + : JdbcUtil.V0_UNREGISTER_TABLE_SQL, + catalogName, + JdbcUtil.namespaceToString(identifier.namespace()), + identifier.name(), + metadataLocation); + } + @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { TableOperations ops = newTableOps(identifier); diff --git a/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java b/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java index 259bd7812555..528eccdf16c9 100644 --- a/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java +++ b/core/src/main/java/org/apache/iceberg/jdbc/JdbcUtil.java @@ -321,6 +321,8 @@ enum SchemaVersion { + " OR " + RECORD_TYPE + " IS NULL)"; + static final String V1_UNREGISTER_TABLE_SQL = + V1_DROP_TABLE_SQL + " AND " + BaseMetastoreTableOperations.METADATA_LOCATION_PROP + " = ?"; static final String V0_DROP_TABLE_SQL = "DELETE FROM " + CATALOG_TABLE_VIEW_NAME @@ -331,6 +333,8 @@ enum SchemaVersion { + " = ? AND " + TABLE_NAME + " = ?"; + static final String V0_UNREGISTER_TABLE_SQL = + V0_DROP_TABLE_SQL + " AND " + BaseMetastoreTableOperations.METADATA_LOCATION_PROP + " = ?"; private static final String GET_NAMESPACE_SQL = "SELECT " + TABLE_NAMESPACE diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index 13089fc07ded..8d738874cd98 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -51,6 +51,7 @@ import org.apache.iceberg.BaseTransaction; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.IncrementalAppendScan; +import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataUpdate.UpgradeFormatVersion; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RetryableValidationException; @@ -95,11 +96,13 @@ import org.apache.iceberg.rest.responses.FetchScanTasksResponse; import org.apache.iceberg.rest.responses.GetNamespaceResponse; import org.apache.iceberg.rest.responses.ImmutableLoadViewResponse; +import org.apache.iceberg.rest.responses.ImmutableUnregisterTableResponse; import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.PlanTableScanResponse; +import org.apache.iceberg.rest.responses.UnregisterTableResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; @@ -488,6 +491,24 @@ public static void dropTable(Catalog catalog, TableIdentifier ident) { } } + public static UnregisterTableResponse unregisterTable(Catalog catalog, TableIdentifier ident) { + if (MetadataTableType.from(ident.name()) != null) { + throw new NoSuchTableException("Table does not exist: %s", ident); + } + + Table table = catalog.unregisterTable(ident); + if (!(table instanceof BaseTable)) { + throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); + } + + TableMetadata metadata = ((BaseTable) table).operations().current(); + + return ImmutableUnregisterTableResponse.builder() + .metadataLocation(metadata.metadataFileLocation()) + .metadata(metadata) + .build(); + } + public static void purgeTable(Catalog catalog, TableIdentifier ident) { boolean dropped = catalog.dropTable(ident, true); if (!dropped) { diff --git a/core/src/main/java/org/apache/iceberg/rest/Endpoint.java b/core/src/main/java/org/apache/iceberg/rest/Endpoint.java index d56a14d18954..ef071c228ea4 100644 --- a/core/src/main/java/org/apache/iceberg/rest/Endpoint.java +++ b/core/src/main/java/org/apache/iceberg/rest/Endpoint.java @@ -62,6 +62,8 @@ public class Endpoint { Endpoint.create("POST", ResourcePaths.V1_TABLE_RENAME); public static final Endpoint V1_REGISTER_TABLE = Endpoint.create("POST", ResourcePaths.V1_TABLE_REGISTER); + public static final Endpoint V1_UNREGISTER_TABLE = + Endpoint.create("POST", ResourcePaths.V1_TABLE_UNREGISTER); public static final Endpoint V1_REPORT_METRICS = Endpoint.create("POST", ResourcePaths.V1_TABLE_METRICS); public static final Endpoint V1_TABLE_CREDENTIALS = diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java index 02bceab4d2a0..dca43aa217cd 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTCatalog.java @@ -252,6 +252,11 @@ public Table registerTable( return delegate.registerTable(ident, metadataFileLocation, overwrite); } + @Override + public Table unregisterTable(TableIdentifier ident) { + return delegate.unregisterTable(ident); + } + @Override public void createNamespace(Namespace ns, Map props) { nsDelegate.createNamespace(ns, props); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java b/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java index c97f4aa2e522..ad3aa936c5b9 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSerializers.java @@ -80,6 +80,7 @@ import org.apache.iceberg.rest.responses.ImmutableLoadCredentialsResponse; import org.apache.iceberg.rest.responses.ImmutableLoadViewResponse; import org.apache.iceberg.rest.responses.ImmutableRemoteSignResponse; +import org.apache.iceberg.rest.responses.ImmutableUnregisterTableResponse; import org.apache.iceberg.rest.responses.LoadCredentialsResponse; import org.apache.iceberg.rest.responses.LoadCredentialsResponseParser; import org.apache.iceberg.rest.responses.LoadTableResponse; @@ -91,6 +92,8 @@ import org.apache.iceberg.rest.responses.PlanTableScanResponseParser; import org.apache.iceberg.rest.responses.RemoteSignResponse; import org.apache.iceberg.rest.responses.RemoteSignResponseParser; +import org.apache.iceberg.rest.responses.UnregisterTableResponse; +import org.apache.iceberg.rest.responses.UnregisterTableResponseParser; import org.apache.iceberg.util.JsonUtil; public class RESTSerializers { @@ -153,6 +156,12 @@ public static void registerAll(ObjectMapper mapper) { .addDeserializer(ConfigResponse.class, new ConfigResponseDeserializer<>()) .addSerializer(LoadTableResponse.class, new LoadTableResponseSerializer<>()) .addDeserializer(LoadTableResponse.class, new LoadTableResponseDeserializer<>()) + .addSerializer(UnregisterTableResponse.class, new UnregisterTableResponseSerializer<>()) + .addSerializer( + ImmutableUnregisterTableResponse.class, new UnregisterTableResponseSerializer<>()) + .addDeserializer(UnregisterTableResponse.class, new UnregisterTableResponseDeserializer<>()) + .addDeserializer( + ImmutableUnregisterTableResponse.class, new UnregisterTableResponseDeserializer<>()) .addSerializer(PlanTableScanRequest.class, new PlanTableScanRequestSerializer<>()) .addDeserializer(PlanTableScanRequest.class, new PlanTableScanRequestDeserializer<>()) .addSerializer(FetchScanTasksRequest.class, new FetchScanTasksRequestSerializer<>()) @@ -532,6 +541,24 @@ public void serialize(T request, JsonGenerator gen, SerializerProvider serialize } } + static class UnregisterTableResponseSerializer + extends JsonSerializer { + @Override + public void serialize(T response, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + UnregisterTableResponseParser.toJson(response, gen); + } + } + + static class UnregisterTableResponseDeserializer + extends JsonDeserializer { + @Override + public T deserialize(JsonParser p, DeserializationContext context) throws IOException { + JsonNode jsonNode = p.getCodec().readTree(p); + return (T) UnregisterTableResponseParser.fromJson(jsonNode); + } + } + static class LoadTableResponseDeserializer extends JsonDeserializer { @Override diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 0b30ef9d88ef..03302ce687e2 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -43,6 +43,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; @@ -100,6 +101,7 @@ import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; import org.apache.iceberg.rest.responses.LoadViewResponse; +import org.apache.iceberg.rest.responses.UnregisterTableResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.EnvironmentUtil; import org.apache.iceberg.util.PropertyUtil; @@ -805,6 +807,40 @@ public Table registerTable( response.labels()); } + /** + * Unregister a table from the catalog without removing its data or metadata files. + * + *

This is the opposite of {@link #registerTable(SessionContext, TableIdentifier, String)}. On + * success, the table no longer exists in the catalog and the returned table is fixed at the last + * metadata registered with the catalog. + * + * @param context session context + * @param identifier a table identifier + * @return a read-only table fixed at the metadata current when it was unregistered + */ + @Override + public Table unregisterTable(SessionContext context, TableIdentifier identifier) { + Endpoint.check(endpoints, Endpoint.V1_UNREGISTER_TABLE); + checkIdentifierIsValid(identifier); + + try { + AuthSession contextualSession = authManager.contextualSession(context, catalogAuth); + UnregisterTableResponse response = + client + .withAuthSession(contextualSession) + .post( + paths.unregister(identifier), + null, + UnregisterTableResponse.class, + mutationHeaders, + ErrorHandlers.tableErrorHandler()); + StaticTableOperations ops = new StaticTableOperations(response.metadata(), io); + return new BaseTable(ops, identifier.name()); + } finally { + invalidateTable(context, identifier); + } + } + @Override public void createNamespace( SessionContext context, Namespace namespace, Map metadata) { diff --git a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java index 5757a6d17624..970e29e0eb8b 100644 --- a/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java +++ b/core/src/main/java/org/apache/iceberg/rest/ResourcePaths.java @@ -38,6 +38,8 @@ public class ResourcePaths { public static final String V1_TABLE_REMOTE_SIGN = "/v1/{prefix}/namespaces/{namespace}/tables/{table}/sign"; public static final String V1_TABLE_REGISTER = "/v1/{prefix}/namespaces/{namespace}/register"; + public static final String V1_TABLE_UNREGISTER = + "/v1/{prefix}/namespaces/{namespace}/tables/{table}/unregister"; public static final String V1_TABLE_METRICS = "/v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics"; public static final String V1_TABLE_RENAME = "/v1/{prefix}/tables/rename"; @@ -108,6 +110,17 @@ public String register(Namespace ns) { return SLASH.join("v1", prefix, "namespaces", pathEncode(ns), "register"); } + public String unregister(TableIdentifier ident) { + return SLASH.join( + "v1", + prefix, + "namespaces", + pathEncode(ident.namespace()), + "tables", + RESTUtil.encodePathSegment(ident.name()), + "unregister"); + } + public String rename() { return SLASH.join("v1", prefix, "tables", "rename"); } diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/UnregisterTableResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/UnregisterTableResponse.java new file mode 100644 index 000000000000..a555a6996275 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/responses/UnregisterTableResponse.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.responses; + +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.rest.RESTResponse; +import org.immutables.value.Value; + +/** + * Represents the response when a table is successfully unregistered from a catalog. + * + *

The response carries the table's last metadata location and the corresponding table metadata + * so that the underlying (still present) files can be registered in another catalog. + */ +@Value.Immutable +public interface UnregisterTableResponse extends RESTResponse { + String metadataLocation(); + + TableMetadata metadata(); + + @Override + default void validate() { + Preconditions.checkArgument(metadataLocation() != null, "Invalid metadata location: null"); + Preconditions.checkArgument(metadata() != null, "Invalid metadata: null"); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/UnregisterTableResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/UnregisterTableResponseParser.java new file mode 100644 index 000000000000..95479fa018c1 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/responses/UnregisterTableResponseParser.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.responses; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; + +public class UnregisterTableResponseParser { + + private static final String METADATA_LOCATION = "metadata-location"; + private static final String METADATA = "metadata"; + + private UnregisterTableResponseParser() {} + + public static String toJson(UnregisterTableResponse response) { + return toJson(response, false); + } + + public static String toJson(UnregisterTableResponse response, boolean pretty) { + return JsonUtil.generate(gen -> toJson(response, gen), pretty); + } + + public static void toJson(UnregisterTableResponse response, JsonGenerator gen) + throws IOException { + Preconditions.checkArgument(null != response, "Invalid unregister table response: null"); + + gen.writeStartObject(); + + gen.writeStringField(METADATA_LOCATION, response.metadataLocation()); + + gen.writeFieldName(METADATA); + TableMetadataParser.toJson(response.metadata(), gen); + + gen.writeEndObject(); + } + + public static UnregisterTableResponse fromJson(String json) { + return JsonUtil.parse(json, UnregisterTableResponseParser::fromJson); + } + + public static UnregisterTableResponse fromJson(JsonNode json) { + Preconditions.checkArgument( + null != json, "Cannot parse unregister table response from null object"); + + String metadataLocation = JsonUtil.getString(METADATA_LOCATION, json); + TableMetadata metadata = TableMetadataParser.fromJson(JsonUtil.get(METADATA, json)); + + if (null == metadata.metadataFileLocation()) { + metadata = TableMetadata.buildFrom(metadata).withMetadataLocation(metadataLocation).build(); + } + + return ImmutableUnregisterTableResponse.builder() + .metadataLocation(metadataLocation) + .metadata(metadata) + .build(); + } +} diff --git a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java index ff40527ebedd..9e6ad888c601 100644 --- a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java @@ -208,6 +208,10 @@ protected boolean supportsVariant() { return false; } + protected boolean supportsUnregister() { + return false; + } + protected String baseTableLocation(TableIdentifier identifier) { return BASE_TABLE_LOCATION + "/" + identifier.namespace() + "/" + identifier.name(); } @@ -3454,6 +3458,47 @@ public void testRegisterExistingTable() { assertThat(catalog.dropTable(identifier)).isTrue(); } + @Test + public void unregisterTable() { + assumeThat(supportsUnregister()).isTrue(); + + C catalog = catalog(); + if (requiresNamespaceCreate()) { + catalog.createNamespace(TABLE.namespace()); + } + + Table original = catalog.buildTable(TABLE, SCHEMA).withPartitionSpec(SPEC).create(); + original.newFastAppend().appendFile(FILE_A).commit(); + String metadataLocation = TableUtil.metadataFileLocation(original); + + Table unregistered = catalog.unregisterTable(TABLE); + + assertThat(unregistered.name()).isEqualTo(TABLE.name()); + assertThat(TableUtil.metadataFileLocation(unregistered)).isEqualTo(metadataLocation); + assertThat(unregistered.currentSnapshot()).isEqualTo(original.currentSnapshot()); + assertThat(catalog.tableExists(TABLE)).isFalse(); + assertThat(unregistered.io().newInputFile(metadataLocation).exists()).isTrue(); + assertThatThrownBy(() -> unregistered.updateProperties().set("unregistered", "true").commit()) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Cannot modify a static table"); + + TableIdentifier registeredIdentifier = + TableIdentifier.of(TABLE.namespace(), "registered-after-unregister"); + Table registered = catalog.registerTable(registeredIdentifier, metadataLocation); + assertThat(registered.currentSnapshot()).isEqualTo(original.currentSnapshot()); + assertFiles(registered, FILE_A); + assertThat(catalog.dropTable(registeredIdentifier)).isTrue(); + } + + @Test + public void unregisterMissingTable() { + assumeThat(supportsUnregister()).isTrue(); + + assertThatThrownBy(() -> catalog().unregisterTable(TABLE)) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining("Table does not exist"); + } + @Test public void testCatalogWithCustomMetricsReporter() throws IOException { C catalogWithCustomReporter = diff --git a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java index 827450d4a398..9cd361aa2638 100644 --- a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java +++ b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryCatalog.java @@ -87,6 +87,11 @@ protected boolean supportsNestedNamespaces() { return true; } + @Override + protected boolean supportsUnregister() { + return true; + } + @Test @Override public void testLoadTableWithMissingMetadataFile(@TempDir Path tempDir) throws IOException { diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java index ce22c8089bc3..225e6da20a9e 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java @@ -112,6 +112,11 @@ protected boolean supportsEmptyNamespace() { return true; } + @Override + protected boolean supportsUnregister() { + return true; + } + @Override protected boolean supportsNamesWithDot() { // namespaces with a dot are not supported diff --git a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java index b99d7ffb63f4..e06f07d445b5 100644 --- a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java +++ b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java @@ -391,6 +391,16 @@ public T handleRequest( }); } + case UNREGISTER_TABLE: + { + return CatalogHandlers.withIdempotency( + httpRequest, + () -> + castResponse( + responseType, + CatalogHandlers.unregisterTable(catalog, tableIdentFromPathVars(vars)))); + } + case UPDATE_TABLE: { return CatalogHandlers.withIdempotency( diff --git a/core/src/test/java/org/apache/iceberg/rest/Route.java b/core/src/test/java/org/apache/iceberg/rest/Route.java index 8680915bff64..4a51fc256c98 100644 --- a/core/src/test/java/org/apache/iceberg/rest/Route.java +++ b/core/src/test/java/org/apache/iceberg/rest/Route.java @@ -45,6 +45,7 @@ import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.iceberg.rest.responses.PlanTableScanResponse; +import org.apache.iceberg.rest.responses.UnregisterTableResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.Pair; @@ -82,6 +83,11 @@ enum Route { ResourcePaths.V1_TABLE_REGISTER, RegisterTableRequest.class, LoadTableResponse.class), + UNREGISTER_TABLE( + HTTPRequest.HTTPMethod.POST, + ResourcePaths.V1_TABLE_UNREGISTER, + null, + UnregisterTableResponse.class), UPDATE_TABLE( HTTPRequest.HTTPMethod.POST, ResourcePaths.V1_TABLE, diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestUnregisterTableResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestUnregisterTableResponseParser.java new file mode 100644 index 000000000000..997be87e1de2 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestUnregisterTableResponseParser.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.responses; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +public class TestUnregisterTableResponseParser { + + @Test + public void nullAndEmptyCheck() { + assertThatThrownBy(() -> UnregisterTableResponseParser.toJson(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid unregister table response: null"); + + assertThatThrownBy(() -> UnregisterTableResponseParser.fromJson((JsonNode) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse unregister table response from null object"); + + assertThatThrownBy(() -> UnregisterTableResponseParser.fromJson("{}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing string: metadata-location"); + } + + @Test + public void missingFields() { + assertThatThrownBy( + () -> + UnregisterTableResponseParser.fromJson( + "{\"metadata-location\": \"custom-location\"}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing field: metadata"); + } + + @Test + public void roundTripSerde() { + String uuid = "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b"; + TableMetadata metadata = + TableMetadata.buildFromEmpty(1) + .assignUUID(uuid) + .setLocation("location") + .setCurrentSchema( + new Schema(Types.NestedField.required(1, "x", Types.LongType.get())), 1) + .addPartitionSpec(PartitionSpec.unpartitioned()) + .addSortOrder(SortOrder.unsorted()) + .discardChanges() + .withMetadataLocation("metadataTestLocation") + .build(); + + UnregisterTableResponse response = + ImmutableUnregisterTableResponse.builder() + .metadataLocation("metadataTestLocation") + .metadata(metadata) + .build(); + + String expectedJson = + String.format( + "{\"metadata-location\":\"metadataTestLocation\",\"metadata\":%s}", + TableMetadataParser.toJson(metadata)); + String actualJson = UnregisterTableResponseParser.toJson(response); + assertThat(actualJson).isEqualTo(expectedJson); + + UnregisterTableResponse parsed = UnregisterTableResponseParser.fromJson(actualJson); + assertThat(parsed.metadata().metadataFileLocation()).isEqualTo("metadataTestLocation"); + assertThat(UnregisterTableResponseParser.toJson(parsed)).isEqualTo(expectedJson); + } +} diff --git a/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java index 611a6a20da05..83343c4e42d0 100644 --- a/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java +++ b/open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java @@ -19,9 +19,13 @@ package org.apache.iceberg.rest; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Map; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.util.PropertyUtil; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -115,6 +119,61 @@ protected boolean supportsVariant() { restCatalog.properties(), RESTCompatibilityKitSuite.RCK_SUPPORTS_VARIANT, false); } + @Override + protected boolean supportsUnregister() { + return true; + } + + @Test + public void testUnregisterTable() { + if (requiresNamespaceCreate()) { + restCatalog.createNamespace(TABLE.namespace()); + } + + Table original = + restCatalog + .buildTable(TABLE, SCHEMA) + .withPartitionSpec(SPEC) + .withSortOrder(WRITE_ORDER) + .create(); + original.newFastAppend().appendFile(FILE_A).commit(); + original.newFastAppend().appendFile(FILE_B).commit(); + + Table unregistered = restCatalog.unregisterTable(TABLE); + + assertThat(unregistered.currentSnapshot()) + .as("Current snapshot must match the unregistered table") + .isEqualTo(original.currentSnapshot()); + assertFiles(unregistered, FILE_A, FILE_B); + assertThat(restCatalog.tableExists(TABLE)) + .as("Table must not exist after being unregistered") + .isFalse(); + assertThatThrownBy(() -> unregistered.updateProperties().set("unregistered", "true").commit()) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Cannot modify a static table"); + + // the underlying files are left in place, so the table can be registered again + Table registered = + restCatalog.registerTable(TABLE, TableUtil.metadataFileLocation(unregistered)); + assertThat(registered.currentSnapshot()) + .as("Current snapshot must match the unregistered table") + .isEqualTo(original.currentSnapshot()); + assertFiles(registered, FILE_A, FILE_B); + + assertThat(restCatalog.dropTable(TABLE)).isTrue(); + } + + @Test + public void testUnregisterMissingTable() { + if (requiresNamespaceCreate()) { + restCatalog.createNamespace(TABLE.namespace()); + } + + assertThatThrownBy(() -> restCatalog.unregisterTable(TABLE)) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining("Table does not exist"); + } + @Disabled("RESTServerExtension isn’t configurable per test") @Test public void createTableInUniqueLocation() {