From 8593b66a6c86243f8f749f920afcead0d537ca83 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Fri, 28 Aug 2026 13:23:28 +0300 Subject: [PATCH 01/13] HIVE-29834: Iceberg: Answer MIN, MAX and COUNT from the handler's column statistics MIN, MAX and COUNT over a column are facts the stored statistics already state, so a query asking only for them is answered from what a storage handler holds rather than by reading the rows. The statistics of every aggregate in a query are fetched at once, and a partitioned table is answered only from statistics that describe the partitions, the columns and the snapshot the scan asks about - a partition whose statistics do not cover every asked column, or which a live delete of no named partition may have changed, is not answered for. A table's size comes from its storage handler rather than from listing what its location holds, and whether a join can be a sort-merge is decided before its big table is elected, so an election made on the handler's numbers is not undone by one made on the listing's. --- .../mr/hive/HiveIcebergStorageHandler.java | 28 +- .../iceberg/mr/hive/IcebergTableUtil.java | 2 +- .../mr/hive/stats/IcebergColStatsWriter.java | 5 +- .../mr/hive/stats/IcebergStoredStats.java | 56 +- .../mr/hive/TestHiveIcebergStatistics.java | 155 +++- .../hive/test/utils/HiveIcebergTestUtils.java | 19 +- .../queries/positive/iceberg_part_colstats.q | 152 ++++ .../src/test/results/positive/col_stats.q.out | 18 +- .../positive/iceberg_part_colstats.q.out | 827 ++++++++++++++++++ .../hive/ql/metadata/HiveStorageHandler.java | 20 +- .../hive/ql/optimizer/StatsOptimizer.java | 709 ++++++--------- .../hadoop/hive/ql/stats/StatsUtils.java | 19 +- .../test/queries/clientpositive/stats_part.q | 3 + .../clientpositive/llap/stats_part.q.out | 27 + .../hadoop/hive/common/StatsSetupConst.java | 15 +- 15 files changed, 1565 insertions(+), 490 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 5d19622f2463..f7b72115e7ac 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -723,6 +723,17 @@ public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsT return HiveMetaHook.ICEBERG.equals(getStatsSource()); } + @Override + public boolean areColumnStatsUptoDate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, List colNames) { + if (canSetColStatistics(hmsTable)) { + return IcebergStoredStats.colStatsAccurate(hmsTable, colNames, conf); + } + // the metastore holds them, and its single row describes the current table: a scan of a + // branch, a tag, a point in time or a metadata table is not described by it + return hmsTable.getQualifier().isEmpty() && + StatsSetupConst.areColumnStatsUptoDate(hmsTable.getParameters(), colNames); + } + @Override public boolean setColStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Iterator colStats) { @@ -812,12 +823,17 @@ private AggrStats aggrColStats(org.apache.hadoop.hive.ql.metadata.Table hmsTable return new AggrStats(aggregated, partNames.size()); } + Set columns = Sets.newHashSet(colNames); Map> statsByPart = IcebergColStatsReader.readPart(table, statsFile, partition -> partitions.contains(partition) && upToDate.test(partition), - Sets.newHashSet(colNames), conf); + // an ask as wide as the schema narrows nothing, so it reads each blob whole + columns.size() == table.schema().columns().size() ? null : columns, conf); List partStats = Lists.newArrayList(); statsByPart.forEach((partition, statsObjs) -> { + // a whole-blob read decodes every stored entry, and a carried blob may hold entries under + // names the schema no longer has: only the asked columns may count toward the ask + statsObjs.removeIf(obj -> !columns.contains(obj.getColName())); // the metastore counts a partition as found only when it has every column asked about if (statsObjs.size() == colNames.size()) { ColumnStatisticsDesc statsDesc = @@ -841,7 +857,7 @@ public Long getRowCount(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { if (hmsTable.getMetaTable() != null) { return null; } - return getStatsSource().equals(HiveMetaHook.ICEBERG) || hmsTable.getSnapshotRef() != null ? + return getStatsSource().equals(HiveMetaHook.ICEBERG) || !hmsTable.getQualifier().isEmpty() ? snapshotRowCount(hmsTable) : metastoreRowCount(hmsTable); } @@ -879,6 +895,14 @@ public Map getRowCount(org.apache.hadoop.hive.ql.metadata.Table hm // does not select return Map.of(); } + // an equality delete under an unpartitioned spec applies to every data file, so no partition's + // entry accounts for it. By spec, not the void name: a dropped field leaves a void transform + boolean globalDeletes = getOrCachePartitionStats(table, snapshot).values().stream() + .anyMatch(stats -> table.specs().get(stats.specId()).isUnpartitioned() && + stats.equalityDeleteRecordCount() > 0); + if (globalDeletes) { + return Map.of(); + } Map rowCounts = Maps.newHashMapWithExpectedSize(partNames.size()); collectPartitionStatsFor(table, snapshot, partNames, (partName, stats) -> { if (stats.equalityDeleteRecordCount() == 0 && stats.positionDeleteRecordCount() == 0) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index e8bdf1bf8d93..666aeda3a5ac 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -213,7 +213,7 @@ static Table getTable(Configuration configuration, Properties properties) { return getTable(configuration, properties, false); } - static Snapshot getTableSnapshot(Table table, org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + public static Snapshot getTableSnapshot(Table table, org.apache.hadoop.hive.ql.metadata.Table hmsTable) { long snapshotId = -1; if (hmsTable.getAsOfTimestamp() != null) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java index 1adb8c41a428..cbac97e2a0fc 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java @@ -58,6 +58,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Writes the column statistics an ANALYZE, or a write told to compute them, produced as the * table's statistics file, per the policy the write's facts resolve to: replacing the stored file, @@ -265,8 +266,8 @@ private static void carryForward(Table tbl, Snapshot snapshot, PuffinWriter writ // was computed stands on its own return; } - Predicate upToDate = IcebergStoredStats.upToDateColStats(tbl, snapshot, statsOldSrc, conf, false); - + Predicate upToDate = + IcebergStoredStats.upToDateColStats(tbl, snapshot, statsOldSrc, conf, false); try (PuffinReader reader = Puffin.read(tbl.io().newInputFile(statsOldSrc.path())) .withFileSize(statsOldSrc.fileSizeInBytes()) .withFooterSize(statsOldSrc.fileFooterSizeInBytes()) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java index 252bdf4485c1..3e1465602407 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java @@ -21,12 +21,15 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.hadoop.util.Sets; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataOperations; @@ -39,6 +42,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.mr.hive.IcebergTableUtil; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.types.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +62,7 @@ public final class IcebergStoredStats { private static final Logger LOG = LoggerFactory.getLogger(IcebergStoredStats.class); + private static final String STATED_FIELD_IDS_KEY = "statedFieldIds.%s.%d.%b"; private static final String CHANGED_PARTITIONS_KEY = "changedPartitions.%s.%d.%d.%d"; /** * What changed between two snapshots is settled the moment the later one commits, so the answer @@ -151,6 +156,49 @@ private static boolean holdsHiveColStats(StatisticsFile stats, boolean partition IcebergColStatsWriter.LEGACY_COL_STATS_BLOB.equals(metadata.type())); } + /** + * Whether the stored column statistics answer for the column: they still describe the snapshot + * the table names, and their file holds an entry for it. The footer names the measured columns - + * a table-level file on each blob, a partition-level file on its first, so a column the schema + * gained since the write, which moved no snapshot, is refused either way. + */ + public static boolean colStatsAccurate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, + List colNames, Configuration conf) { + Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); + Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable); + if (snapshot == null) { + return false; + } + Set stated = statedFieldIds(table, snapshot, conf); + return colNames.stream().allMatch(colName -> { + Types.NestedField field = table.schema().caseInsensitiveFindField(colName); + return field != null && stated.contains(field.fieldId()); + }); + } + + /** + * The fields the stored statistics state for the snapshot. Which file answers and what it names + * is the same question for every column, and a partition-level file names them over one blob per + * partition, so it is asked once for the query rather than once per column asked about. + */ + private static Set statedFieldIds(Table table, Snapshot snapshot, Configuration conf) { + boolean partitionLevel = IcebergTableUtil.isPartitionStats(table, conf); + String cacheKey = STATED_FIELD_IDS_KEY.formatted(table.name(), snapshot.snapshotId(), partitionLevel); + Optional cached = SessionStateUtil.getResource(conf, cacheKey); + if (cached.isPresent()) { + @SuppressWarnings("unchecked") + Set hit = (Set) cached.get(); + return hit; + } + StatisticsFile statsFile = getColStatsFile(table, snapshot.snapshotId(), partitionLevel); + Set fields = statsFile == null ? Set.of() : + statsFile.blobMetadata().stream() + .flatMap(metadata -> metadata.fields().stream()) + .collect(Collectors.toSet()); + SessionStateUtil.addResource(conf, cacheKey, fields); + return fields; + } + /** * Whether the stored column statistics still describe the table: the current snapshot owns them, * or only row-preserving commits (compaction) separate it from the snapshot that does. Derived @@ -171,10 +219,10 @@ public static boolean colStatsAccurate(Table table, Snapshot snapshot, boolean p * changed. The bound is on manifests read, not on snapshots walked. */ static Set partitionsChangedSince(Table table, Snapshot snapshot, long sinceSnapshotId, - Configuration conf, boolean capped) { + Configuration conf, boolean bounded) { // the bound is what a read will wait for; a write settles its file for good, so it walks the // whole way. It keys the answer, since sessions may bound the same walk differently - int snapshotLookback = capped ? + int snapshotLookback = bounded ? HiveConf.getIntVar(conf, ConfVars.HIVE_ICEBERG_STATS_MAX_SNAPSHOT_LOOKBACK) : Integer.MAX_VALUE; // the walk reads manifests, and one query can ask it more than once: a table scanned twice // over, or a DESC that asks column by column @@ -266,9 +314,9 @@ private static Iterable> changedFiles(Table table, Snap * between cannot be traced. */ public static Predicate upToDateColStats(Table table, Snapshot snapshot, - StatisticsFile statsFile, Configuration conf, boolean capped) { + StatisticsFile statsFile, Configuration conf, boolean bounded) { Set changed = - partitionsChangedSince(table, snapshot, statsFile.snapshotId(), conf, capped); + partitionsChangedSince(table, snapshot, statsFile.snapshotId(), conf, bounded); return partition -> changed != null && !changed.contains(partition); } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index 51043af398b6..111662cc4e6d 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -57,6 +57,7 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.DataOperations; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.GenericBlobMetadata; @@ -68,8 +69,10 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.Record; import org.apache.iceberg.hadoop.ConfigProperties; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mr.TestHelper; import org.apache.iceberg.mr.hive.stats.IcebergColStatsReader; import org.apache.iceberg.mr.hive.stats.IcebergColStatsWriter; import org.apache.iceberg.mr.hive.stats.IcebergPartitionStatsReader; @@ -98,6 +101,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; + /** * Tests verifying correct statistics generation behaviour on Iceberg tables triggered by: ANALYZE queries, inserts, * CTAS, etc... @@ -1210,6 +1214,41 @@ public void anotherEnginesBlobBesideHivesLeavesHivesReadableAndKeepsItsVector() read.getFirst().getStatsData().getLongStats().isSetBitVectors()); } + @Test + public void testAGlobalDeleteRefusesEveryPartitionsRowCount() throws Exception { + // an equality delete written under the unpartitioned spec - as a foreign engine writes it - + // applies to the rows of every partition, while the statistics bookkeep it under the + // partition of no value: no partition's own entry accounts for it + assumeParquetHiveCatalogIceberg(); + Assume.assumeTrue("equality deletes need format v2", formatVersion >= 2); + + TableIdentifier identifier = TableIdentifier.of("default", "customers_global_delete"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of(), formatVersion); + Table tbl = testTables.loadTable(identifier); + tbl.updateSpec().addField("last_name").commit(); + shell.executeStatement(testTables.getInsertQuery( + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false)); + + HiveIcebergStorageHandler handler = storageHandler(); + List partNames = partitionNames(handler, hmsTable(identifier)); + Assert.assertEquals("with no delete live, every partition's count is served", + partNames.size(), handler.getRowCount(hmsTable(identifier), partNames).size()); + + tbl.refresh(); + List toDelete = TestHelper.RecordsBuilder + .newInstance(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA).add(0L, "Alice", "Brown").build(); + DeleteFile deleteFile = HiveIcebergTestUtils.createEqualityDeleteFile(tbl, tbl.specs().get(0), + "global-eq-delete", ImmutableList.of("customer_id"), fileFormat, toDelete); + tbl.newRowDelta().addDeletes(deleteFile).commit(); + // a write of Hive's own publishes the partition statistics that carry the delete + shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, 'Eve', 'Green')"); + + Assert.assertTrue("a delete of no partition refuses every partition's count", + handler.getRowCount(hmsTable(identifier), partitionNames(handler, hmsTable(identifier))).isEmpty()); + } + @Test public void testAnalyzePartitionSpecRejected() { assumeParquetHiveCatalogIceberg(); @@ -2019,7 +2058,18 @@ public void testMergeCompletesOnlyTheColumnsTheStoredFileDescribes() { shell.executeStatement("INSERT INTO " + identifier + " VALUES (2, 5)"); - // the file holds no half-truth for v: the increment's entry was not promoted + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + HiveIcebergStorageHandler handler = storageHandler(); + Assert.assertTrue("the analyzed column, completed by the increment, still answers", + handler.areColumnStatsUptoDate(hmsTable, List.of("id"))); + Assert.assertFalse("a column the stored file never described must not answer", + handler.areColumnStatsUptoDate(hmsTable, List.of("v"))); + // asked of several at once it answers for all of them or for none + Assert.assertFalse("one column short of an answer leaves the ask unanswered", + handler.areColumnStatsUptoDate(hmsTable, List.of("id", "v"))); + Assert.assertTrue("and a repeated ask of the answered one still answers", + handler.areColumnStatsUptoDate(hmsTable, List.of("id", "id"))); + // the file itself holds no half-truth for v: the increment's entry was not promoted List stored = readCurrentColStats(identifier).getFirst().getStatsObj(); Assert.assertEquals(List.of("id"), stored.stream().map(ColumnStatisticsObj::getColName).toList()); } @@ -2090,6 +2140,32 @@ public void testAGatherThatStoresNothingLeavesWhatIsStoredAlone() throws Excepti hasColStatsForCurrentSnapshot(identifier)); } + @Test + public void testTimeTravelIsNeverAnsweredFromTheMetastoreRow() { + // the metastore's single row describes the current snapshot; a scan of an older one must + // read its own snapshot whatever the statistics source says + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_time_travel_stats"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint) STORED BY ICEBERG STORED AS PARQUET"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1), (5)"); + long oldSnapshot = testTables.loadTable(identifier).currentSnapshot().snapshotId(); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (7), (9), (11)"); + + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE.varname, "metastore"); + try { + org.apache.hadoop.hive.ql.metadata.Table asOf = hmsTable(identifier); + asOf.setAsOfVersion(String.valueOf(oldSnapshot)); + Assert.assertEquals("the scan reads two rows, however the table now holds five", + Long.valueOf(2), storageHandler().getRowCount(asOf)); + Assert.assertFalse("the metastore's row must not answer for a point in time", + storageHandler().areColumnStatsUptoDate(asOf, List.of("id"))); + } finally { + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE.varname, "iceberg"); + } + } + @Test public void testEmptyWriteWithoutStoredColStatsPersistsNothing() { // the same insert onto a table that carries no statistics: an increment gathered over no rows @@ -2256,39 +2332,30 @@ public void testAggrColStatsCountsOnlyPartitionsCarryingEveryColumnAsked() throw } @Test - public void aWholeTableReadTakesNoPerPartitionFile() throws Exception { - // statistics are served at the granularity the session keeps them at. A file holding - // partitions states them, and what it folds from them states the table only while it holds - // every one - so a whole-table read passes it by rather than answer from part of a table + public void testACarriedEntryOfARenamedColumnCannotAnswerForTheNewName() throws Exception { + // ANALYZE full table -> rename a column, which moves no snapshot -> ANALYZE one partition. + // The other partition's entry is carried under the old name and holds as many columns as + // the ask, so it must be refused by identity, not by count. assumeParquetHiveCatalogIceberg(); - TableIdentifier identifier = TableIdentifier.of("default", "orders_two_granularities"); + TableIdentifier identifier = TableIdentifier.of("default", "orders_renamed_column"); shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); - HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); - shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, val bigint, p string) " + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); - shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 100, 'a'), (7, 7, 'b')"); shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); - Assert.assertFalse("whole-table numbers were stored and nothing has happened since", - storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); - // the same table gathered per partition: the file at the current snapshot holds partitions - HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); - shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'c')"); - shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + List partNames = ImmutableList.of("p=a", "p=b"); + Assert.assertEquals("both partitions carry every column asked about", 2, + storageHandler().getAggrColStatsFor(hmsTable(identifier), ImmutableList.of("id", "val", "p"), partNames) + .getPartsFound()); - HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); - Assert.assertTrue("a whole-table read is not answered from the partitions of a later gather", - storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + shell.executeStatement("ALTER TABLE " + identifier + " CHANGE COLUMN val val2 bigint"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p = 'b') COMPUTE STATISTICS FOR COLUMNS"); - // and the partitions still answer for themselves, at the granularity they were kept at - HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); - AggrStats aggrStats = storageHandler().getAggrColStatsFor(hmsTable(identifier), - ImmutableList.of("id"), ImmutableList.of("p=a", "p=b", "p=c")); - Assert.assertEquals("every partition the ask names", 3, aggrStats.getPartsFound()); - LongColumnStatsData stats = aggrStats.getColStats().getFirst().getStatsData().getLongStats(); - Assert.assertEquals("the least value of every partition", 1L, stats.getLowValue()); - Assert.assertEquals("and the greatest", 9L, stats.getHighValue()); + AggrStats aggrStats = storageHandler().getAggrColStatsFor( + hmsTable(identifier), ImmutableList.of("id", "val2", "p"), partNames); + Assert.assertEquals("the carried entry holds no column of the asked name", 1, aggrStats.getPartsFound()); } @Test @@ -2350,6 +2417,42 @@ public void theFoldDoesNotAnswerForAPartitionItNeverDescribed() throws Exception 3, aggrStats.getPartsFound()); } + @Test + public void aWholeTableReadTakesNoPerPartitionFile() throws Exception { + // statistics are served at the granularity the session keeps them at. A file holding + // partitions states them, and what it folds from them states the table only while it holds + // every one - so a whole-table read passes it by rather than answer from part of a table + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_two_granularities"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + Assert.assertFalse("whole-table numbers were stored and nothing has happened since", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + + // the same table gathered per partition: the file at the current snapshot holds partitions + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (9, 'c')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + Assert.assertTrue("a whole-table read is not answered from the partitions of a later gather", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + + // and the partitions still answer for themselves, at the granularity they were kept at + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + AggrStats aggrStats = storageHandler().getAggrColStatsFor(hmsTable(identifier), + ImmutableList.of("id"), ImmutableList.of("p=a", "p=b", "p=c")); + Assert.assertEquals("every partition the ask names", 3, aggrStats.getPartsFound()); + LongColumnStatsData stats = aggrStats.getColStats().getFirst().getStatsData().getLongStats(); + Assert.assertEquals("the least value of every partition", 1L, stats.getLowValue()); + Assert.assertEquals("and the greatest", 9L, stats.getHighValue()); + } + @Test public void aPartitionScopedGatherWithNothingToCarryStatesNoTable() { // it measured one partition and had no stored file to carry the others from, so the file holds diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/utils/HiveIcebergTestUtils.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/utils/HiveIcebergTestUtils.java index 02dd18b777a4..2e3d21068744 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/utils/HiveIcebergTestUtils.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/test/utils/HiveIcebergTestUtils.java @@ -65,6 +65,7 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.HistoryEntry; import org.apache.iceberg.PartitionKey; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.data.GenericAppenderFactory; @@ -349,18 +350,30 @@ public static void validateDataWithSQL(TestHiveShell shell, String tableName, Li */ public static DeleteFile createEqualityDeleteFile(Table table, String deleteFilePath, List equalityFields, FileFormat fileFormat, List rowsToDelete) throws IOException { + return createEqualityDeleteFile(table, table.spec(), deleteFilePath, equalityFields, fileFormat, rowsToDelete); + } + + /** + * The spec-taking variant writes the delete under any of the table's specs: an older + * unpartitioned spec gives the global delete a foreign engine writes. + */ + public static DeleteFile createEqualityDeleteFile(Table table, PartitionSpec spec, String deleteFilePath, + List equalityFields, FileFormat fileFormat, List rowsToDelete) throws IOException { List equalityFieldIds = equalityFields.stream() .map(id -> table.schema().findField(id).fieldId()) .toList(); Schema eqDeleteRowSchema = table.schema().select(equalityFields.toArray(new String[]{})); - FileAppenderFactory appenderFactory = new GenericAppenderFactory(table.schema(), table.spec(), + FileAppenderFactory appenderFactory = new GenericAppenderFactory(table.schema(), spec, ArrayUtil.toIntArray(equalityFieldIds), eqDeleteRowSchema, null); EncryptedOutputFile outputFile = table.encryption().encrypt(HadoopOutputFile.fromPath( new org.apache.hadoop.fs.Path(table.location(), deleteFilePath), new Configuration())); - PartitionKey part = new PartitionKey(table.spec(), eqDeleteRowSchema); - part.partition(rowsToDelete.get(0)); + PartitionKey part = null; + if (spec.isPartitioned()) { + part = new PartitionKey(spec, eqDeleteRowSchema); + part.partition(rowsToDelete.get(0)); + } EqualityDeleteWriter eqWriter = appenderFactory.newEqDeleteWriter(outputFile, fileFormat, part); try (EqualityDeleteWriter writer = eqWriter) { writer.write(rowsToDelete); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q new file mode 100644 index 000000000000..ccf156218595 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q @@ -0,0 +1,152 @@ +--! qt:replace:/(\s+Statistics\: Num rows\: \d+ Data size\:\s+)\S+(\s+Basic stats\: \S+ Column stats\: \S+)/$1#Masked#$2/ + +-- Column statistics kept per partition answer a query over the partitions it pruned to, and stop +-- answering only for the partitions a later write reached. + +set hive.explain.user=false; +set hive.compute.query.using.stats=true; +set hive.fetch.task.conversion=none; +set hive.iceberg.stats.collect.partlevel=true; + +create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c'); +analyze table ice_part_stats compute statistics for columns; + +-- answered from the statistics of the pruned partition alone +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +-- a write that reaches p=a only +insert into ice_part_stats values (11, 'a'); + +-- a scan spanning the partition the write reached and one it did not has statistics for only part +-- of what it reads, which is what PARTIAL says +explain +select id from ice_part_stats where p in ('a', 'b'); + +-- p=a describes itself no longer, so the query has to read it +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +-- the partitions that write never touched still answer from their statistics +explain +select max(id) from ice_part_stats where p = 'b'; + +select max(id) from ice_part_stats where p = 'b'; + +-- an ANALYZE naming the written partition measures it again, and it answers from statistics once +-- more while the partitions carried across that ANALYZE keep the numbers they were computed with +analyze table ice_part_stats partition (p = 'a') compute statistics for columns; + +explain +select max(id) from ice_part_stats where p = 'a'; + +select max(id) from ice_part_stats where p = 'a'; + +explain +select max(id) from ice_part_stats where p = 'b'; + +select max(id) from ice_part_stats where p = 'b'; + +-- count(col) needs a row count as well as the column's null count, and a handler keeps no +-- partition parameters to read one from: it is asked of the table for the pruned partitions +explain +select count(id) from ice_part_stats where p = 'b'; + +select count(id) from ice_part_stats where p = 'b'; + +-- both partitions now carry fresh statistics, so the span a stale subset could not answer is answered +explain +select max(id) from ice_part_stats where p in ('a', 'b'); + +select max(id) from ice_part_stats where p in ('a', 'b'); + +drop table ice_part_stats; + +-- an unpartitioned table keeps its statistics in the same file, which the metastore never holds: +-- reaching them takes the handler, and only the accuracy check stands between a query and stale ones +create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_unpart values (1), (5), (9); +analyze table ice_unpart compute statistics for columns; + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +-- an incremental gather keeps them describing the table, so it still answers +insert into ice_unpart values (11); + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +-- a write that records nothing, as another engine's would, leaves them behind: only the accuracy +-- check stands between the query and a value the table no longer holds +set hive.stats.autogather=false; +insert into ice_unpart values (20); +set hive.stats.autogather=true; + +explain +select max(id) from ice_unpart; + +select max(id) from ice_unpart; + +drop table ice_unpart; + +-- statistics kept for the table as a whole describe no partition in particular, so a query over +-- one of them cannot be answered from them however fresh they are +set hive.iceberg.stats.collect.partlevel=false; + +create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b'); +analyze table ice_tbl_level compute statistics for columns; + +-- a scan reading every partition reads the whole table, which is what they do describe +explain +select max(id) from ice_tbl_level; + +select max(id) from ice_tbl_level; + +explain +select count(id) from ice_tbl_level; + +select count(id) from ice_tbl_level; + +explain +select max(id) from ice_tbl_level where p = 'a'; + +select max(id) from ice_tbl_level where p = 'a'; + +drop table ice_tbl_level; + +set hive.iceberg.stats.collect.partlevel=true; + +-- with the statistics kept by the metastore there are no per-partition numbers to answer from +set hive.iceberg.stats.source=metastore; + +create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b'); + +explain +select max(id) from ice_part_stats_hms where p = 'a'; + +select max(id) from ice_part_stats_hms where p = 'a'; + +drop table ice_part_stats_hms; diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out index c9684a228078..685a51d86efa 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out @@ -321,25 +321,9 @@ POSTHOOK: Input: default@tbl_ice_puffin POSTHOOK: Output: hdfs://### HDFS PATH ### Plan optimized by CBO. -Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) - Stage-0 Fetch Operator - limit:-1 - Stage-1 - Reducer 2 vectorized - File Output Operator [FS_11] - Group By Operator [GBY_10] (rows=1 width=8) - Output:["_col0","_col1"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_9] - Group By Operator [GBY_8] (rows=1 width=8) - Output:["_col0","_col1"],aggregations:["min(a)","max(c)"] - Select Operator [SEL_7] (rows=5 width=8) - Output:["a","c"] - TableScan [TS_0] (rows=5 width=8) - default@tbl_ice_puffin,tbl_ice_puffin,Tbl:COMPLETE,Col:COMPLETE,Output:["a","c"] + limit:1 PREHOOK: query: desc formatted tbl_ice_puffin C PREHOOK: type: DESCTABLE diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out new file mode 100644 index 000000000000..35dc90b880d7 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out @@ -0,0 +1,827 @@ +PREHOOK: query: create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: create external table ice_part_stats (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: insert into ice_part_stats values (1, 'a'), (9, 'a'), (7, 'b'), (3, 'c') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: analyze table ice_part_stats compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats@p=a +PREHOOK: Output: default@ice_part_stats@p=b +PREHOOK: Output: default@ice_part_stats@p=c +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_part_stats compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats@p=a +POSTHOOK: Output: default@ice_part_stats@p=b +POSTHOOK: Output: default@ice_part_stats@p=c +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: insert into ice_part_stats values (11, 'a') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: insert into ice_part_stats values (11, 'a') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: explain +select id from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select id from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats + filterExpr: (p) IN ('a', 'b') (type: boolean) + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +7 +PREHOOK: query: analyze table ice_part_stats partition (p = 'a') compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats +PREHOOK: Output: default@ice_part_stats@p=a +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_part_stats partition (p = 'a') compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats +POSTHOOK: Output: default@ice_part_stats@p=a +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +7 +PREHOOK: query: explain +select count(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(id) from ice_part_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(id) from ice_part_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +PREHOOK: query: explain +select max(id) from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats where p in ('a', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats where p in ('a', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: drop table ice_part_stats +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_part_stats +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats +POSTHOOK: query: drop table ice_part_stats +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_part_stats +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: create external table ice_unpart (id bigint) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: insert into ice_unpart values (1), (5), (9) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (1), (5), (9) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: analyze table ice_unpart compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_unpart compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: insert into ice_unpart values (11) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (11) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: insert into ice_unpart values (20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: insert into ice_unpart values (20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: explain +select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_unpart + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 5 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_unpart +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_unpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: hdfs://### HDFS PATH ### +20 +PREHOOK: query: drop table ice_unpart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_unpart +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_unpart +POSTHOOK: query: drop table ice_unpart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_unpart +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_unpart +PREHOOK: query: create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: create external table ice_tbl_level (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: insert into ice_tbl_level values (1, 'a'), (9, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: analyze table ice_tbl_level compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: default@ice_tbl_level +PREHOOK: Output: default@ice_tbl_level@p=a +PREHOOK: Output: default@ice_tbl_level@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_tbl_level compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: default@ice_tbl_level +POSTHOOK: Output: default@ice_tbl_level@p=a +POSTHOOK: Output: default@ice_tbl_level@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select max(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: explain +select count(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(id) from ice_tbl_level +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(id) from ice_tbl_level +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +3 +PREHOOK: query: explain +select max(id) from ice_tbl_level where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_tbl_level where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_tbl_level + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 2 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_tbl_level where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_tbl_level where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: drop table ice_tbl_level +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_tbl_level +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_tbl_level +POSTHOOK: query: drop table ice_tbl_level +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_tbl_level +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: create external table ice_part_stats_hms (id bigint, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats_hms +PREHOOK: query: insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: insert into ice_part_stats_hms values (1, 'a'), (9, 'a'), (7, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_part_stats_hms +PREHOOK: query: explain +select max(id) from ice_part_stats_hms where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select max(id) from ice_part_stats_hms where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice_part_stats_hms + filterExpr: (p = 'a') (type: boolean) + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: bigint) + outputColumnNames: id + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: max(id) + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select max(id) from ice_part_stats_hms where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select max(id) from ice_part_stats_hms where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: hdfs://### HDFS PATH ### +9 +PREHOOK: query: drop table ice_part_stats_hms +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_part_stats_hms +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_part_stats_hms +POSTHOOK: query: drop table ice_part_stats_hms +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_part_stats_hms +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_part_stats_hms diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index fb8d8fcaa93e..472a884293c5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -30,6 +30,7 @@ import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.common.type.SnapshotContext; @@ -307,7 +308,12 @@ default List getColStatistics(org.apache.hadoop.hive.ql.met } /** - * Returns an aggregated column statistics for the supplied partition list + * Returns an aggregated column statistics for the supplied partition list. + * + *

{@code AggrStats.partsFound} must count only the partitions the aggregate is exact for: a + * caller may answer a query from it in place of reading the data once it equals + * {@code partNames.size()}. + * * @param table table object * @param colNames list of column names * @param partNames list of partition names @@ -372,6 +378,18 @@ default boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table tab return false; } + /** + * Whether the column statistics the handler holds still describe the table, so that a query may + * be answered from them rather than by reading the data. The metastore's accuracy marker only + * records what Hive itself wrote, while a handler's table may be written by other engines. + * @param table table object + * @param colNames the columns being asked about + * @return true if the statistics still describe the table for every column asked + */ + default boolean areColumnStatsUptoDate(org.apache.hadoop.hive.ql.metadata.Table table, List colNames) { + return StatsSetupConst.areColumnStatsUptoDate(table.getParameters(), colNames); + } + /** * Returns the row count of the table, letting queries like count(1) be answered from statistics. * @param hmsTable table object diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index bff675b0d98f..6076859e25ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -21,12 +21,21 @@ import com.google.common.collect.Lists; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; +import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -49,7 +58,6 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.lib.SemanticRule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; -import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; @@ -79,20 +87,19 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.List; import java.util.Map; import java.util.Set; import java.util.Stack; - /** There is a set of queries which can be answered entirely from statistics stored in metastore. * Examples of such queries are count(*), count(a), max(a), min(b) etc. Hive already collects * these basic statistics for query planning purposes. These same statistics can be used to @@ -127,7 +134,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { String SEL = SelectOperator.getOperatorName() + "%"; String FS = FileSinkOperator.getOperatorName() + "%"; - Map opRules = new LinkedHashMap(); + Map opRules = new LinkedHashMap<>(); opRules.put(new RuleRegExp("R1", TS + SEL + GBY + RS + GBY + SEL + FS), new MetaDataProcessor(pctx)); opRules.put(new RuleRegExp("R2", TS + SEL + GBY + RS + GBY + FS), @@ -137,8 +144,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { SemanticDispatcher disp = new DefaultRuleDispatcher(null, opRules, soProcCtx); SemanticGraphWalker ogw = new DefaultGraphWalker(disp); - ArrayList topNodes = new ArrayList(); - topNodes.addAll(pctx.getTopOps().values()); + List topNodes = new ArrayList<>(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; } @@ -215,24 +221,36 @@ private StatType getType(String origType) { return StatType.Unsupported; } - private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { - - switch(type) { - case Integer : - return statData.getLongStats().getNumNulls(); - case Double: - return statData.getDoubleStats().getNumNulls(); - case String: - return statData.getStringStats().getNumNulls(); - case Boolean: - return statData.getBooleanStats().getNumNulls(); - case Binary: - return statData.getBinaryStats().getNumNulls(); - case Date: - return statData.getDateStats().getNumNulls(); - default: - return null; - } + private Long getNullCountFor(StatType type, ColumnStatisticsData statData) { + return switch (type) { + case Integer -> statData.getLongStats().getNumNulls(); + case Double -> statData.getDoubleStats().getNumNulls(); + case String -> statData.getStringStats().getNumNulls(); + case Boolean -> statData.getBooleanStats().getNumNulls(); + case Binary -> statData.getBinaryStats().getNumNulls(); + case Date -> statData.getDateStats().getNumNulls(); + // named rather than defaulted, so a type added to StatType fails to compile here + case Unsupported -> null; + }; + } + + /** + * The statistics of no rows: nothing counted, no low or high value. The branches below already + * fold that to the right answer - zero for a count, NULL for a min or a max. + * + * @return null for a type this rewrite cannot answer for + */ + private static ColumnStatisticsData emptyColStats(StatType type) { + return switch (type) { + case Integer -> ColumnStatisticsData.longStats(new LongColumnStatsData()); + case Double -> ColumnStatisticsData.doubleStats(new DoubleColumnStatsData()); + case String -> ColumnStatisticsData.stringStats(new StringColumnStatsData()); + case Boolean -> ColumnStatisticsData.booleanStats(new BooleanColumnStatsData()); + case Binary -> ColumnStatisticsData.binaryStats(new BinaryColumnStatsData()); + case Date -> ColumnStatisticsData.dateStats(new DateColumnStatsData()); + // named rather than defaulted, so a type added to StatType fails to compile here + case Unsupported -> null; + }; } private GbyKeyType getGbyKeyType(GroupByOperator gbyOp) { @@ -256,7 +274,7 @@ private GbyKeyType getGbyKeyType(GroupByOperator gbyOp) { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { + Object... nodeOutputs) { // 1. Do few checks to determine eligibility of optimization // 2. look at ExprNodeFuncGenericDesc in select list to see if its min, max, count etc. @@ -311,7 +329,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } - Long rowCnt = getRowCnt(tsOp, tbl); + final Long rowCnt = getRowCnt(tsOp, tbl); // if we can not have correct table stats, then both the table stats and column stats are not useful. if (rowCnt == null) { return null; @@ -336,7 +354,7 @@ else if (getGbyKeyType(pgbyOp) == GbyKeyType.CONSTANT && rowCnt == 0) { return null; } ReduceSinkOperator rsOp = (ReduceSinkOperator)stack.get(3); - if (rsOp.getConf().getDistinctColumnIndices().size() > 0) { + if (!rsOp.getConf().getDistinctColumnIndices().isEmpty()) { // we can't handle distinct return null; } @@ -388,10 +406,15 @@ else if (getGbyKeyType(cgbyOp) == GbyKeyType.CONSTANT && rowCnt == 0) { return null; // todo we can collapse this part of tree into single TS } - List oneRow = new ArrayList(); + List oneRow = new ArrayList<>(); - AcidUtils.TableSnapshot tableSnapshot = - AcidUtils.getTableSnapshot(pctx.getConf(), tbl); + // Every aggregate of one query asks the same partitions about a column of the same table, + // and the statistics of one partition carry every column, so asking once for all of them + // reads what a thousand aggregates would have read a thousand times. + PrunedPartitionList prunedList = tbl.isPartitioned() ? + pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp) : null; + ScanColStats scanColStats = + new ScanColStats(hive, tbl, aggregateColumns(pgbyOp, exprMap), prunedList); for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) { if (aggr.getDistinct()) { @@ -437,382 +460,50 @@ else if (getGbyKeyType(cgbyOp) == GbyKeyType.CONSTANT && rowCnt == 0) { } } else if (udaf instanceof GenericUDAFCount) { - // always long - rowCnt = 0L; - if (aggr.getParameters().isEmpty()) { - // Its either count (*) or count() case - rowCnt = getRowCnt(tsOp, tbl); - if (rowCnt == null) { - return null; - } - } else if (aggr.getParameters().get(0) instanceof ExprNodeConstantDesc) { - if (((ExprNodeConstantDesc) aggr.getParameters().get(0)).getValue() != null) { - // count (1) - rowCnt = getRowCnt(tsOp, tbl); - if (rowCnt == null) { - return null; - } - } - // otherwise it is count(null), should directly return 0. - } else if ((aggr.getParameters().get(0) instanceof ExprNodeColumnDesc) - && exprMap.get(((ExprNodeColumnDesc) aggr.getParameters().get(0)).getColumn()) instanceof ExprNodeConstantDesc) { - if (((ExprNodeConstantDesc) (exprMap.get(((ExprNodeColumnDesc) aggr.getParameters() - .get(0)).getColumn()))).getValue() != null) { - rowCnt = getRowCnt(tsOp, tbl); - if (rowCnt == null) { - return null; - } - } - } else { - // Its count(col) case - ExprNodeColumnDesc desc = (ExprNodeColumnDesc) exprMap.get(((ExprNodeColumnDesc) aggr - .getParameters().get(0)).getColumn()); - String colName = desc.getColumn(); - StatType type = getType(desc.getTypeString()); - if (!tbl.isPartitioned()) { - if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(tbl, tbl.getParameters())) { - Logger.debug("Stats for table : " + tbl.getTableName() + " are not up to date."); - return null; - } - rowCnt = Long.valueOf(tbl.getProperty(StatsSetupConst.ROW_COUNT)); - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; - } - - List stats = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), - Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); - if (stats.isEmpty()) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; - } - Long nullCnt = getNullcountFor(type, stats.get(0).getStatsData()); - if (null == nullCnt) { - Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " - + "metadata optimizer for column : " + colName); - return null; - } else { - rowCnt -= nullCnt; - } - } else { - Set parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp) - .getPartitions(); - for (Partition part : parts) { - if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters())) { - Logger.debug("Stats for part : " + part.getSpec() + " are not up to date."); - return null; - } - long partRowCnt = Long.parseLong(part.getParameters().get( - StatsSetupConst.ROW_COUNT)); - rowCnt += partRowCnt; - } - Collection> result = verifyAndGetPartColumnStats(hive, - tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) - return null; - Long nullCnt = getNullcountFor(type, statData); - if (nullCnt == null) { - Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " - + "metadata optimizer for column : " + colName); - return null; - } else { - rowCnt -= nullCnt; - } - } - } + Long cnt = countFor(aggr, exprMap, rowCnt, scanColStats); + if (cnt == null) { + return null; // logging inside } - oneRow.add(rowCnt); - } else if (udaf instanceof GenericUDAFMax) { - ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); + oneRow.add(cnt); + } else if (udaf instanceof GenericUDAFMax || udaf instanceof GenericUDAFMin) { + // one branch for both: an unset bound is SQL NULL rather than zero, and that rule has + // to read the same way for the least value as for the greatest + ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get( + ((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); String colName = colDesc.getColumn(); StatType type = getType(colDesc.getTypeString()); - if(!tbl.isPartitioned()) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; - } - - List stats = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), - Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); - if (stats.isEmpty()) { - Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); - return null; - } - ColumnStatisticsData statData = stats.get(0).getStatsData(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetHighValue()) { - oneRow.add(subType.cast(lstats.getHighValue())); - } else { - oneRow.add(null); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetHighValue()) { - oneRow.add(subType.cast(dstats.getHighValue())); - } else { - oneRow.add(null); - } - break; - } - case Date: { - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetHighValue()) { - oneRow.add(DateSubType.DAYS.cast(dstats.getHighValue().getDaysSinceEpoch())); - } else { - oneRow.add(null); - } - break; - } - default: - // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; - } - } else { - Set parts = pctx.getPrunedPartitions( - tsOp.getConf().getAlias(), tsOp).getPartitions(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - - Long maxVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - LongColumnStatsData lstats = statData.getLongStats(); - if (!lstats.isSetHighValue()) { - continue; - } - long curVal = lstats.getHighValue(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); - } - if (maxVal != null) { - oneRow.add(subType.cast(maxVal)); - } else { - oneRow.add(maxVal); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - - Double maxVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (!dstats.isSetHighValue()) { - continue; - } - double curVal = statData.getDoubleStats().getHighValue(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); - } - if (maxVal != null) { - oneRow.add(subType.cast(maxVal)); - } else { - oneRow.add(null); - } - break; - } - case Date: { - Long maxVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DateColumnStatsData dstats = statData.getDateStats(); - if (!dstats.isSetHighValue()) { - continue; - } - long curVal = dstats.getHighValue().getDaysSinceEpoch(); - maxVal = maxVal == null ? curVal : Math.max(maxVal, curVal); - } - if (maxVal != null) { - oneRow.add(DateSubType.DAYS.cast(maxVal)); - } else { - oneRow.add(null); - } - break; - } - default: - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; - } + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside } - } else if (udaf instanceof GenericUDAFMin) { - ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); - String colName = colDesc.getColumn(); - StatType type = getType(colDesc.getTypeString()); - if (!tbl.isPartitioned()) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colName)) { - Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName - + " are not up to date."); - return null; + String name = colDesc.getTypeString().toUpperCase(); + boolean high = udaf instanceof GenericUDAFMax; + switch (type) { + case Integer: { + LongColumnStatsData lstats = statData.getLongStats(); + boolean isSet = high ? lstats.isSetHighValue() : lstats.isSetLowValue(); + oneRow.add(isSet ? LongSubType.valueOf(name).cast( + high ? lstats.getHighValue() : lstats.getLowValue()) : null); + break; } - ColumnStatisticsData statData = - hive.getMSC().getTableColumnStatistics( - tbl.getDbName(), tbl.getTableName(), Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null) - .get(0).getStatsData(); - String name = colDesc.getTypeString().toUpperCase(); - switch (type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - LongColumnStatsData lstats = statData.getLongStats(); - if (lstats.isSetLowValue()) { - oneRow.add(subType.cast(lstats.getLowValue())); - } else { - oneRow.add(null); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (dstats.isSetLowValue()) { - oneRow.add(subType.cast(dstats.getLowValue())); - } else { - oneRow.add(null); - } - break; - } - case Date: { - DateColumnStatsData dstats = statData.getDateStats(); - if (dstats.isSetLowValue()) { - oneRow.add(DateSubType.DAYS.cast(dstats.getLowValue().getDaysSinceEpoch())); - } else { - oneRow.add(null); - } - break; - } - default: // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; + case Double: { + DoubleColumnStatsData dstats = statData.getDoubleStats(); + boolean isSet = high ? dstats.isSetHighValue() : dstats.isSetLowValue(); + oneRow.add(isSet ? DoubleSubType.valueOf(name).cast( + high ? dstats.getHighValue() : dstats.getLowValue()) : null); + break; } - } else { - Set parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions(); - String name = colDesc.getTypeString().toUpperCase(); - switch(type) { - case Integer: { - LongSubType subType = LongSubType.valueOf(name); - - Long minVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - LongColumnStatsData lstats = statData.getLongStats(); - if (!lstats.isSetLowValue()) { - continue; - } - long curVal = lstats.getLowValue(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); - } - if (minVal != null) { - oneRow.add(subType.cast(minVal)); - } else { - oneRow.add(minVal); - } - break; - } - case Double: { - DoubleSubType subType = DoubleSubType.valueOf(name); - - Double minVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DoubleColumnStatsData dstats = statData.getDoubleStats(); - if (!dstats.isSetLowValue()) { - continue; - } - double curVal = statData.getDoubleStats().getLowValue(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); - } - if (minVal != null) { - oneRow.add(subType.cast(minVal)); - } else { - oneRow.add(minVal); - } - break; - } - case Date: { - Long minVal = null; - Collection> result = - verifyAndGetPartColumnStats(hive, tbl, colName, parts); - if (result == null) { - return null; // logging inside - } - for (List statObj : result) { - ColumnStatisticsData statData = validateSingleColStat(statObj); - if (statData == null) return null; - DateColumnStatsData dstats = statData.getDateStats(); - if (!dstats.isSetLowValue()) { - continue; - } - long curVal = dstats.getLowValue().getDaysSinceEpoch(); - minVal = minVal == null ? curVal : Math.min(minVal, curVal); - } - if (minVal != null) { - oneRow.add(DateSubType.DAYS.cast(minVal)); - } else { - oneRow.add(null); - } - break; - } - default: // unsupported type - Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + - "metadata optimizer for column : " + colName); - return null; - + case Date: { + DateColumnStatsData dstats = statData.getDateStats(); + boolean isSet = high ? dstats.isSetHighValue() : dstats.isSetLowValue(); + oneRow.add(isSet ? DateSubType.DAYS.cast((high ? + dstats.getHighValue() : dstats.getLowValue()).getDaysSinceEpoch()) : null); + break; } + default: + Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); + return null; } } else { // Unsupported aggregation. Logger.debug("Unsupported aggregation for metadata optimizer: " @@ -821,9 +512,9 @@ else if (udaf instanceof GenericUDAFCount) { } } - List> allRows = new ArrayList>(); - List colNames = new ArrayList(); - List ois = new ArrayList(); + List> allRows = new ArrayList<>(); + List colNames = new ArrayList<>(); + List ois = new ArrayList<>(); if (cselOp == null) { List oneRowWithConstant = new ArrayList<>(); oneRowWithConstant.addAll(posToConstant.values()); @@ -899,39 +590,198 @@ else if (udaf instanceof GenericUDAFCount) { } } - private ColumnStatisticsData validateSingleColStat(List statObj) { - if (statObj.size() > 1) { - Logger.error("More than one stat for a single column!"); - return null; - } else if (statObj.isEmpty()) { - Logger.debug("No stats for some partition and column"); - return null; - } - return statObj.get(0).getStatsData(); + /** The columns the aggregates read, which are the ones statistics have to be fetched for. */ + private static List aggregateColumns(GroupByOperator pgbyOp, Map exprMap) { + return pgbyOp.getConf().getAggregators().stream() + .filter(aggr -> !aggr.getParameters().isEmpty()) + .map(aggr -> aggr.getParameters().get(0)) + .filter(ExprNodeColumnDesc.class::isInstance) + .map(desc -> exprMap.get(((ExprNodeColumnDesc) desc).getColumn())) + .filter(ExprNodeColumnDesc.class::isInstance) + .map(desc -> ((ExprNodeColumnDesc) desc).getColumn()) + .distinct() + .collect(Collectors.toList()); } - private Collection> verifyAndGetPartColumnStats( - Hive hive, Table tbl, String colName, Set parts) throws TException, LockException { - List partNames = new ArrayList(parts.size()); - for (Partition part : parts) { - if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(), part.getParameters(), colName)) { - Logger.debug("Stats for part : " + part.getSpec() + " column " + colName + /** + * The statistics of the columns a scan's aggregates read, fetched once when the first + * aggregate needs them and shared by the rest. An aggregate this rewrite cannot answer leaves + * the query for execution, whole or not at all. Answers for a scan of a partitioned table. + */ + private static final class ScanColStats { + private final Hive hive; + private final Table tbl; + private final List colNames; + private final PrunedPartitionList prunedList; + private Map colStatsByName; + private boolean fetched; + + ScanColStats(Hive hive, Table tbl, List colNames, PrunedPartitionList prunedList) { + this.hive = hive; + this.tbl = tbl; + this.colNames = colNames; + this.prunedList = prunedList; + } + + /** + * One column's statistics. A scan pruned to no partitions reads no rows, and the statistics + * of no rows are the empty ones: nothing counted, and no least or greatest to name. + */ + ColumnStatisticsData statsFor(String colName, StatType type) throws HiveException { + if (prunedList != null && prunedList.getPartitions().isEmpty()) { + return emptyColStats(type); + } + if (!fetched) { + fetched = true; + colStatsByName = prunedList == null ? tableColStats() : partitionColStats(); + } + ColumnStatisticsObj stat = colStatsByName == null ? null : colStatsByName.get(colName); + if (stat == null) { + Logger.debug("No stats for " + tbl.getTableName() + " column " + colName); + return null; + } + return stat.getStatsData(); + } + + /** + * Whether the table's own statistics answer for this scan: it keeps them for the table as + * a whole, and the scan reads every partition. They then describe exactly the rows read. + */ + private boolean answeredByTableStats() { + return !StatsUtils.isPartitionStats(tbl, hive.getConf()) && + prunedList.getReferredPartCols().isEmpty() && !prunedList.hasUnknownPartitions(); + } + + /** The table's own statistics, taken only while they still describe it. */ + private Map tableColStats() throws HiveException { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl, tbl.getParameters(), colNames)) { + Logger.debug("Stats for table : " + tbl.getTableName() + " columns " + colNames + " are not up to date."); return null; } - partNames.add(part.getName()); + return indexByColumnName(hive.getTableColumnStatistics(tbl, colNames, true)); + } + + /** What the scan's partitions hold for every column asked about, or null to decline. */ + private Map partitionColStats() throws HiveException { + Set parts = prunedList.getPartitions(); + List partNames = new ArrayList<>(parts.size()); + // a storage handler holds no partition parameters, and one kept per partition describes no + // partition in particular: whether each still describes itself is answered by the aggregate + // below, which is told the partitions this query pruned to + if (tbl.isNonNative()) { + if (!StatsUtils.checkCanProvideColumnStats(tbl)) { + Logger.debug("Table : " + tbl.getTableName() + " provides no column statistics."); + return null; + } + if (answeredByTableStats()) { + return tableColStats(); + } + parts.forEach(part -> partNames.add(part.getName())); + } else { + for (Partition part : parts) { + if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering( + part.getTable(), part.getParameters(), colNames)) { + Logger.debug("Stats for part : " + part.getSpec() + " columns " + colNames + + " are not up to date."); + return null; + } + partNames.add(part.getName()); + } + } + // Aggregated rather than per partition: the callers fold these with min, max or a sum, so + // merging first gives the same answer. A handler aggregates its own statistics, which + // the metastore cannot hold: PART_COL_STATS rows need a partition Iceberg never creates. + AggrStats aggrStats; + try { + aggrStats = tbl.isNonNative() + ? tbl.getStorageHandler().getAggrColStatsFor(tbl, colNames, partNames) + : exactAggrColStats(partNames); + } catch (MetaException e) { + throw new HiveException(e); + } + if (aggrStats == null || aggrStats.getColStats() == null) { + Logger.debug("No stats for " + tbl.getTableName() + " columns " + colNames); + return null; + } + if (aggrStats.getPartsFound() != parts.size()) { + // a partition whose statistics are missing would leave the answer describing a subset + Logger.debug("Received " + aggrStats.getPartsFound() + " stats for " + parts.size() + " partitions"); + return null; + } + return indexByColumnName(aggrStats.getColStats()); + } + + /** + * Each partition fetched and folded the way a storage handler folds its own: the + * metastore's aggregate endpoint may serve a cached aggregate of a different partition + * set within its variance, which estimates a plan fine but must not answer a query. + */ + private AggrStats exactAggrColStats(List partNames) throws HiveException, MetaException { + Map> statsByPart = hive.getPartitionColumnStatistics( + tbl.getDbName(), tbl.getTableName(), partNames, colNames, true); + List partStats = new ArrayList<>(); + statsByPart.forEach((partitionName, statsObjs) -> { + // a partition counts as found only when it holds every column asked about + if (statsObjs.size() == colNames.size()) { + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, tbl.getDbName(), tbl.getTableName()); + statsDesc.setPartName(partitionName); + partStats.add(new ColumnStatistics(statsDesc, statsObjs)); + } + }); + HiveConf conf = hive.getConf(); + List aggregated = MetaStoreServerUtils.aggrPartitionStats(partStats, + MetaStoreUtils.getDefaultCatalog(conf), tbl.getDbName(), tbl.getTableName(), + partNames, colNames, + partStats.size() == partNames.size(), + MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION), + MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER)); + return new AggrStats(aggregated, partStats.size()); + } + + /** + * The statistics by the column they describe. A source naming one column twice disagrees with + * itself: collecting without a merge function throws, and the query leaves for execution + * rather than an arbitrary one of them standing as an exact answer. + */ + private static Map indexByColumnName( + List colStats) { + return colStats.stream().collect( + Collectors.toMap(ColumnStatisticsObj::getColName, Function.identity())); + } + } + + /** The rows a COUNT reads, or null to decline - logged. */ + private Long countFor(AggregationDesc aggr, Map exprMap, long rowCnt, + ScanColStats scanColStats) throws HiveException { + if (aggr.getParameters().isEmpty()) { + // count(*) or count() + return rowCnt; + } + ExprNodeDesc param = aggr.getParameters().get(0); + if (param instanceof ExprNodeColumnDesc column) { + param = exprMap.get(column.getColumn()); + } + if (param instanceof ExprNodeConstantDesc constant) { + // count(1) reads every row, count(null) none + return constant.getValue() == null ? 0L : rowCnt; + } + // count(col): the rows where it is set + ExprNodeColumnDesc desc = (ExprNodeColumnDesc) param; + String colName = desc.getColumn(); + StatType type = getType(desc.getTypeString()); + + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); + if (statData == null) { + return null; // logging inside } - AcidUtils.TableSnapshot tableSnapshot = - AcidUtils.getTableSnapshot(hive.getConf(), tbl); - - Map> result = hive.getMSC().getPartitionColumnStatistics( - tbl.getDbName(), tbl.getTableName(), partNames, Lists.newArrayList(colName), - Constants.HIVE_ENGINE, tableSnapshot != null ? tableSnapshot.getValidWriteIdList() : null); - if (result.size() != parts.size()) { - Logger.debug("Received " + result.size() + " stats for " + parts.size() + " partitions"); + Long nullCnt = getNullCountFor(type, statData); + if (nullCnt == null) { + Logger.debug("Unsupported type: " + desc.getTypeString() + " encountered in " + + "metadata optimizer for column : " + colName); return null; } - return result.values(); + return rowCnt - nullCnt; } private Long getRowCnt(TableScanOperator tsOp, Table tbl) throws HiveException { @@ -950,6 +800,7 @@ private Long getRowCnt(TableScanOperator tsOp, Table tbl) throws HiveException { for (Partish partish : partishList) { Map basicStats = partish.getPartParameters(); if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(partish.getTable(), basicStats)) { + Logger.debug("Stats for {} are not up to date.", partish.getSimpleName()); return null; } rowCnt += Long.parseLong(basicStats.get(StatsSetupConst.ROW_COUNT)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 2dd1dffc3417..8f612a092677 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -2070,8 +2070,23 @@ public static boolean areBasicStatsUptoDateForQueryAnswering(Table table, Map params, String colName) { - return checkCanProvideStats(table) && StatsSetupConst.areColumnStatsUptoDate(params, colName); + public static boolean areColumnStatsUptoDateForQueryAnswering(Table table, Map params, + String colName) { + return areColumnStatsUptoDateForQueryAnswering(table, params, List.of(colName)); + } + + /** + * The same, asked of every column at once: a handler settles which of its statistics answer + * once, and that question is the same whatever column is asked about. + */ + public static boolean areColumnStatsUptoDateForQueryAnswering(Table table, Map params, + List colNames) { + // a handler keeps its own statistics and knows what happened to them, including writes by + // other engines that never touched the metastore marker + return checkCanProvideStats(table) && ( + table.isNonNative() ? table.getStorageHandler().areColumnStatsUptoDate(table, colNames) : + StatsSetupConst.areColumnStatsUptoDate(params, colNames) + ); } /** diff --git a/ql/src/test/queries/clientpositive/stats_part.q b/ql/src/test/queries/clientpositive/stats_part.q index d0812e100781..01af904a0026 100644 --- a/ql/src/test/queries/clientpositive/stats_part.q +++ b/ql/src/test/queries/clientpositive/stats_part.q @@ -45,6 +45,9 @@ explain select count(key) from stats_part; --select count(*) from stats_part where p = 100; explain select count(key) from stats_part where p > 100; --select count(*) from stats_part where p > 100; +-- no partitions yet, so no rows: count is zero but max has no greatest to name +explain select max(key) from stats_part; +select max(key) from stats_part; desc formatted stats_part; --explain insert into table stats_part partition(p=100) select distinct key, value from mysource where p == 100; diff --git a/ql/src/test/results/clientpositive/llap/stats_part.q.out b/ql/src/test/results/clientpositive/llap/stats_part.q.out index c474362fa244..a69590cc2f21 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part.q.out @@ -134,6 +134,33 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: explain select max(key) from stats_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +#### A masked pattern was here #### +POSTHOOK: query: explain select max(key) from stats_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select max(key) from stats_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_part +#### A masked pattern was here #### +POSTHOOK: query: select max(key) from stats_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_part +#### A masked pattern was here #### +NULL PREHOOK: query: desc formatted stats_part PREHOOK: type: DESCTABLE PREHOOK: Input: default@stats_part diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java index 904ae6245c70..a441c8a7d4de 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.common; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.ArrayList; import java.util.Map; @@ -281,12 +282,20 @@ public static boolean areBasicStatsUptoDate(Map params) { return stats.basicStats; } - public static boolean areColumnStatsUptoDate(Map params, String colName) { + /** Whether the stored column statistics are up to date for every column asked. */ + public static boolean areColumnStatsUptoDate(Map params, List colNames) { if (params == null) { - return false; + // every column of no columns is up to date + return colNames.isEmpty(); } + // the marker is one document holding every column, so it is parsed for the ask rather than + // once per column: a caller asking per partition would otherwise reparse it per column too ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); - return stats.columnStats.containsKey(colName); + return stats.columnStats.keySet().containsAll(colNames); + } + + public static boolean areColumnStatsUptoDate(Map params, String colName) { + return areColumnStatsUptoDate(params, Collections.singletonList(colName)); } // It will only throw JSONException when stats.put(BASIC_STATS, TRUE) From 2fb4e7fbb88f1f07389f9fad7e6cebe78242fb05 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 10 Sep 2026 18:28:36 +0300 Subject: [PATCH 02/13] HIVE-29834: Refuse an answer whose exactness is unproven Entries in a partition blob answer by their field id even when a read decodes the whole blob, so what a dropped column left behind is stepped over rather than served under a namesake added since. And values aggregated from only some of the partitions a scan reads carry that mark, so the filter-reduction rule keeps estimating from them but never folds a predicate to a constant over them. --- .../mr/hive/stats/IcebergColStatsReader.java | 12 +++- .../mr/hive/TestHiveIcebergStatistics.java | 57 +++++++++++++++++++ .../HiveReduceExpressionsWithStatsRule.java | 5 +- .../hadoop/hive/ql/plan/ColStatistics.java | 11 ++++ .../hadoop/hive/ql/stats/StatsUtils.java | 9 ++- 5 files changed, 86 insertions(+), 8 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java index 0216f225acef..1f4b120bb35f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java @@ -452,12 +452,18 @@ private static Optional hadoopStream(SeekableInputStream in) return Optional.empty(); } - /** The fields the asked columns are, so a read can step over the entries of the rest. */ + /** + * The fields the asked columns are, so a read can step over the entries of the rest. An entry + * answers by its field id, never by its name alone: a full ask filters by the schema's own + * fields, so an entry a dropped column left behind is stepped over even when a column added + * since carries its name. + */ private static IntPredicate fieldsOf(Table table, Set columns) { + Set fields = Sets.newHashSet(); if (columns == null) { - return null; + table.schema().columns().forEach(field -> fields.add(field.fieldId())); + return fields::contains; } - Set fields = Sets.newHashSet(); for (String column : columns) { Types.NestedField field = table.schema().caseInsensitiveFindField(column); if (field != null) { diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index 111662cc4e6d..a01673b590ab 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -2688,6 +2688,63 @@ public void testAColumnAddedBackDoesNotInheritWhatItsNamesakeStated() { checkColStatMinMaxValue(identifier.name(), "id", 1, 3); } + @Test + public void testAFilterIsNotFoldedFromAPartitionSubsetsRange() { + // values aggregated from some of the scanned partitions estimate, but never answer: a filter + // probing a value the analyzed partition never held must still run, not fold to false + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_subset_fold"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_OPTIMIZE_REDUCE_WITH_STATS.varname, true); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (100, 'a'), (900, 'b')"); + shell.executeStatement( + "ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + List served = shell.executeStatement("SELECT id FROM " + identifier + " WHERE id = 900"); + Assert.assertEquals("the row outside the analyzed partition's range is found", 1, served.size()); + Assert.assertEquals(900L, served.get(0)[0]); + + // and where the statistics answer for every scanned partition, the fold still fires + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + List plan = shell.executeStatement("EXPLAIN SELECT id FROM " + identifier + " WHERE id = 5000"); + boolean probes = plan.stream().map(row -> String.valueOf(row[0])).anyMatch(line -> line.contains("5000")); + Assert.assertFalse("a probe beyond every partition's range is folded away", probes); + } + + @Test + public void testARecreatedColumnDoesNotAnswerFromItsNamesakesPartitionEntry() throws Exception { + // a full ask decodes each partition blob whole; the entries still answer by field id, so what + // a dropped column left behind is stepped over even though a column added since bears its name + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_readded_part"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, amount bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 100, 'a'), (2, 200, 'a')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // moves no snapshot: the stored partition entries stay fresh, only the field behind the name changes + shell.executeStatement("ALTER TABLE " + identifier + " REPLACE COLUMNS (id bigint, p string)"); + shell.executeStatement("ALTER TABLE " + identifier + " ADD COLUMNS (amount bigint)"); + + HiveIcebergStorageHandler handler = storageHandler(); + org.apache.hadoop.hive.ql.metadata.Table hmsTable = hmsTable(identifier); + AggrStats aggr = handler.getAggrColStatsFor(hmsTable, + List.of("id", "amount", "p"), partitionNames(handler, hmsTable)); + Assert.assertEquals("a partition whose blob answers for a dropped field does not count as found", + 0, aggr.getPartsFound()); + Assert.assertTrue("and nothing is served under the recreated column's name", + aggr.getColStats().stream().noneMatch(statsObj -> "amount".equals(statsObj.getColName()))); + } + @Test public void testTheTableMetadataRegistersOnePartitionEntryNamingEveryFieldAndThePartitionCount() { // a partition's statistics are addressed through the file's own footer: registering an entry diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java index 30b331f5d675..8b2c4f538e5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java @@ -303,8 +303,9 @@ private ColStatistics extractColStats(RexInputRef ref) { if (table != null) { ColStatistics colStats = table.getColStat(Lists.newArrayList(columnOrigin.getOriginColumnOrdinal()), false).get(0); - if (colStats != null && StatsUtils.areColumnStatsUptoDateForQueryAnswering( - table.getHiveTableMD(), table.getHiveTableMD().getParameters(), colStats.getColumnName())) { + if (colStats != null && !colStats.isPartialAggregate() && + StatsUtils.areColumnStatsUptoDateForQueryAnswering( + table.getHiveTableMD(), table.getHiveTableMD().getParameters(), colStats.getColumnName())) { return colStats; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java index f334b44e504b..e835c107a657 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ColStatistics.java @@ -31,6 +31,7 @@ public class ColStatistics { private Range range; private boolean isPrimaryKey; private boolean isEstimated; + private boolean partialAggregate; private boolean isFilteredColumn; private byte[] bitVectors; private byte[] histogram; @@ -171,6 +172,7 @@ public ColStatistics clone() { clone.setHistogram(histogram); clone.setPrimaryKey(isPrimaryKey); clone.setIsEstimated(isEstimated); + clone.setPartialAggregate(partialAggregate); clone.setIsFilteredColumn(isFilteredColumn); if (range != null ) { clone.setRange(range.clone()); @@ -182,6 +184,15 @@ public boolean isPrimaryKey() { return isPrimaryKey; } + /** Whether these values were aggregated from only some of the partitions the scan reads. */ + public boolean isPartialAggregate() { + return partialAggregate; + } + + public void setPartialAggregate(boolean partialAggregate) { + this.partialAggregate = partialAggregate; + } + public void setPrimaryKey(boolean isPrimaryKey) { this.isPrimaryKey = isPrimaryKey; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 8f612a092677..380542802737 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -464,9 +464,9 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p stats.addToColumnStats(columnStats); } else { - if (statsRetrieved) { - columnStats.addAll(convertColStats(aggrStats.getColStats())); - } + List aggregatedStats = statsRetrieved ? + convertColStats(aggrStats.getColStats()) : Collections.emptyList(); + columnStats.addAll(aggregatedStats); int colStatsAvailable = neededColumns.size() + partitionCols.size() - partitionColsToRetrieve.size(); if (columnStats.size() != colStatsAvailable) { LOG.debug("Column stats requested for : {} columns. Able to retrieve for {} columns", @@ -491,6 +491,9 @@ private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList p // Change if we could not retrieve for all partitions if (aggrStats != null && aggrStats.getPartsFound() != partNames.size() && stats.getColumnStatsState() != State.NONE) { stats.updateColumnStatsState(State.PARTIAL); + // values aggregated from a subset of the scanned partitions estimate, but never + // answer; a partition column's stats come from the pruned values and stay exact + aggregatedStats.forEach(colStats -> colStats.setPartialAggregate(true)); LOG.debug("Column stats requested for : {} partitions. Able to retrieve for {} partitions", partNames.size(), aggrStats.getPartsFound()); } From 3a66ead86760cfffbfb962c629e66fbb19cf794a Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 10 Sep 2026 19:42:55 +0300 Subject: [PATCH 03/13] HIVE-29834: Serve a whole-table read from a partition-level file's aggregates A partition-level file holds the table-level aggregates at its tail, so a session reading at table level answers from them rather than finding nothing at its own granularity - but only while the file states the table, which its registered entry marks. A gather over every partition states it, and a merge does while the file it carried from did; a partition-scoped gather with nothing to carry describes its partition alone and marks nothing. The write side stays strict: a table-level increment merges only into a file gathered as one, and a partition-level increment only into a partition-level file, so the leniency is a read-side courtesy over what is physically already there, never a cross-granularity merge. --- .../mr/hive/stats/IcebergColStatsReader.java | 33 +-- .../stats/IcebergColStatsWritePolicy.java | 2 +- .../mr/hive/stats/IcebergColStatsWriter.java | 92 +++++-- .../mr/hive/stats/IcebergStoredStats.java | 50 +++- .../mr/hive/TestHiveIcebergStatistics.java | 231 +++++++++++++++++- .../hive/stats/TestIcebergColStatsFormat.java | 2 +- 6 files changed, 344 insertions(+), 66 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java index 1f4b120bb35f..7fd83ae8a919 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java @@ -122,12 +122,12 @@ static List readOrThrow(Table table, StatisticsFile statsFi .withFooterSize(statsFile.fileFooterSizeInBytes()) .build()) { + IntPredicate liveFields = liveFieldsOf(table); + List blobMetadata = reader.fileMetadata().blobs().stream() .filter(IcebergColStatsReader::holdsColStats) + .filter(blob -> liveFields.test(blob.inputFields().getFirst())) .filter(holdsNeededColumn) - // a column dropped and added back is a different field: what was stored for the one it - // replaced describes rows the column of that name never held - .filter(blob -> table.schema().findField(blob.inputFields().getFirst()) != null) .toList(); LOG.info("Using column stats from: {}", statsPath); @@ -199,7 +199,7 @@ private static List readAggr(Table table, StatisticsFile st // the registered entry states how many partitions the file describes: an ask of another // size cannot be the exact set, and is turned away without opening the file for (var blob : statsFile.blobMetadata()) { - String numPartitions = blob.properties().get(IcebergColStatsWriter.NUM_PARTITIONS_FIELD); + String numPartitions = blob.properties().get(IcebergColStatsWriter.NUM_PARTITIONS_PROP); if (numPartitions != null && !numPartitions.equals(String.valueOf(asked.size()))) { return null; } @@ -215,7 +215,7 @@ private static List readAggr(Table table, StatisticsFile st // what answers without opening the file, and this read is opening it anyway Set described = Sets.newHashSet(); for (BlobMetadata blob : reader.fileMetadata().blobs()) { - String partName = blob.properties().get(IcebergColStatsWriter.PARTITION_FIELD); + String partName = blob.properties().get(IcebergColStatsWriter.PARTITION_PROP); if (partName != null) { described.add(partName); } @@ -225,11 +225,11 @@ private static List readAggr(Table table, StatisticsFile st if (described.isEmpty() || !described.equals(asked) || !described.stream().allMatch(upToDate)) { return null; } + // a dead field resolves to no name of its own, so blobsForColumns leaves its entry out Predicate holdsNeededColumn = blobsForColumns(table, columns); List blobMetadata = reader.fileMetadata().blobs().stream() .filter(IcebergColStatsReader::holdsColStats) - .filter(blob -> table.schema().findField(blob.inputFields().getFirst()) != null) .filter(holdsNeededColumn) .toList(); @@ -272,7 +272,7 @@ public static Map> readPart(Table table, Stati if (!IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type())) { return false; } - String partName = metadata.properties().get(IcebergColStatsWriter.PARTITION_FIELD); + String partName = metadata.properties().get(IcebergColStatsWriter.PARTITION_PROP); return partName != null && (partitionFilter == null || partitionFilter.test(partName)); }) .toList(); @@ -345,7 +345,7 @@ static void readBlobs(SeekableInputStream in, List blobs, Set hadoopStream(SeekableInputStream in) } /** - * The fields the asked columns are, so a read can step over the entries of the rest. An entry - * answers by its field id, never by its name alone: a full ask filters by the schema's own - * fields, so an entry a dropped column left behind is stepped over even when a column added - * since carries its name. + * Whether the field is one the schema still has. An entry answers by its field id, never by its + * name alone: a column dropped and added back keeps the name and takes a new field, so an entry + * the dropped one left behind is stepped over even though a column of that name exists. */ + static IntPredicate liveFieldsOf(Table table) { + return id -> table.schema().findField(id) != null; + } + + /** The fields the asked columns are, so a read can step over the entries of the rest. */ private static IntPredicate fieldsOf(Table table, Set columns) { - Set fields = Sets.newHashSet(); if (columns == null) { - table.schema().columns().forEach(field -> fields.add(field.fieldId())); - return fields::contains; + return liveFieldsOf(table); } + Set fields = Sets.newHashSet(); for (String column : columns) { Types.NestedField field = table.schema().caseInsensitiveFindField(column); if (field != null) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWritePolicy.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWritePolicy.java index ea4365487a65..90fa3b547a1f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWritePolicy.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWritePolicy.java @@ -198,7 +198,7 @@ private static boolean isAnalyze(Configuration conf) { } /** Whether the ANALYZE named the partitions it is for, leaving the rest of the table alone. */ - private static boolean isAnalyzePartition(Configuration conf) { + static boolean isAnalyzePartition(Configuration conf) { return SessionStateUtil.getQueryState(conf).map(QueryState::isAnalyzePartition) .orElse(false); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java index cbac97e2a0fc..7b7629a2fc99 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.function.IntPredicate; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -95,9 +96,16 @@ public final class IcebergColStatsWriter { */ public static final String HIVE_PART_COL_STATS_BLOB_V1 = "hive-partition-column-statistics-v1"; /** What a blob describing one partition names it under, in the metadata that stands for it. */ - public static final String PARTITION_FIELD = "partition"; + public static final String PARTITION_PROP = "partition"; /** How many partitions the file describes, stated on its registered entry. */ - public static final String NUM_PARTITIONS_FIELD = "numPartitions"; + public static final String NUM_PARTITIONS_PROP = "numPartitions"; + /** + * Whether the file's aggregates answer for the whole table, stated on its registered entry: a + * gather over every partition does, and a merge does while the file it carried from did and + * this gather measured every partition changed since it. A partition-scoped gather with nothing + * to carry describes its partition alone, however the granularity of a later read. + */ + public static final String FULL_TABLE_AGGR_PROP = "fullTableAggr"; /** * What a table-level entry is named now that it holds a Thrift struct rather than a serialized * Java object. A reader that knows neither name reads it as absent, and one that knows both @@ -163,8 +171,10 @@ private static boolean writeTable(Table tbl, Snapshot snapshot, Iterator written) { + if (policy != IcebergColStatsWritePolicy.MERGE) { + return !IcebergColStatsWritePolicy.isAnalyzePartition(conf); + } + Set changed = statsOldSrc == null ? null : IcebergStoredStats.partitionsChangedSince( + tbl, snapshot, statsOldSrc.snapshotId(), conf, false); + return IcebergStoredStats.hasFullTableAggr(statsOldSrc) && + changed != null && written.containsAll(changed); + } + /** * Carries forward, bytes for bytes, the stored entries of the partitions this write never * measured, as long as no write since the stored file changed them. Carrying is the one place @@ -277,7 +305,7 @@ private static void carryForward(Table tbl, Snapshot snapshot, PuffinWriter writ List carried = Lists.newArrayList(); for (BlobMetadata metadata : reader.fileMetadata().blobs()) { - String partName = metadata.properties().get(PARTITION_FIELD); + String partName = metadata.properties().get(PARTITION_PROP); if (!HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type()) || partName == null) { continue; } @@ -291,8 +319,12 @@ private static void carryForward(Table tbl, Snapshot snapshot, PuffinWriter writ // instead of being rebuilt by decoding every carried blob boolean seedFromStored = Sets.intersection(written, storedPartitions).isEmpty() && carried.size() == storedPartitions.size(); + // by field id, not name: a column dropped and added back keeps the name and takes a new + // field, and folding the dead field's entry in would answer for the live one + IntPredicate liveFields = IcebergColStatsReader.liveFieldsOf(tbl); + if (seedFromStored) { - aggregate.seedFrom(reader, carried.size()); + aggregate.seedFrom(reader, carried.size(), liveFields); } for (Pair blob : reader.readAll(carried)) { ByteBuffer carriedBytes = blob.second(); @@ -300,7 +332,8 @@ private static void carryForward(Table tbl, Snapshot snapshot, PuffinWriter writ // travels on untouched try { if (!seedFromStored) { - aggregate.addPartition(IcebergColStatsReader.decodePartBlob(carriedBytes, null, true)); + aggregate.addPartition( + IcebergColStatsReader.decodePartBlob(carriedBytes, null, true, liveFields)); } } catch (InvalidObjectException e) { throw new IOException(e); @@ -310,8 +343,8 @@ private static void carryForward(Table tbl, Snapshot snapshot, PuffinWriter writ snapshot.snapshotId(), snapshot.sequenceNumber(), carriedBytes, PuffinCompressionCodec.NONE, - Map.of(PARTITION_FIELD, - blob.first().properties().get(PARTITION_FIELD)))); + Map.of(PARTITION_PROP, + blob.first().properties().get(PARTITION_PROP)))); } } } @@ -343,9 +376,11 @@ private void addPartition(List statsObjs) throws InvalidObj * aggregating its partitions again would reach: an entry is written only when every partition * states the column, and what suppressed it then is carried unchanged now. */ - private void seedFrom(PuffinReader reader, int carriedPartitions) throws IOException { + private void seedFrom(PuffinReader reader, int carriedPartitions, IntPredicate liveFields) + throws IOException { List aggregateBlobs = reader.fileMetadata().blobs().stream() .filter(metadata -> HIVE_COL_STATS_BLOB_V1.equals(metadata.type())) + .filter(metadata -> liveFields.test(metadata.inputFields().getFirst())) .toList(); List entries = Lists.newArrayList(); for (Pair blob : reader.readAll(aggregateBlobs)) { @@ -398,13 +433,18 @@ private void write(PuffinWriter writer, Snapshot snapshot, Schema schema) throws } } + /** + * What a write leaves for the table metadata: the field ids the file names, empty where each blob + * names its own, and whether the registered partition entry carries the full-table mark - which a + * file with no partition entry never does, whatever its aggregates cover. + */ + private record RegisteredStats(List namedFields, boolean fullTableAggr) { + } + @FunctionalInterface private interface BlobWriter { - /** - * Streams the blobs; returns the field ids the file names in the table metadata, empty where - * each blob names its own. - */ - List write(PuffinWriter writer) throws IOException, InvalidObjectException; + /** Streams the blobs; returns what the table metadata keeps of what was written. */ + RegisteredStats write(PuffinWriter writer) throws IOException, InvalidObjectException; } /** @@ -434,7 +474,8 @@ private static List mergedFieldIds(List gathered, StatisticsFi * every commit. */ private static List registeredBlobs( - List written, List namedFields) { + List written, List namedFields, + boolean fullTableAggr) { long numPartitions = written.stream() .filter(blob -> HIVE_PART_COL_STATS_BLOB_V1.equals(blob.type())) .count(); @@ -447,13 +488,15 @@ private static List registeredBlobs( } fieldsNamed = true; // the count lets a read turn away an ask of another size without opening the file - Map properties = ImmutableMap.builder() + ImmutableMap.Builder properties = ImmutableMap.builder() .putAll(blob.properties()) - .put(NUM_PARTITIONS_FIELD, String.valueOf(numPartitions)) - .build(); + .put(NUM_PARTITIONS_PROP, String.valueOf(numPartitions)); + if (fullTableAggr) { + properties.put(FULL_TABLE_AGGR_PROP, "true"); + } registered.add(GenericBlobMetadata.from(new org.apache.iceberg.puffin.BlobMetadata( blob.type(), namedFields, blob.snapshotId(), blob.sequenceNumber(), - blob.offset(), blob.length(), blob.compressionCodec(), properties))); + blob.offset(), blob.length(), blob.compressionCodec(), properties.build()))); continue; } registered.add(GenericBlobMetadata.from(blob)); @@ -486,7 +529,7 @@ private static boolean commitFile(Table tbl, Snapshot snapshot, BlobWriter blobs try (PuffinWriter writer = Puffin.write(tbl.io().newOutputFile(statsPath)) .createdBy(Constants.HIVE_ENGINE) .build()) { - List namedFields = blobs.write(writer); + RegisteredStats registeredStats = blobs.write(writer); if (writer.writtenBlobsMetadata().isEmpty()) { // committing this would register a file describing nothing, in place of one that may // describe something: a read resolves it, finds no statistics of ours in it, and the @@ -501,7 +544,8 @@ private static boolean commitFile(Table tbl, Snapshot snapshot, BlobWriter blobs statsPath, writer.fileSize(), writer.footerSize(), - registeredBlobs(writer.writtenBlobsMetadata(), namedFields)); + registeredBlobs(writer.writtenBlobsMetadata(), + registeredStats.namedFields(), registeredStats.fullTableAggr())); } catch (Exception e) { tbl.io().deleteFile(statsPath); if (!(e instanceof IOException)) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java index 3e1465602407..1258992957e9 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java @@ -62,7 +62,7 @@ public final class IcebergStoredStats { private static final Logger LOG = LoggerFactory.getLogger(IcebergStoredStats.class); - private static final String STATED_FIELD_IDS_KEY = "statedFieldIds.%s.%d.%b"; + private static final String STORED_FIELD_IDS_KEY = "storedFieldIds.%s.%d.%s"; private static final String CHANGED_PARTITIONS_KEY = "changedPartitions.%s.%d.%d.%d"; /** * What changed between two snapshots is settled the moment the later one commits, so the answer @@ -140,22 +140,42 @@ private static StatisticsFile colStatsFileOf(Table table, long snapshotId, boole * The file whose blobs are Hive's own - Iceberg keeps statistics of its own in the same format - * at the asked-for granularity: a blob describing one partition names it in its metadata. * - *

A file that holds any partition is a per partition one, whatever else it holds. The entries - * it aggregates from them state the table only while it holds every partition, which a gather of some - * of them does not, so a whole-table read passes it by and takes the file gathered as one. + *

A file that holds any partition is a per partition one, whatever else it holds. Its + * aggregates serve a whole-table read only while they aggregate the full table - what a gather + * over every partition marked on it, and a gather of some of them did not. */ private static boolean holdsHiveColStats(StatisticsFile stats, boolean partitionLevel) { boolean holdsPartitions = stats.blobMetadata().stream() - .anyMatch(metadata -> metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_FIELD)); + .anyMatch(metadata -> metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)); if (partitionLevel) { return holdsPartitions && stats.blobMetadata().stream().anyMatch( metadata -> IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type())); } - return !holdsPartitions && stats.blobMetadata().stream().anyMatch( + if (holdsPartitions) { + return hasFullTableAggr(stats); + } + return stats.blobMetadata().stream().anyMatch( metadata -> IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(metadata.type()) || IcebergColStatsWriter.LEGACY_COL_STATS_BLOB.equals(metadata.type())); } + /** Whether the file's aggregates answer for the whole table: its registered entry says so. */ + static boolean hasFullTableAggr(StatisticsFile stats) { + return stats != null && stats.blobMetadata().stream().anyMatch( + metadata -> "true".equals(metadata.properties().get(IcebergColStatsWriter.FULL_TABLE_AGGR_PROP))); + } + + /** + * The stored table-level file, taken as it was gathered: a write merges only into a file + * gathered as one, where a read may also take a partition-level file's aggregates. + */ + static StatisticsFile getTableOnlyColStatsFile(Table table, long snapshotId) { + StatisticsFile stats = getColStatsFile(table, snapshotId, false); + return stats == null || stats.blobMetadata().stream() + .anyMatch(metadata -> metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)) ? + null : stats; + } + /** * Whether the stored column statistics answer for the column: they still describe the snapshot * the table names, and their file holds an entry for it. The footer names the measured columns - @@ -169,21 +189,21 @@ public static boolean colStatsAccurate(org.apache.hadoop.hive.ql.metadata.Table if (snapshot == null) { return false; } - Set stated = statedFieldIds(table, snapshot, conf); + Set stored = storedFieldIds(table, snapshot, conf); return colNames.stream().allMatch(colName -> { Types.NestedField field = table.schema().caseInsensitiveFindField(colName); - return field != null && stated.contains(field.fieldId()); + return field != null && stored.contains(field.fieldId()); }); } /** - * The fields the stored statistics state for the snapshot. Which file answers and what it names - * is the same question for every column, and a partition-level file names them over one blob per - * partition, so it is asked once for the query rather than once per column asked about. + * The fields the stored statistics state for the snapshot. Which file answers is the same + * question for every column and costs a walk of the snapshot's parentage, so it is asked once + * for the query rather than once per column asked about. */ - private static Set statedFieldIds(Table table, Snapshot snapshot, Configuration conf) { + private static Set storedFieldIds(Table table, Snapshot snapshot, Configuration conf) { boolean partitionLevel = IcebergTableUtil.isPartitionStats(table, conf); - String cacheKey = STATED_FIELD_IDS_KEY.formatted(table.name(), snapshot.snapshotId(), partitionLevel); + String cacheKey = STORED_FIELD_IDS_KEY.formatted(table.name(), snapshot.snapshotId(), partitionLevel); Optional cached = SessionStateUtil.getResource(conf, cacheKey); if (cached.isPresent()) { @SuppressWarnings("unchecked") @@ -191,8 +211,12 @@ private static Set statedFieldIds(Table table, Snapshot snapshot, Confi return hit; } StatisticsFile statsFile = getColStatsFile(table, snapshot.snapshotId(), partitionLevel); + // a table-level ask counts the fields the aggregates state: the partition entry names every + // field any partition stated, which a column not every partition holds would ride into Set fields = statsFile == null ? Set.of() : statsFile.blobMetadata().stream() + .filter(metadata -> partitionLevel || + !metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)) .flatMap(metadata -> metadata.fields().stream()) .collect(Collectors.toSet()); SessionStateUtil.addResource(conf, cacheKey, fields); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index a01673b590ab..6e00fadd220c 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -1805,7 +1805,7 @@ public void testTableLevelColStatsFallbackForPartitioned() throws Exception { // the file is table-level shaped: no blob carries a partition name Assert.assertTrue(testTables.loadTable(identifier).statisticsFiles().stream() .flatMap(statsFile -> statsFile.blobMetadata().stream()) - .noneMatch(blob -> blob.properties().containsKey(IcebergColStatsWriter.PARTITION_FIELD))); + .noneMatch(blob -> blob.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP))); shell.executeStatement("INSERT INTO " + identifier + " VALUES (5, date '2024-05-05')"); // the increment merged into the table-level statistics @@ -2361,7 +2361,7 @@ public void testACarriedEntryOfARenamedColumnCannotAnswerForTheNewName() throws @Test public void testTheFoldLeavesOutAColumnAPartitionDidNotState() throws Exception { // a rename moves no snapshot, so the partitions this gather did not write stay named as they - // were. Folding what they hold under the new name would state the table from one partition, + // were. Folding what they hold under the new name would aggregate the full table from one, // so the fold leaves such a column out and the whole-table question is declined assumeParquetHiveCatalogIceberg(); @@ -2418,10 +2418,10 @@ public void theFoldDoesNotAnswerForAPartitionItNeverDescribed() throws Exception } @Test - public void aWholeTableReadTakesNoPerPartitionFile() throws Exception { - // statistics are served at the granularity the session keeps them at. A file holding - // partitions states them, and what it folds from them states the table only while it holds - // every one - so a whole-table read passes it by rather than answer from part of a table + public void aWholeTableReadTakesTheFullTableAggrOfAPerPartitionGather() throws Exception { + // a file holding partitions states them; what it aggregates from them answers a whole-table + // read too, but only while they aggregate the full table - which a gather over every partition + // marks, whatever granularity a later read is kept at assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "orders_two_granularities"); @@ -2440,8 +2440,9 @@ public void aWholeTableReadTakesNoPerPartitionFile() throws Exception { shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); - Assert.assertTrue("a whole-table read is not answered from the partitions of a later gather", + Assert.assertFalse("a whole-table read is answered from the full-table aggregate a full gather leaves", storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + checkColStatMinMaxValue(identifier.name(), "id", 1, 9); // and the partitions still answer for themselves, at the granularity they were kept at HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); @@ -2454,7 +2455,7 @@ public void aWholeTableReadTakesNoPerPartitionFile() throws Exception { } @Test - public void aPartitionScopedGatherWithNothingToCarryStatesNoTable() { + public void aPartitionScopedGatherWithNothingToCarryHasNoFullTableAggr() { // it measured one partition and had no stored file to carry the others from, so the file holds // that partition alone. A fold of it would read as the table's, and answer for rows it never saw assumeParquetHiveCatalogIceberg(); @@ -2469,10 +2470,216 @@ public void aPartitionScopedGatherWithNothingToCarryStatesNoTable() { // read at the granularity the table is configured for by default HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); - Assert.assertTrue("the partition it measured does not state the table", + Assert.assertTrue("the partition it measured is no full-table aggregate", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + } + + @Test + public void testATableLevelReadServesTheFullTableAggrOfAPartitionLevelGather() { + // a gather over every partition aggregates the full table on its file; a session reading at table + // level takes those aggregates rather than finding nothing at its own granularity + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_lenient_read"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (900, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + checkColStatMinMaxValue(identifier.name(), "id", 1, 900); + } + + @Test + public void testANewerFullPartitionGatherAnswersATableReadOverAnOlderTableLevelFile() { + // s1 aggregates the full table at table level; a write moves the snapshot; s2 is a full gather at + // partition level that also aggregates the full table. The table-level read walks back from the + // current snapshot, takes the newer s2, and answers from its aggregates - not stale s1 + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_two_snapshots"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + // s1: whole-table gather + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + checkColStatMinMaxValue(identifier.name(), "id", 1, 7); + + // a write moves the snapshot, then s2: a full gather at partition level, over every partition + shell.executeStatement("INSERT INTO " + identifier + " VALUES (900, 'c')"); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // a table-level read takes the newer s2 and answers from its aggregates - the 900 proves it + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + checkColStatMinMaxValue(identifier.name(), "id", 1, 900); + } + + @Test + public void testAFullTableAggrIsStillRefusedOnceANewPartitionArrives() { + // the full-table aggregate marks what a write covered, never overrides freshness: a full gather + // of a one-partition table leaves one, but an insert adding a partition moves the snapshot, + // and the whole-table read stops at that change rather than serve the now-incomplete aggregates + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_full_aggr_then_grows"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'a')"); + // a full gather of the one partition aggregates the full table + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + checkColStatMinMaxValue(identifier.name(), "id", 1, 7); + + // a new partition arrives with no re-analyze: the stored aggregates no longer describe the table + shell.executeStatement("INSERT INTO " + identifier + " VALUES (900, 'b')"); + Assert.assertTrue("the full-table aggregate is refused once a partition it never saw exists", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + } + + @Test + public void testAMergeKeepsTheFullTableAggrOfTheFileItCarriedFrom() { + // an increment merged into a file aggregating the full table leaves one that still does; the + // marker rides the merge, not the granularity of the write that happened to refresh it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_lenient_merge"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET " + + "TBLPROPERTIES ('external.table.purge'='true')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (900, 'b')"); + // a whole-table gather aggregates the full table, then a partition-scoped gather merges into it + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + checkColStatMinMaxValue(identifier.name(), "id", 1, 900); + } + + @Test + public void aMergeKeepsNoFullTableAggrOnceAPartitionItNeverMeasuredArrived() { + // the file it carried from aggregated the full table, but a partition arrived after it and + // this gather measured another: the new one is neither measured nor carried, so the merged + // file holds a strict subset. Inheriting the mark would answer the whole table from it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_merge_misses_new_partition"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + // a plain insert stores no statistics of its own, and adds a partition the stored file never saw + shell.executeStatement("INSERT INTO " + identifier + " VALUES (900, 'c')"); + // merges: measures p=a, carries p=b, leaves p=c held by neither. It registers at the very + // snapshot the insert committed, so no later walk can catch it + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + Assert.assertTrue("a merge that never measured the new partition is no full-table aggregate", + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); + } + + @Test + public void aMergeKeepsNoFullTableAggrOnceAPartitionItNeverMeasuredChanged() { + // the same without a new partition: a write into one this gather did not measure leaves it + // stale, so the merge drops it rather than carrying it, and the file is short of a partition + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_merge_drops_stale_partition"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 'a'), (7, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (900, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + Assert.assertTrue("a merge that dropped the changed partition is no full-table aggregate", storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")).isEmpty()); } + @Test + public void aMergeDoesNotFoldADeadFieldIntoItsNamesake() { + // a column dropped and added back keeps its name and takes a new field id. A merge folds the + // partitions it carries into the table's entries, and folding by name would let the dead + // field's numbers answer for the live one - under the live field id, where no read can see it + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_namesake_field"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, amount bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 100, 'a'), (2, 200, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + // the dropped column's entries stay in the stored blobs of p=a and p=b, under its old field + shell.executeStatement("ALTER TABLE " + identifier + " REPLACE COLUMNS (id bigint, p string)"); + shell.executeStatement("ALTER TABLE " + identifier + " ADD COLUMNS (amount bigint)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, 'c', 5)"); + // merges: measures p=c, carries p=a and p=b, whose amount entries are of the dead field + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='c') COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + // p=a and p=b hold NULL for the live amount, so no partition but p=c states it and the table + // states nothing for it at all: the dead field's entries folded in would make it 5..200 + List stats = + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("amount")); + Assert.assertTrue("the dead field's numbers do not answer for its namesake, stats were " + stats, + stats.isEmpty()); + // and the merge still answers for a column every partition does state + List ids = + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("id")); + Assert.assertEquals("a column every partition states still answers", 3L, + ids.get(0).getStatsData().getLongStats().getHighValue()); + } + + @Test + public void aMergeDoesNotFoldADeadFieldOutOfACarriedBlob() { + // the same hazard by the other door: re-measuring a partition the stored file already held + // makes the merge rebuild its aggregate by decoding every carried blob rather than seeding + // from the stored one, so what guards it is the field filter inside the blob, not the one + // over the blobs + assumeParquetHiveCatalogIceberg(); + + TableIdentifier identifier = TableIdentifier.of("default", "orders_namesake_carried"); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, true); + shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " (id bigint, amount bigint, p string) " + + "PARTITIONED BY SPEC (p) STORED BY ICEBERG STORED AS PARQUET TBLPROPERTIES ('format-version'='2')"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 100, 'a'), (2, 200, 'b')"); + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + + shell.executeStatement("ALTER TABLE " + identifier + " REPLACE COLUMNS (id bigint, p string)"); + shell.executeStatement("ALTER TABLE " + identifier + " ADD COLUMNS (amount bigint)"); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (3, 'a', 7)"); + // measures p=a, which the file already describes: p=b is carried and decoded, not seeded from + shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p='a') COMPUTE STATISTICS FOR COLUMNS"); + + HiveConf.setBoolVar(shell.getHiveConf(), HiveConf.ConfVars.HIVE_ICEBERG_STATS_COLLECT_PART_LEVEL, false); + // p=b holds NULL for the live amount, so the table states nothing for it; the 200 the dead + // field left in p=b's carried blob would make it 7..200 + List stats = + storageHandler().getColStatistics(hmsTable(identifier), ImmutableList.of("amount")); + Assert.assertTrue("a carried blob's dead entry does not answer for its namesake, stats were " + stats, + stats.isEmpty()); + } + @Test public void aStalePartitionIsNotAnsweredForFromWhatWasFoldedOverIt() throws Exception { // the fold held every partition when it was written; a write since leaves one no longer @@ -2580,7 +2787,7 @@ private Set colStatsPartitions(Table icebergTable) { .withFooterSize(statsFile.fileFooterSizeInBytes()) .build()) { return reader.fileMetadata().blobs().stream() - .map(metadata -> metadata.properties().get(IcebergColStatsWriter.PARTITION_FIELD)) + .map(metadata -> metadata.properties().get(IcebergColStatsWriter.PARTITION_PROP)) .filter(Objects::nonNull) .collect(Collectors.toSet()); } catch (IOException e) { @@ -2764,13 +2971,13 @@ public void testTheTableMetadataRegistersOnePartitionEntryNamingEveryFieldAndThe Table icebergTable = testTables.loadTable(identifier); var registered = currentColStatsFile(icebergTable).blobMetadata(); var partitionEntries = registered.stream() - .filter(blob -> blob.properties().containsKey(IcebergColStatsWriter.PARTITION_FIELD)) + .filter(blob -> blob.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)) .toList(); Assert.assertEquals("one entry stands for the partitions", 1, partitionEntries.size()); var partitionEntry = partitionEntries.get(0); Assert.assertFalse("and it names the fields", partitionEntry.fields().isEmpty()); Assert.assertEquals("and states how many partitions the file describes", - "3", partitionEntry.properties().get(IcebergColStatsWriter.NUM_PARTITIONS_FIELD)); + "3", partitionEntry.properties().get(IcebergColStatsWriter.NUM_PARTITIONS_PROP)); // the footer still names every partition, and both reads still answer Assert.assertEquals(Set.of("p=a", "p=b", "p=c"), colStatsPartitions(icebergTable)); checkColStatMinMaxValue(identifier.name(), "id", 1, 7); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java index a57b90984c6a..62c6eebc1abf 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java @@ -194,7 +194,7 @@ private static RecordingStream layOut(List blobs, List offsets, Li System.arraycopy(blobs.get(i), 0, file, offsets.get(i).intValue(), blobs.get(i).length); meta.add(new BlobMetadata(IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1, List.of(1), 1L, 1L, offsets.get(i), blobs.get(i).length, null, - Map.of(IcebergColStatsWriter.PARTITION_FIELD, "p=" + i))); + Map.of(IcebergColStatsWriter.PARTITION_PROP, "p=" + i))); } return new RecordingStream(file); } From 1d99435391777fe000a04b4a5cf7f726f640bd5b Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 10 Sep 2026 20:26:01 +0300 Subject: [PATCH 04/13] HIVE-29834: Answer IS NULL from the row count of the scan's own snapshot The reduce rule folds IS NULL and IS NOT NULL when a column's null count equals the row count. The null count comes from the snapshot the scan reads - a branch or as-of, resolved through the handler - so the row count must come from the same snapshot, not the current table's metastore parameters. A branch null count read against the main row count folds IS NOT NULL to false and drops the branch's non-null rows. --- .../mr/hive/TestHiveIcebergStatistics.java | 28 +++++++++++++++++++ .../HiveReduceExpressionsWithStatsRule.java | 5 +--- .../hadoop/hive/ql/stats/StatsUtils.java | 24 ++++++++++------ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index 6e00fadd220c..f58dedb0fda2 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -2923,6 +2923,34 @@ public void testAFilterIsNotFoldedFromAPartitionSubsetsRange() { Assert.assertFalse("a probe beyond every partition's range is folded away", probes); } + @Test + public void testIsNotNullIsNotFoldedFromABranchNullCountAgainstTheMainRowCount() { + // the null count comes from the branch the scan reads; the row count must come from the same + // branch. Main holds two rows and its stats are fresh, so its count is two; the branch column + // gains exactly two nulls among five rows. A branch null count read against the main row count + // would fold IS NOT NULL to false and drop the branch's three non-null rows + assumeParquetHiveCatalogIceberg(); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, false); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_OPTIMIZE_REDUCE_WITH_STATS.varname, true); + TableIdentifier identifier = TableIdentifier.of("default", "orders_branch_isnull"); + Schema schema = new Schema( + NestedField.optional(1, "id", Types.LongType.get()), + NestedField.optional(2, "c", Types.LongType.get())); + testTables.createTable(shell, identifier.name(), schema, PartitionSpec.unpartitioned(), + fileFormat, ImmutableList.of(), 2); + shell.executeStatement("INSERT INTO " + identifier + " VALUES (1, 10), (2, 20)"); + // main's row count is fresh at two + shell.executeStatement("ANALYZE TABLE " + identifier + " COMPUTE STATISTICS FOR COLUMNS"); + shell.executeStatement("ALTER TABLE " + identifier + " CREATE BRANCH b1"); + // db-qualified three-part name so the branch resolves as a table, not a database + shell.executeStatement("INSERT INTO " + identifier + ".branch_b1 VALUES (3, NULL), (4, NULL), (5, 50)"); + shell.executeStatement("ANALYZE TABLE " + identifier + ".branch_b1 COMPUTE STATISTICS FOR COLUMNS"); + + List rows = + shell.executeStatement("SELECT id FROM " + identifier + ".branch_b1 WHERE c IS NOT NULL"); + Assert.assertEquals("the branch's non-null rows are not folded away", 3, rows.size()); + } + @Test public void testARecreatedColumnDoesNotAnswerFromItsNamesakesPartitionEntry() throws Exception { // a full ask decodes each partition blob whole; the entries still answer by field id, so what diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java index 8b2c4f538e5f..66d87fdcc9dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java @@ -318,10 +318,7 @@ private Long extractRowCount(RexInputRef ref) { if (columnOrigin != null) { RelOptHiveTable table = (RelOptHiveTable) columnOrigin.getOriginTable(); if (table != null) { - if (StatsUtils.areBasicStatsUptoDateForQueryAnswering(table.getHiveTableMD(), - table.getHiveTableMD().getParameters())) { - return StatsUtils.getNumRows(table.getHiveTableMD()); - } + return StatsUtils.getRowCnt(table.getHiveTableMD()); } } return null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 380542802737..fde920352147 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -1791,15 +1791,6 @@ private static long getNDVFor(ExprNodeGenericFuncDesc engfd, long numRows, Stati return Collections.min(Lists.newArrayList(countDistincts, udfNDV, numRows)); } - /** - * Get number of rows of a give table - * @return number of rows - */ - @Deprecated - public static long getNumRows(Table table) { - return getBasicStatForTable(table, StatsSetupConst.ROW_COUNT); - } - /** * Get total size of a give table * @return total size @@ -2069,6 +2060,21 @@ public static boolean areBasicStatsUptoDateForQueryAnswering(Table table, Map Date: Thu, 10 Sep 2026 20:54:43 +0300 Subject: [PATCH 05/13] HIVE-29834: Read basic statistics of a versioned scan from its snapshot The metastore holds one unversioned set of parameters describing the current table, so a branch, a tag or a point in time cannot be answered from it. getBasicStatistics guarded only the named-ref case and read the current parameters for an as-of scan; guarding on the whole qualifier, as the row count and the column-stats freshness already do, sends every versioned scan to its own snapshot summary. --- .../apache/iceberg/mr/hive/HiveIcebergStorageHandler.java | 5 +++-- .../apache/iceberg/mr/hive/TestHiveIcebergStatistics.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index f7b72115e7ac..b047498665f3 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -511,8 +511,9 @@ private Map getBasicStatistics(org.apache.hadoop.hive.ql.metadat stats = emptyStatsMap(); } else if (!HiveMetaHook.ICEBERG.equals(getStatsSource()) && !quickStats && - hmsTable.getSnapshotRef() == null) { - // the metastore parameters describe the table, not a branch: use the snapshot's counters + hmsTable.getQualifier().isEmpty()) { + // the metastore holds one unversioned set of parameters describing the current table, so a + // branch, a tag or a point in time is not answered from it - only a plain scan is stats = hmsTable.getParameters(); } else { diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index f58dedb0fda2..d5f0777a35c1 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -2159,6 +2159,8 @@ public void testTimeTravelIsNeverAnsweredFromTheMetastoreRow() { asOf.setAsOfVersion(String.valueOf(oldSnapshot)); Assert.assertEquals("the scan reads two rows, however the table now holds five", Long.valueOf(2), storageHandler().getRowCount(asOf)); + Assert.assertEquals("and the basic statistics count the point in time, not the current row", + "2", storageHandler().getBasicStatistics(asOf).get(StatsSetupConst.ROW_COUNT)); Assert.assertFalse("the metastore's row must not answer for a point in time", storageHandler().areColumnStatsUptoDate(asOf, List.of("id"))); } finally { From 09090611ece52a80940f831ab13a0559f1d4842f Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 15 Sep 2026 19:38:44 +0300 Subject: [PATCH 06/13] HIVE-29834: Name a stored column-stats entry from its field id - the schema resolves the id to the column's current name, lower cased - an entry whose field the schema dropped is left out, not renamed - a read resolves the asked columns to field ids once, not per entry --- .../mr/hive/HiveIcebergStorageHandler.java | 7 +- .../apache/iceberg/mr/hive/SchemaUtils.java | 11 ++ .../mr/hive/stats/IcebergColStatsCodec.java | 16 +- .../mr/hive/stats/IcebergColStatsReader.java | 139 ++++++++---------- .../mr/hive/stats/IcebergColStatsWriter.java | 25 ++-- .../mr/hive/TestHiveIcebergStatistics.java | 24 +-- .../hive/stats/TestIcebergColStatsFormat.java | 42 ++++-- 7 files changed, 140 insertions(+), 124 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index b047498665f3..ff3136aca10b 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -826,15 +826,10 @@ private AggrStats aggrColStats(org.apache.hadoop.hive.ql.metadata.Table hmsTable Set columns = Sets.newHashSet(colNames); Map> statsByPart = IcebergColStatsReader.readPart(table, statsFile, - partition -> partitions.contains(partition) && upToDate.test(partition), - // an ask as wide as the schema narrows nothing, so it reads each blob whole - columns.size() == table.schema().columns().size() ? null : columns, conf); + partition -> partitions.contains(partition) && upToDate.test(partition), columns, conf); List partStats = Lists.newArrayList(); statsByPart.forEach((partition, statsObjs) -> { - // a whole-blob read decodes every stored entry, and a carried blob may hold entries under - // names the schema no longer has: only the asked columns may count toward the ask - statsObjs.removeIf(obj -> !columns.contains(obj.getColName())); // the metastore counts a partition as found only when it has every column asked about if (statsObjs.size() == colNames.size()) { ColumnStatisticsDesc statsDesc = diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java index 8444da80510f..852a7850550a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java @@ -20,6 +20,7 @@ package org.apache.iceberg.mr.hive; import java.util.List; +import java.util.Locale; import org.apache.hadoop.hive.ql.parse.TransformSpec; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -33,6 +34,16 @@ public class SchemaUtils { private SchemaUtils() { } + /** + * Returns the name the schema gives the field now, or null where it no longer has it. Lower + * case, as Hive keeps a column name wherever it keeps one, while an Iceberg schema keeps + * whatever case the table was created with. + */ + public static String getColumnName(Schema schema, int fieldId) { + String name = schema.findColumnName(fieldId); + return name == null ? null : name.toLowerCase(Locale.ROOT); + } + public static UnboundTerm toTerm(TransformSpec spec) { if (spec == null) { return null; diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java index ab2fc2c37a22..417e5ab7cbf5 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java @@ -94,33 +94,31 @@ static byte[] encodeBlob(List parts, List fieldIds) throws IOEx /** * The entries the given places name, the rest skipped rather than copied. A scan of a wide table - * asks about a few of its columns, and an entry it does not want costs a read nothing beyond the + * asks about a few of its columns, and an entry it does not need costs a read nothing beyond the * length it steps over. */ - static List decodeBlob(ByteBuffer buf, IntPredicate wanted) { + static List decodeBlob(ByteBuffer buf, IntPredicate needed) { ByteBuffer data = buf.duplicate().order(ByteOrder.BIG_ENDIAN); if (data.remaining() < Integer.BYTES || data.getInt() != BLOB_VERSION) { return List.of(); } int count = data.getInt(); - List parts = Lists.newArrayListWithCapacity(count); + List entries = Lists.newArrayListWithCapacity(count); for (int i = 0; i < count; i++) { int fieldId = data.getInt(); int length = data.getInt(); - if (wanted.test(fieldId)) { + if (needed.test(fieldId)) { byte[] part = new byte[length]; data.get(part); - parts.add(part); + entries.add(new FieldEntry(fieldId, part)); } else { data.position(data.position() + length); } } - return parts; + return entries; } - /** What the blob holds, or nothing where it was written in a shape this does not know. */ - static List decodeBlob(ByteBuffer buf) { - return decodeBlob(buf, fieldId -> true); + record FieldEntry(int fieldId, byte[] bytes) { } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java index 7fd83ae8a919..22a35cf2f489 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java @@ -36,12 +36,14 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.util.functional.FutureIO; +import org.apache.iceberg.Schema; import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.io.DelegatingInputStream; import org.apache.iceberg.io.IOUtil; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.SeekableInputStream; +import org.apache.iceberg.mr.hive.SchemaUtils; import org.apache.iceberg.puffin.BlobMetadata; import org.apache.iceberg.puffin.Puffin; import org.apache.iceberg.puffin.PuffinReader; @@ -112,8 +114,8 @@ static List read(Table table, StatisticsFile statsFile, static List readOrThrow(Table table, StatisticsFile statsFile, Collection columns, boolean withVectors) throws IOException { - Predicate holdsNeededColumn = - columns != null ? blobsForColumns(table, columns) : blob -> true; + Schema schema = table.schema(); + IntPredicate needed = neededFields(schema, columns); List entries = Lists.newArrayList(); String statsPath = statsFile.path(); @@ -122,20 +124,15 @@ static List readOrThrow(Table table, StatisticsFile statsFi .withFooterSize(statsFile.fileFooterSizeInBytes()) .build()) { - IntPredicate liveFields = liveFieldsOf(table); - List blobMetadata = reader.fileMetadata().blobs().stream() .filter(IcebergColStatsReader::holdsColStats) - .filter(blob -> liveFields.test(blob.inputFields().getFirst())) - .filter(holdsNeededColumn) + .filter(blob -> needed.test(blob.inputFields().getFirst())) .toList(); LOG.info("Using column stats from: {}", statsPath); - for (Pair blob : reader.readAll(blobMetadata)) { - byte[] raw = ByteBuffers.toByteArray(blob.second()); - entries.add(decodeTableEntry(raw, blob.first().type(), withVectors)); - } + entries.addAll( + readTableEntries(reader, blobMetadata, withVectors, schema)); } return entries; } @@ -150,7 +147,24 @@ private static boolean holdsColStats(BlobMetadata blob) { IcebergColStatsWriter.LEGACY_COL_STATS_BLOB.equals(blob.type()); } - /** An entry as the blob that names it was written: a Thrift struct, or a serialized Java object. */ + /** + * The entries the given blobs hold, each under the name the schema gives its field now rather + * than the one it was stored under. The caller has already left behind the blobs of fields the + * schema no longer has, which have no name to take. + */ + static List readTableEntries(PuffinReader reader, List blobs, + boolean withVectors, Schema schema) { + List entries = Lists.newArrayListWithCapacity(blobs.size()); + for (Pair blob : reader.readAll(blobs)) { + ColumnStatisticsObj statsObj = decodeTableEntry( + ByteBuffers.toByteArray(blob.second()), blob.first().type(), withVectors); + statsObj.setColName( + SchemaUtils.getColumnName(schema, blob.first().inputFields().getFirst())); + entries.add(statsObj); + } + return entries; + } + private static ColumnStatisticsObj decodeTableEntry(byte[] raw, String blobType, boolean withVectors) { if (IcebergColStatsWriter.HIVE_COL_STATS_BLOB_V1.equals(blobType)) { return IcebergColStatsCodec.decodeEntry(raw, withVectors); @@ -176,11 +190,25 @@ private static boolean fetchVectors(Configuration conf) { return MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); } - /** The blobs naming any of the asked columns, by the name the table's schema gives the field now. */ - private static Predicate blobsForColumns(Table table, Collection columns) { - return metadata -> metadata.inputFields().stream() - .map(fieldId -> table.schema().findColumnName(fieldId)) - .anyMatch(columns::contains); + /** + * The fields the needed columns are, by the name the schema gives each now; a null column set + * asks for all of them. A field the schema no longer has is none of them: the name its entry + * was stored under may since have moved to another column. + */ + static IntPredicate neededFields(Schema schema, Collection columns) { + if (columns == null) { + return fieldId -> schema.findField(fieldId) != null; + } + // the schema matches the asked name whatever case it keeps its own in, so an entry is chosen + // by the field its column is now, never by the name it was stored under + Set fieldIds = Sets.newHashSetWithExpectedSize(columns.size()); + for (String column : columns) { + Types.NestedField field = schema.caseInsensitiveFindField(column); + if (field != null) { + fieldIds.add(field.fieldId()); + } + } + return fieldIds::contains; } /** @@ -225,26 +253,21 @@ private static List readAggr(Table table, StatisticsFile st if (described.isEmpty() || !described.equals(asked) || !described.stream().allMatch(upToDate)) { return null; } - // a dead field resolves to no name of its own, so blobsForColumns leaves its entry out - Predicate holdsNeededColumn = blobsForColumns(table, columns); + Schema schema = table.schema(); + IntPredicate needed = neededFields(schema, columns); List blobMetadata = reader.fileMetadata().blobs().stream() .filter(IcebergColStatsReader::holdsColStats) - .filter(holdsNeededColumn) + .filter(blob -> needed.test(blob.inputFields().getFirst())) .toList(); - for (Pair blob : reader.readAll(blobMetadata)) { - byte[] raw = ByteBuffers.toByteArray(blob.second()); - aggregated.add(decodeTableEntry(raw, blob.first().type(), withVectors)); - } + aggregated.addAll( + readTableEntries(reader, blobMetadata, withVectors, schema)); } catch (Exception e) { // serving no stats degrades the planner to estimates - never wrong LOG.warn("Unable to read column stats: {}", e.getMessage()); return null; } - // a rename keeps the field, so a blob written before it still names the field under the old - // column name: it answers for the field asked about but not for the column, and is left out - aggregated.removeIf(statsObj -> !columns.contains(statsObj.getColName())); return aggregated.size() == colNames.size() ? aggregated : null; } @@ -283,9 +306,11 @@ public static Map> readPart(Table table, Stati // than the seek it saves. Reading each on its own is what makes a scan of many partitions // expensive. if (!blobs.isEmpty()) { + Schema schema = table.schema(); InputFile file = table.io().newInputFile(statsFile.path(), statsFile.fileSizeInBytes()); try (SeekableInputStream in = file.newStream()) { - readBlobs(in, blobs, columns, withVectors, result, fieldsOf(table, columns)); + // the asked columns are the same fields in every blob, so they are resolved once here + readBlobs(in, blobs, neededFields(schema, columns), withVectors, result, schema); } } } catch (Exception e) { @@ -305,8 +330,8 @@ public static Map> readPart(Table table, Stati * per round trip, which a file holding a blob per partition cannot afford. It leaves in favor * of Iceberg's reader once that one coalesces runs and takes them in one vectored call. */ - static void readBlobs(SeekableInputStream in, List blobs, Set columns, - boolean withVectors, Map> result, IntPredicate fields) + static void readBlobs(SeekableInputStream in, List blobs, IntPredicate needed, + boolean withVectors, Map> result, Schema schema) throws IOException { List ordered = blobs.stream() .sorted(Comparator.comparingLong(BlobMetadata::offset)) @@ -346,7 +371,7 @@ static void readBlobs(SeekableInputStream in, List blobs, Set readRanges(SeekableInputStream in, List decodePartBlob(ByteBuffer blob, Set columns, boolean withVectors) { - return decodePartBlob(blob, columns, withVectors, null); - } - - /** - * The asked columns of a partition. Every entry names the field it is for, so the ones a scan did - * not ask about are stepped over rather than decoded - and a blob a merge carried from another - * gather needs hold neither the same columns nor the same order for that to hold. - */ - static List decodePartBlob(ByteBuffer blob, Set columns, - boolean withVectors, IntPredicate fields) { - List stored = fields == null ? - IcebergColStatsCodec.decodeBlob(blob) : IcebergColStatsCodec.decodeBlob(blob, fields); + static List decodePartBlob(ByteBuffer blob, IntPredicate needed, + boolean withVectors, Schema schema) { + List stored = IcebergColStatsCodec.decodeBlob(blob, needed); List entries = Lists.newArrayListWithCapacity(stored.size()); - for (byte[] entry : stored) { - ColumnStatisticsObj statsObj = IcebergColStatsCodec.decodeEntry(entry, withVectors); - if (columns == null || columns.contains(statsObj.getColName())) { - entries.add(statsObj); - } + for (IcebergColStatsCodec.FieldEntry entry : stored) { + ColumnStatisticsObj statsObj = IcebergColStatsCodec.decodeEntry(entry.bytes(), withVectors); + statsObj.setColName(SchemaUtils.getColumnName(schema, entry.fieldId())); + entries.add(statsObj); } return entries; } @@ -452,27 +466,4 @@ private static Optional hadoopStream(SeekableInputStream in) return Optional.empty(); } - /** - * Whether the field is one the schema still has. An entry answers by its field id, never by its - * name alone: a column dropped and added back keeps the name and takes a new field, so an entry - * the dropped one left behind is stepped over even though a column of that name exists. - */ - static IntPredicate liveFieldsOf(Table table) { - return id -> table.schema().findField(id) != null; - } - - /** The fields the asked columns are, so a read can step over the entries of the rest. */ - private static IntPredicate fieldsOf(Table table, Set columns) { - if (columns == null) { - return liveFieldsOf(table); - } - Set fields = Sets.newHashSet(); - for (String column : columns) { - Types.NestedField field = table.schema().caseInsensitiveFindField(column); - if (field != null) { - fields.add(field.fieldId()); - } - } - return fields::contains; - } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java index 7b7629a2fc99..1a4838fc875a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java @@ -54,7 +54,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.ByteBuffers; import org.apache.iceberg.util.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -319,12 +318,15 @@ private static void carryForward(Table tbl, Snapshot snapshot, PuffinWriter writ // instead of being rebuilt by decoding every carried blob boolean seedFromStored = Sets.intersection(written, storedPartitions).isEmpty() && carried.size() == storedPartitions.size(); - // by field id, not name: a column dropped and added back keeps the name and takes a new - // field, and folding the dead field's entry in would answer for the live one - IntPredicate liveFields = IcebergColStatsReader.liveFieldsOf(tbl); + // entries fold into a map keyed by column name, so each is taken under the name the schema + // gives its field now: a rename moves a name to another column, and a column dropped and + // added back keeps its name while taking a new field + Schema schema = tbl.schema(); + // a carry takes every column the blob holds, minus the fields the schema has since dropped + IntPredicate liveFields = IcebergColStatsReader.neededFields(schema, null); if (seedFromStored) { - aggregate.seedFrom(reader, carried.size(), liveFields); + aggregate.seedFrom(reader, carried.size(), schema); } for (Pair blob : reader.readAll(carried)) { ByteBuffer carriedBytes = blob.second(); @@ -333,7 +335,7 @@ private static void carryForward(Table tbl, Snapshot snapshot, PuffinWriter writ try { if (!seedFromStored) { aggregate.addPartition( - IcebergColStatsReader.decodePartBlob(carriedBytes, null, true, liveFields)); + IcebergColStatsReader.decodePartBlob(carriedBytes, liveFields, true, schema)); } } catch (InvalidObjectException e) { throw new IOException(e); @@ -376,17 +378,14 @@ private void addPartition(List statsObjs) throws InvalidObj * aggregating its partitions again would reach: an entry is written only when every partition * states the column, and what suppressed it then is carried unchanged now. */ - private void seedFrom(PuffinReader reader, int carriedPartitions, IntPredicate liveFields) + private void seedFrom(PuffinReader reader, int carriedPartitions, Schema schema) throws IOException { List aggregateBlobs = reader.fileMetadata().blobs().stream() .filter(metadata -> HIVE_COL_STATS_BLOB_V1.equals(metadata.type())) - .filter(metadata -> liveFields.test(metadata.inputFields().getFirst())) + .filter(metadata -> schema.findField(metadata.inputFields().getFirst()) != null) .toList(); - List entries = Lists.newArrayList(); - for (Pair blob : reader.readAll(aggregateBlobs)) { - entries.add(IcebergColStatsCodec.decodeEntry( - ByteBuffers.toByteArray(blob.second()), true)); - } + List entries = + IcebergColStatsReader.readTableEntries(reader, aggregateBlobs, true, schema); try { addEntries(entries, carriedPartitions); } catch (InvalidObjectException e) { diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index d5f0777a35c1..2a7fa9869712 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -1720,6 +1720,11 @@ public void testAggrColStatsForCaseSensitivePartitionField() throws Exception { List partNames = ImmutableList.of("eventDate=2023-03-04", "eventDate=2024-06-01"); Assert.assertEquals(partNames, colStatsPartNames(identifier)); assertAggrColStatsRange(identifier, "id", partNames, 1, 2); + // an entry is stored, and a column asked about, under the name Hive lower cases it to, while + // the schema keeps the case the table was created with: a stored entry answers by the former + Assert.assertEquals("a mixed-case column answers under the name Hive asks by", 2, + storageHandler().getAggrColStatsFor(hmsTable(identifier), ImmutableList.of("eventdate"), + partNames).getPartsFound()); } @Test @@ -2334,10 +2339,10 @@ public void testAggrColStatsCountsOnlyPartitionsCarryingEveryColumnAsked() throw } @Test - public void testACarriedEntryOfARenamedColumnCannotAnswerForTheNewName() throws Exception { + public void testACarriedEntryOfARenamedColumnAnswersForTheNewName() throws Exception { // ANALYZE full table -> rename a column, which moves no snapshot -> ANALYZE one partition. - // The other partition's entry is carried under the old name and holds as many columns as - // the ask, so it must be refused by identity, not by count. + // A rename moves a name, not a field, and the rows it was measured from never moved: the + // carried entry is the renamed column's own, and answers for it. assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "orders_renamed_column"); @@ -2357,14 +2362,15 @@ public void testACarriedEntryOfARenamedColumnCannotAnswerForTheNewName() throws AggrStats aggrStats = storageHandler().getAggrColStatsFor( hmsTable(identifier), ImmutableList.of("id", "val2", "p"), partNames); - Assert.assertEquals("the carried entry holds no column of the asked name", 1, aggrStats.getPartsFound()); + Assert.assertEquals("the carried entry answers for the field it was measured from", 2, + aggrStats.getPartsFound()); } @Test public void testTheFoldLeavesOutAColumnAPartitionDidNotState() throws Exception { - // a rename moves no snapshot, so the partitions this gather did not write stay named as they - // were. Folding what they hold under the new name would aggregate the full table from one, - // so the fold leaves such a column out and the whole-table question is declined + // a column added later takes a new field, which the partitions this gather did not write hold + // no entry for. Folding it would aggregate the full table from one partition, so the fold + // leaves such a column out and the whole-table question is declined assumeParquetHiveCatalogIceberg(); TableIdentifier identifier = TableIdentifier.of("default", "orders_folded_rename"); @@ -2379,10 +2385,10 @@ public void testTheFoldLeavesOutAColumnAPartitionDidNotState() throws Exception storageHandler().getAggrColStatsFor(hmsTable(identifier), ImmutableList.of("val"), everyPartition).getPartsFound()); - shell.executeStatement("ALTER TABLE " + identifier + " CHANGE COLUMN val val2 bigint"); + shell.executeStatement("ALTER TABLE " + identifier + " ADD COLUMNS (val2 bigint)"); shell.executeStatement("ANALYZE TABLE " + identifier + " PARTITION (p = 'b') COMPUTE STATISTICS FOR COLUMNS"); - Assert.assertEquals("only the partition just written states the new name, so the fold leaves it out", + Assert.assertEquals("only the partition just written states the new column, so the fold leaves it out", 1, storageHandler().getAggrColStatsFor(hmsTable(identifier), ImmutableList.of("val2"), everyPartition).getPartsFound()); Assert.assertEquals("a column every partition still states is folded as before", 2, diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java index 62c6eebc1abf..384c8139ee94 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java @@ -24,13 +24,17 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.function.IntPredicate; +import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.iceberg.Schema; import org.apache.iceberg.puffin.BlobMetadata; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Test; @@ -49,7 +53,7 @@ public void nothingInTheFrameIsPacked() throws Exception { ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(List.of(statsObj), ids(1)); Assert.assertTrue("the sketch takes its own size in the blob", blob.remaining() > sketch.length); - Assert.assertEquals(List.of(statsObj), IcebergColStatsReader.decodePartBlob(blob, null, true)); + Assert.assertEquals(List.of(statsObj), IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, true, SCHEMA)); } @Test @@ -62,14 +66,14 @@ public void aVectorComesBackOnlyWhenAskedForAndAHistogramAlways() throws Excepti ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(List.of(statsObj), ids(1)); ColumnStatisticsObj asked = - IcebergColStatsReader.decodePartBlob(blob, null, true).getFirst(); + IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, true, SCHEMA).getFirst(); Assert.assertArrayEquals("the vector is put back where it was taken from", new byte[] {1, 2, 3, 4}, asked.getStatsData().getLongStats().getBitVectors()); Assert.assertArrayEquals("and so is the histogram", new byte[] {5, 6, 7}, asked.getStatsData().getLongStats().getHistogram()); ColumnStatisticsObj unasked = - IcebergColStatsReader.decodePartBlob(blob, null, false).getFirst(); + IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, false, SCHEMA).getFirst(); Assert.assertFalse("a read that did not ask for the vector does not get it", unasked.getStatsData().getLongStats().isSetBitVectors()); Assert.assertArrayEquals("the histogram comes back whatever was asked", @@ -84,7 +88,7 @@ public void aFrameFromAVersionThisReaderDoesNotKnowReadsAsAbsent() throws Except ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(columns(2), ids(2)); blob.putInt(0, IcebergColStatsCodec.BLOB_VERSION + 1); - Assert.assertTrue(IcebergColStatsReader.decodePartBlob(blob, null, true).isEmpty()); + Assert.assertTrue(IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, true, SCHEMA).isEmpty()); } @Test @@ -92,8 +96,8 @@ public void aWideFrameStillYieldsOnlyTheAskedColumns() throws Exception { // 3000 columns push the header well past anything a single small read would hold ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(columns(3000), ids(3000)); - List read = - IcebergColStatsReader.decodePartBlob(blob, Set.of("c0", "c1499", "c2999"), true); + List read = IcebergColStatsReader.decodePartBlob( + blob, IcebergColStatsReader.neededFields(SCHEMA, Set.of("c0", "c1499", "c2999")), true, SCHEMA); Assert.assertEquals(List.of("c0", "c1499", "c2999"), read.stream().map(ColumnStatisticsObj::getColName).toList()); @@ -129,6 +133,18 @@ public void aVectorLeftBehindByAMergeIsStillStored() throws Exception { .getStatsData().getLongStats().isSetBitVectors()); } + /** + * A schema pairing up with {@link #columns} and {@link #ids}: field 1 is c0, field 2 is c1, and + * so on, so a decoded entry takes the name its field carries here. + */ + private static final Schema SCHEMA = new Schema( + IntStream.rangeClosed(1, 3000) + .mapToObj(id -> Types.NestedField.optional(id, "c" + (id - 1), Types.LongType.get())) + .collect(Collectors.toList())); + + /** The fields the schema still has, which is what a read asking for no columns in particular takes. */ + private static final IntPredicate LIVE_FIELDS = IcebergColStatsReader.neededFields(SCHEMA, null); + private static List columns(int count) { return IntStream.range(0, count).mapToObj(i -> { LongColumnStatsData longStats = new LongColumnStatsData(0, i + 1); @@ -204,7 +220,7 @@ public void blobsLyingCloseTogetherAreTakenInOneRead() throws Exception { List blobs = Lists.newArrayList(); for (int i = 0; i < 3; i++) { blobs.add(IcebergColStatsCodec.encodeBlob( - List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(1))); + List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(i + 1))); } // laid end to end, so no gap is worth a second request List offsets = List.of(0L, (long) blobs.get(0).length, @@ -213,7 +229,7 @@ public void blobsLyingCloseTogetherAreTakenInOneRead() throws Exception { RecordingStream in = layOut(blobs, offsets, meta); Map> read = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(in, meta, null, true, read, null); + IcebergColStatsReader.readBlobs(in, meta, LIVE_FIELDS, true, read, SCHEMA); Assert.assertEquals("three adjacent blobs are one request", 1, in.reads.size()); Assert.assertEquals(3, read.size()); @@ -229,7 +245,7 @@ public void aBlobBeyondTheSeekWorthMakingIsTakenOnItsOwn() throws Exception { List blobs = Lists.newArrayList(); for (int i = 0; i < 2; i++) { blobs.add(IcebergColStatsCodec.encodeBlob( - List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(1))); + List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(i + 1))); } // a gap wider than any seek is worth crossing List offsets = List.of(0L, 8L * 1024 * 1024); @@ -237,7 +253,7 @@ public void aBlobBeyondTheSeekWorthMakingIsTakenOnItsOwn() throws Exception { RecordingStream in = layOut(blobs, offsets, meta); Map> read = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(in, meta, null, true, read, null); + IcebergColStatsReader.readBlobs(in, meta, LIVE_FIELDS, true, read, SCHEMA); Assert.assertEquals("a wide gap costs a second request", 2, in.reads.size()); Assert.assertEquals("c0", read.get("p=0").getFirst().getColName()); @@ -251,7 +267,7 @@ public void theVectoredPathServesTheSameBlobsAsTheSerialOne() throws Exception { List blobs = Lists.newArrayList(); for (int i = 0; i < 3; i++) { blobs.add(IcebergColStatsCodec.encodeBlob( - List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(1))); + List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(i + 1))); } // the gap must beat minSeek (16K on both paths) so two runs, and so two ranges, reach the // vectored call - a narrower gap coalesces everything into one and the per-run math goes untried @@ -260,7 +276,7 @@ public void theVectoredPathServesTheSameBlobsAsTheSerialOne() throws Exception { List meta = Lists.newArrayList(); RecordingStream serial = layOut(blobs, offsets, meta); Map> read = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(serial, meta, null, true, read, null); + IcebergColStatsReader.readBlobs(serial, meta, LIVE_FIELDS, true, read, SCHEMA); java.nio.file.Path file = java.nio.file.Files.createTempFile("colstats", ".puffin"); try { @@ -270,7 +286,7 @@ public void theVectoredPathServesTheSameBlobsAsTheSerialOne() throws Exception { Assert.assertTrue("the stream unwraps to Hadoop's, so the vectored call is the one tested", in instanceof org.apache.iceberg.io.DelegatingInputStream); Map> vectored = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(in, meta, null, true, vectored, null); + IcebergColStatsReader.readBlobs(in, meta, LIVE_FIELDS, true, vectored, SCHEMA); Assert.assertEquals(read, vectored); } } finally { From cf1e7609d1e9d9f21ce4fc61fd1f962f4198d484 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 15 Sep 2026 23:09:20 +0300 Subject: [PATCH 07/13] HIVE-29834: Fold an aggregate only where every partition has statistics - the caller answers from nothing less, so folding a subset is work it discards --- .../apache/hadoop/hive/ql/optimizer/StatsOptimizer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index 6076859e25ee..ea6efac2a810 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -729,11 +729,14 @@ private AggrStats exactAggrColStats(List partNames) throws HiveException partStats.add(new ColumnStatistics(statsDesc, statsObjs)); } }); + if (partStats.size() != partNames.size()) { + // only an aggregate of every partition answers, so folding a subset is work for nothing + return null; + } HiveConf conf = hive.getConf(); List aggregated = MetaStoreServerUtils.aggrPartitionStats(partStats, MetaStoreUtils.getDefaultCatalog(conf), tbl.getDbName(), tbl.getTableName(), - partNames, colNames, - partStats.size() == partNames.size(), + partNames, colNames, true, MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION), MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER)); return new AggrStats(aggregated, partStats.size()); From 4afe9e3c43ab744ba36379d655ddcc6ec0b54ad7 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 15 Sep 2026 23:09:20 +0300 Subject: [PATCH 08/13] HIVE-29834: Compare the operation of a snapshot that may state none - a snapshot written without a summary names no operation --- .../iceberg/mr/hive/compaction/IcebergTableOptimizer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java index 604d2a058273..e8303d057c85 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java @@ -287,6 +287,7 @@ private Stream getRelevantSnapshots(org.apache.iceberg.Table icebergTa return StreamSupport.stream(icebergTable.snapshots().spliterator(), false) .filter(s -> pastSnapshotTimeMil == null || s.timestampMillis() > pastSnapshotTimeMil) .filter(s -> s.timestampMillis() <= currentSnapshot.timestampMillis()) - .filter(s -> !s.operation().equals(DataOperations.REPLACE)); + // a snapshot written without a summary names no operation + .filter(s -> !DataOperations.REPLACE.equals(s.operation())); } } From a5726ad529108266211ecc7f5cfa57912e364fc6 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 15 Sep 2026 23:09:25 +0300 Subject: [PATCH 09/13] HIVE-29834: Refuse a column-stats blob that states more than it holds - a count or a length the rest of the blob cannot hold is refused outright - the codec frames a partition blob whole, so nothing outside it writes one - each name says the level it reads and whether it goes to the stream --- .../mr/hive/stats/IcebergColStatsCodec.java | 36 +++++++---- .../mr/hive/stats/IcebergColStatsReader.java | 12 ++-- .../mr/hive/stats/IcebergColStatsWriter.java | 15 +---- .../hive/stats/TestIcebergColStatsFormat.java | 60 +++++++++++++------ 4 files changed, 74 insertions(+), 49 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java index 417e5ab7cbf5..ec9f0dcc8665 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsCodec.java @@ -29,6 +29,7 @@ import java.util.function.IntPredicate; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; @@ -78,18 +79,22 @@ private IcebergColStatsCodec() { * entry rather than once for the blob because a merge carries partitions gathered separately - * they need not hold the same columns, nor hold them in the same order. */ - static byte[] encodeBlob(List parts, List fieldIds) throws IOException { + static ByteBuffer encodePartBlob(List statsObjs, List fieldIds) + throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); DataOutputStream data = new DataOutputStream(out); data.writeInt(BLOB_VERSION); - data.writeInt(parts.size()); - for (int i = 0; i < parts.size(); i++) { + data.writeInt(statsObjs.size()); + + for (int i = 0; i < statsObjs.size(); i++) { + // vectors and histograms alike: what a read wants of them it settles once they are in hand + byte[] entry = encodeEntry(statsObjs.get(i)); data.writeInt(fieldIds.get(i)); - data.writeInt(parts.get(i).length); - data.write(parts.get(i)); + data.writeInt(entry.length); + data.write(entry); } data.flush(); - return out.toByteArray(); + return ByteBuffer.wrap(out.toByteArray()); } /** @@ -97,20 +102,29 @@ static byte[] encodeBlob(List parts, List fieldIds) throws IOEx * asks about a few of its columns, and an entry it does not need costs a read nothing beyond the * length it steps over. */ - static List decodeBlob(ByteBuffer buf, IntPredicate needed) { + static List decodePartBlob(ByteBuffer buf, IntPredicate needed) { ByteBuffer data = buf.duplicate().order(ByteOrder.BIG_ENDIAN); - if (data.remaining() < Integer.BYTES || data.getInt() != BLOB_VERSION) { + if (data.remaining() < 2 * Integer.BYTES || data.getInt() != BLOB_VERSION) { return List.of(); } int count = data.getInt(); - List entries = Lists.newArrayListWithCapacity(count); + Preconditions.checkArgument(count >= 0 && count <= data.remaining() / (2 * Integer.BYTES), + "Column statistics blob states %s entries, of which its remaining %s bytes cannot hold " + + "even the field and the length each states before its own bytes", + count, data.remaining()); + + List entries = Lists.newArrayListWithCapacity(count); for (int i = 0; i < count; i++) { int fieldId = data.getInt(); int length = data.getInt(); + Preconditions.checkArgument(length >= 0 && length <= data.remaining(), + "Entry of field %s states %s bytes, of the %s the blob has left to read or step over", + fieldId, length, data.remaining()); + if (needed.test(fieldId)) { byte[] part = new byte[length]; data.get(part); - entries.add(new FieldEntry(fieldId, part)); + entries.add(new EncodedStats(fieldId, part)); } else { data.position(data.position() + length); } @@ -118,7 +132,7 @@ static List decodeBlob(ByteBuffer buf, IntPredicate needed) { return entries; } - record FieldEntry(int fieldId, byte[] bytes) { + record EncodedStats(int fieldId, byte[] bytes) { } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java index 22a35cf2f489..2fcf6584c8c8 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsReader.java @@ -310,7 +310,7 @@ public static Map> readPart(Table table, Stati InputFile file = table.io().newInputFile(statsFile.path(), statsFile.fileSizeInBytes()); try (SeekableInputStream in = file.newStream()) { // the asked columns are the same fields in every blob, so they are resolved once here - readBlobs(in, blobs, neededFields(schema, columns), withVectors, result, schema); + readPartEntries(in, blobs, neededFields(schema, columns), withVectors, result, schema); } } } catch (Exception e) { @@ -330,7 +330,7 @@ public static Map> readPart(Table table, Stati * per round trip, which a file holding a blob per partition cannot afford. It leaves in favor * of Iceberg's reader once that one coalesces runs and takes them in one vectored call. */ - static void readBlobs(SeekableInputStream in, List blobs, IntPredicate needed, + static void readPartEntries(SeekableInputStream in, List blobs, IntPredicate needed, boolean withVectors, Map> result, Schema schema) throws IOException { List ordered = blobs.stream() @@ -371,7 +371,7 @@ static void readBlobs(SeekableInputStream in, List blobs, IntPredi part.position((int) (blob.offset() - start)); part.limit((int) (blob.offset() - start + blob.length())); result.put(blob.properties().get(IcebergColStatsWriter.PARTITION_PROP), - decodePartBlob(part.slice(), needed, withVectors, schema)); + decodePartEntries(part.slice(), needed, withVectors, schema)); } } } @@ -421,12 +421,12 @@ private static List readRanges(SeekableInputStream in, List decodePartBlob(ByteBuffer blob, IntPredicate needed, + static List decodePartEntries(ByteBuffer blob, IntPredicate needed, boolean withVectors, Schema schema) { - List stored = IcebergColStatsCodec.decodeBlob(blob, needed); + List stored = IcebergColStatsCodec.decodePartBlob(blob, needed); List entries = Lists.newArrayListWithCapacity(stored.size()); - for (IcebergColStatsCodec.FieldEntry entry : stored) { + for (IcebergColStatsCodec.EncodedStats entry : stored) { ColumnStatisticsObj statsObj = IcebergColStatsCodec.decodeEntry(entry.bytes(), withVectors); statsObj.setColName(SchemaUtils.getColumnName(schema, entry.fieldId())); entries.add(statsObj); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java index 1a4838fc875a..8e9b9ac4d4fe 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java @@ -246,7 +246,7 @@ private static boolean writePart(Table tbl, Snapshot snapshot, Iterator registeredBlobs( return registered; } - static ByteBuffer encodePartBlob(List statsObjs, List fieldIds) - throws IOException { - List entries = Lists.newArrayListWithCapacity(statsObjs.size()); - for (ColumnStatisticsObj obj : statsObjs) { - // vectors and histograms alike: what a read wants of them it settles once they are in hand - entries.add(IcebergColStatsCodec.encodeEntry(obj)); - } - return ByteBuffer.wrap( - IcebergColStatsCodec.encodeBlob(entries, fieldIds)); - } - /** * Opens a statistics file for the snapshot, lets the caller add its blobs, and registers it on * the table. A file no blob was added to is left uncommitted, so the statistics standing for the diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java index 384c8139ee94..24c89005d2c6 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java @@ -35,6 +35,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ByteBuffers; import org.junit.Assert; import org.junit.Test; @@ -50,10 +51,10 @@ public void nothingInTheFrameIsPacked() throws Exception { new Random(11).nextBytes(sketch); statsObj.getStatsData().getLongStats().setBitVectors(sketch); - ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(List.of(statsObj), ids(1)); + ByteBuffer blob = IcebergColStatsCodec.encodePartBlob(List.of(statsObj), ids(1)); Assert.assertTrue("the sketch takes its own size in the blob", blob.remaining() > sketch.length); - Assert.assertEquals(List.of(statsObj), IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, true, SCHEMA)); + Assert.assertEquals(List.of(statsObj), IcebergColStatsReader.decodePartEntries(blob, LIVE_FIELDS, true, SCHEMA)); } @Test @@ -63,17 +64,17 @@ public void aVectorComesBackOnlyWhenAskedForAndAHistogramAlways() throws Excepti ColumnStatisticsObj statsObj = columns(1).getFirst(); statsObj.getStatsData().getLongStats().setBitVectors(new byte[] {1, 2, 3, 4}); statsObj.getStatsData().getLongStats().setHistogram(new byte[] {5, 6, 7}); - ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(List.of(statsObj), ids(1)); + ByteBuffer blob = IcebergColStatsCodec.encodePartBlob(List.of(statsObj), ids(1)); ColumnStatisticsObj asked = - IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, true, SCHEMA).getFirst(); + IcebergColStatsReader.decodePartEntries(blob, LIVE_FIELDS, true, SCHEMA).getFirst(); Assert.assertArrayEquals("the vector is put back where it was taken from", new byte[] {1, 2, 3, 4}, asked.getStatsData().getLongStats().getBitVectors()); Assert.assertArrayEquals("and so is the histogram", new byte[] {5, 6, 7}, asked.getStatsData().getLongStats().getHistogram()); ColumnStatisticsObj unasked = - IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, false, SCHEMA).getFirst(); + IcebergColStatsReader.decodePartEntries(blob, LIVE_FIELDS, false, SCHEMA).getFirst(); Assert.assertFalse("a read that did not ask for the vector does not get it", unasked.getStatsData().getLongStats().isSetBitVectors()); Assert.assertArrayEquals("the histogram comes back whatever was asked", @@ -85,18 +86,39 @@ public void aVectorComesBackOnlyWhenAskedForAndAHistogramAlways() throws Excepti @Test public void aFrameFromAVersionThisReaderDoesNotKnowReadsAsAbsent() throws Exception { - ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(columns(2), ids(2)); + ByteBuffer blob = IcebergColStatsCodec.encodePartBlob(columns(2), ids(2)); blob.putInt(0, IcebergColStatsCodec.BLOB_VERSION + 1); - Assert.assertTrue(IcebergColStatsReader.decodePartBlob(blob, LIVE_FIELDS, true, SCHEMA).isEmpty()); + Assert.assertTrue(IcebergColStatsReader.decodePartEntries(blob, LIVE_FIELDS, true, SCHEMA).isEmpty()); + } + + @Test + public void aFrameStatingMoreThanItHoldsIsRefusedRatherThanRead() throws Exception { + // a blob the file cut short, or one a length was read out of that is not a count of entries: + // either is refused outright, so nothing is sized or stepped over from a number this far off + ByteBuffer blob = IcebergColStatsCodec.encodePartBlob(columns(2), ids(2)); + blob.putInt(Integer.BYTES, Integer.MAX_VALUE); + + Assert.assertThrows(IllegalArgumentException.class, + () -> IcebergColStatsReader.decodePartEntries(blob, LIVE_FIELDS, true, SCHEMA)); + } + + @Test + public void anEntryStatingMoreBytesThanTheFrameHasLeftIsRefused() throws Exception { + ByteBuffer blob = IcebergColStatsCodec.encodePartBlob(columns(2), ids(2)); + // the length of the first entry, behind the version, the count and the field it is for + blob.putInt(3 * Integer.BYTES, blob.remaining()); + + Assert.assertThrows(IllegalArgumentException.class, + () -> IcebergColStatsReader.decodePartEntries(blob, LIVE_FIELDS, true, SCHEMA)); } @Test public void aWideFrameStillYieldsOnlyTheAskedColumns() throws Exception { // 3000 columns push the header well past anything a single small read would hold - ByteBuffer blob = IcebergColStatsWriter.encodePartBlob(columns(3000), ids(3000)); + ByteBuffer blob = IcebergColStatsCodec.encodePartBlob(columns(3000), ids(3000)); - List read = IcebergColStatsReader.decodePartBlob( + List read = IcebergColStatsReader.decodePartEntries( blob, IcebergColStatsReader.neededFields(SCHEMA, Set.of("c0", "c1499", "c2999")), true, SCHEMA); Assert.assertEquals(List.of("c0", "c1499", "c2999"), @@ -219,8 +241,8 @@ private static RecordingStream layOut(List blobs, List offsets, Li public void blobsLyingCloseTogetherAreTakenInOneRead() throws Exception { List blobs = Lists.newArrayList(); for (int i = 0; i < 3; i++) { - blobs.add(IcebergColStatsCodec.encodeBlob( - List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(i + 1))); + blobs.add(ByteBuffers.toByteArray(IcebergColStatsCodec.encodePartBlob( + List.of(longColumn("c" + i, i)), List.of(i + 1)))); } // laid end to end, so no gap is worth a second request List offsets = List.of(0L, (long) blobs.get(0).length, @@ -229,7 +251,7 @@ public void blobsLyingCloseTogetherAreTakenInOneRead() throws Exception { RecordingStream in = layOut(blobs, offsets, meta); Map> read = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(in, meta, LIVE_FIELDS, true, read, SCHEMA); + IcebergColStatsReader.readPartEntries(in, meta, LIVE_FIELDS, true, read, SCHEMA); Assert.assertEquals("three adjacent blobs are one request", 1, in.reads.size()); Assert.assertEquals(3, read.size()); @@ -244,8 +266,8 @@ public void blobsLyingCloseTogetherAreTakenInOneRead() throws Exception { public void aBlobBeyondTheSeekWorthMakingIsTakenOnItsOwn() throws Exception { List blobs = Lists.newArrayList(); for (int i = 0; i < 2; i++) { - blobs.add(IcebergColStatsCodec.encodeBlob( - List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(i + 1))); + blobs.add(ByteBuffers.toByteArray(IcebergColStatsCodec.encodePartBlob( + List.of(longColumn("c" + i, i)), List.of(i + 1)))); } // a gap wider than any seek is worth crossing List offsets = List.of(0L, 8L * 1024 * 1024); @@ -253,7 +275,7 @@ public void aBlobBeyondTheSeekWorthMakingIsTakenOnItsOwn() throws Exception { RecordingStream in = layOut(blobs, offsets, meta); Map> read = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(in, meta, LIVE_FIELDS, true, read, SCHEMA); + IcebergColStatsReader.readPartEntries(in, meta, LIVE_FIELDS, true, read, SCHEMA); Assert.assertEquals("a wide gap costs a second request", 2, in.reads.size()); Assert.assertEquals("c0", read.get("p=0").getFirst().getColName()); @@ -266,8 +288,8 @@ public void theVectoredPathServesTheSameBlobsAsTheSerialOne() throws Exception { // readVectored instead of reading them one by one, and every partition still gets its own bytes List blobs = Lists.newArrayList(); for (int i = 0; i < 3; i++) { - blobs.add(IcebergColStatsCodec.encodeBlob( - List.of(IcebergColStatsCodec.encodeEntry(longColumn("c" + i, i))), List.of(i + 1))); + blobs.add(ByteBuffers.toByteArray(IcebergColStatsCodec.encodePartBlob( + List.of(longColumn("c" + i, i)), List.of(i + 1)))); } // the gap must beat minSeek (16K on both paths) so two runs, and so two ranges, reach the // vectored call - a narrower gap coalesces everything into one and the per-run math goes untried @@ -276,7 +298,7 @@ public void theVectoredPathServesTheSameBlobsAsTheSerialOne() throws Exception { List meta = Lists.newArrayList(); RecordingStream serial = layOut(blobs, offsets, meta); Map> read = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(serial, meta, LIVE_FIELDS, true, read, SCHEMA); + IcebergColStatsReader.readPartEntries(serial, meta, LIVE_FIELDS, true, read, SCHEMA); java.nio.file.Path file = java.nio.file.Files.createTempFile("colstats", ".puffin"); try { @@ -286,7 +308,7 @@ public void theVectoredPathServesTheSameBlobsAsTheSerialOne() throws Exception { Assert.assertTrue("the stream unwraps to Hadoop's, so the vectored call is the one tested", in instanceof org.apache.iceberg.io.DelegatingInputStream); Map> vectored = Maps.newLinkedHashMap(); - IcebergColStatsReader.readBlobs(in, meta, LIVE_FIELDS, true, vectored, SCHEMA); + IcebergColStatsReader.readPartEntries(in, meta, LIVE_FIELDS, true, vectored, SCHEMA); Assert.assertEquals(read, vectored); } } finally { From 94e4cda8fac738e836381b2b15913d9313a29a32 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 15 Sep 2026 23:09:25 +0300 Subject: [PATCH 10/13] HIVE-29834: Answer count(col) and a sum of a constant from partition statistics --- .../queries/positive/iceberg_part_colstats.q | 29 ++++ .../positive/iceberg_part_colstats.q.out | 131 ++++++++++++++++++ 2 files changed, 160 insertions(+) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q index ccf156218595..12f4be2536b9 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q @@ -70,6 +70,35 @@ select max(id) from ice_part_stats where p in ('a', 'b'); drop table ice_part_stats; +-- count(col) reads the column's null count, which a string keeps elsewhere in its entry than a +-- number does, and sum of a constant reads the row count alone - a handler keeps no partition +-- parameters for either of them to be read from +create external table ice_count_stats (id bigint, s string, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_count_stats values (1, 'x', 'a'), (2, null, 'a'), (3, 'y', 'b'); +analyze table ice_count_stats compute statistics for columns; + +-- p=a holds two rows, one of which states no s +explain +select count(s) from ice_count_stats where p = 'a'; + +select count(s) from ice_count_stats where p = 'a'; + +explain +select sum(1) from ice_count_stats where p = 'a'; + +select sum(1) from ice_count_stats where p = 'a'; + +-- count(*) and count(1) read every row of the partition, count(null) none of them +explain +select count(*), count(1), count(null) from ice_count_stats where p = 'a'; + +select count(*), count(1), count(null) from ice_count_stats where p = 'a'; + +drop table ice_count_stats; + -- an unpartitioned table keeps its statistics in the same file, which the metastore never holds: -- reaching them takes the handler, and only the accuracy check stands between a query and stale ones create external table ice_unpart (id bigint) diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out index 35dc90b880d7..3293f88e1d25 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out @@ -357,6 +357,137 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@ice_part_stats POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_part_stats +PREHOOK: query: create external table ice_count_stats (id bigint, s string, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_count_stats +POSTHOOK: query: create external table ice_count_stats (id bigint, s string, p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_count_stats +PREHOOK: query: insert into ice_count_stats values (1, 'x', 'a'), (2, null, 'a'), (3, 'y', 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_count_stats +POSTHOOK: query: insert into ice_count_stats values (1, 'x', 'a'), (2, null, 'a'), (3, 'y', 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_count_stats +PREHOOK: query: analyze table ice_count_stats compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: default@ice_count_stats +PREHOOK: Output: default@ice_count_stats@p=a +PREHOOK: Output: default@ice_count_stats@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_count_stats compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: default@ice_count_stats +POSTHOOK: Output: default@ice_count_stats@p=a +POSTHOOK: Output: default@ice_count_stats@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select count(s) from ice_count_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(s) from ice_count_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(s) from ice_count_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(s) from ice_count_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 +PREHOOK: query: explain +select sum(1) from ice_count_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select sum(1) from ice_count_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(1) from ice_count_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select sum(1) from ice_count_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +2 +PREHOOK: query: explain +select count(*), count(1), count(null) from ice_count_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select count(*), count(1), count(null) from ice_count_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*), count(1), count(null) from ice_count_stats where p = 'a' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*), count(1), count(null) from ice_count_stats where p = 'a' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +2 2 0 +PREHOOK: query: drop table ice_count_stats +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_count_stats +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_count_stats +POSTHOOK: query: drop table ice_count_stats +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_count_stats +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_count_stats PREHOOK: query: create external table ice_unpart (id bigint) stored by iceberg tblproperties ('format-version'='2') PREHOOK: type: CREATETABLE From ec4cf6b791f538e95f3d848587f41e39de2e19ff Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 15 Sep 2026 23:41:47 +0300 Subject: [PATCH 11/13] HIVE-29834: Keep on the registered entry only what stands for every partition - the blob type says the file holds partitions, so the name of one is not copied up - a partition names itself on its own blob, in the file's own footer --- .../iceberg/mr/hive/stats/IcebergColStatsWriter.java | 4 ++-- .../iceberg/mr/hive/stats/IcebergStoredStats.java | 11 +++++------ .../iceberg/mr/hive/TestHiveIcebergStatistics.java | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java index 8e9b9ac4d4fe..fdcad07e771c 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergColStatsWriter.java @@ -486,9 +486,9 @@ private static List registeredBlobs( continue; } fieldsNamed = true; - // the count lets a read turn away an ask of another size without opening the file + // this entry stands for every partition blob, so it keeps only what is true of all of + // them: the count, which turns away an ask of another size without opening the file ImmutableMap.Builder properties = ImmutableMap.builder() - .putAll(blob.properties()) .put(NUM_PARTITIONS_PROP, String.valueOf(numPartitions)); if (fullTableAggr) { properties.put(FULL_TABLE_AGGR_PROP, "true"); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java index 1258992957e9..cbcadca0919c 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/stats/IcebergStoredStats.java @@ -138,7 +138,7 @@ private static StatisticsFile colStatsFileOf(Table table, long snapshotId, boole /** * The file whose blobs are Hive's own - Iceberg keeps statistics of its own in the same format - - * at the asked-for granularity: a blob describing one partition names it in its metadata. + * at the asked-for granularity: a blob describing one partition is of a type of its own. * *

A file that holds any partition is a per partition one, whatever else it holds. Its * aggregates serve a whole-table read only while they aggregate the full table - what a gather @@ -146,10 +146,9 @@ private static StatisticsFile colStatsFileOf(Table table, long snapshotId, boole */ private static boolean holdsHiveColStats(StatisticsFile stats, boolean partitionLevel) { boolean holdsPartitions = stats.blobMetadata().stream() - .anyMatch(metadata -> metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)); + .anyMatch(metadata -> IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type())); if (partitionLevel) { - return holdsPartitions && stats.blobMetadata().stream().anyMatch( - metadata -> IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type())); + return holdsPartitions; } if (holdsPartitions) { return hasFullTableAggr(stats); @@ -172,7 +171,7 @@ static boolean hasFullTableAggr(StatisticsFile stats) { static StatisticsFile getTableOnlyColStatsFile(Table table, long snapshotId) { StatisticsFile stats = getColStatsFile(table, snapshotId, false); return stats == null || stats.blobMetadata().stream() - .anyMatch(metadata -> metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)) ? + .anyMatch(metadata -> IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type())) ? null : stats; } @@ -216,7 +215,7 @@ private static Set storedFieldIds(Table table, Snapshot snapshot, Confi Set fields = statsFile == null ? Set.of() : statsFile.blobMetadata().stream() .filter(metadata -> partitionLevel || - !metadata.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)) + !IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(metadata.type())) .flatMap(metadata -> metadata.fields().stream()) .collect(Collectors.toSet()); SessionStateUtil.addResource(conf, cacheKey, fields); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index 2a7fa9869712..313aba59aeef 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -3007,7 +3007,7 @@ public void testTheTableMetadataRegistersOnePartitionEntryNamingEveryFieldAndThe Table icebergTable = testTables.loadTable(identifier); var registered = currentColStatsFile(icebergTable).blobMetadata(); var partitionEntries = registered.stream() - .filter(blob -> blob.properties().containsKey(IcebergColStatsWriter.PARTITION_PROP)) + .filter(blob -> IcebergColStatsWriter.HIVE_PART_COL_STATS_BLOB_V1.equals(blob.type())) .toList(); Assert.assertEquals("one entry stands for the partitions", 1, partitionEntries.size()); var partitionEntry = partitionEntries.get(0); From 0ccd69021e4cfde9f07aee868c56f48217c945e4 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Wed, 16 Sep 2026 00:13:01 +0300 Subject: [PATCH 12/13] HIVE-29834: Answer no basic statistics for a metadata table - what the snapshot counts is the rows of the table the metadata describes --- .../org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index ff3136aca10b..807ed4ee0579 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -503,6 +503,9 @@ private Map getBasicStatistics(org.apache.hadoop.hive.ql.metadat boolean quickStats) { Map stats; + if (hmsTable.getMetaTable() != null) { + return Map.of(); + } // For write queries where rows got modified, don't fetch from cache as values could have changed. Table table = getTable(hmsTable); Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable); From 753446fecea4c44b703c13bc306b33c677b069f4 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Wed, 16 Sep 2026 12:24:21 +0300 Subject: [PATCH 13/13] HIVE-29834: Take the value a MIN or MAX folds out of its conditional - an arrow switch per type, the bound read into a local before it is cast - aggregateColumns collects in one pass, pattern matching instead of casts --- .../hive/stats/TestIcebergColStatsFormat.java | 3 +- .../hive/ql/optimizer/StatsOptimizer.java | 47 ++++++++++--------- .../hadoop/hive/ql/stats/StatsUtils.java | 2 +- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java index 24c89005d2c6..c0efe95c8ae7 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/stats/TestIcebergColStatsFormat.java @@ -25,7 +25,6 @@ import java.util.Random; import java.util.Set; import java.util.function.IntPredicate; -import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -162,7 +161,7 @@ public void aVectorLeftBehindByAMergeIsStillStored() throws Exception { private static final Schema SCHEMA = new Schema( IntStream.rangeClosed(1, 3000) .mapToObj(id -> Types.NestedField.optional(id, "c" + (id - 1), Types.LongType.get())) - .collect(Collectors.toList())); + .toList()); /** The fields the schema still has, which is what a read asking for no columns in particular takes. */ private static final IntPredicate LIVE_FIELDS = IcebergColStatsReader.neededFields(SCHEMA, null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index ea6efac2a810..4539b424c0b9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Date; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; @@ -93,6 +94,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.function.Function; import java.util.stream.Collectors; import java.util.List; @@ -472,38 +474,38 @@ else if (udaf instanceof GenericUDAFCount) { ((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); String colName = colDesc.getColumn(); StatType type = getType(colDesc.getTypeString()); + ColumnStatisticsData statData = scanColStats.statsFor(colName, type); if (statData == null) { return null; // logging inside } String name = colDesc.getTypeString().toUpperCase(); boolean high = udaf instanceof GenericUDAFMax; + switch (type) { - case Integer: { + case Integer -> { LongColumnStatsData lstats = statData.getLongStats(); boolean isSet = high ? lstats.isSetHighValue() : lstats.isSetLowValue(); - oneRow.add(isSet ? LongSubType.valueOf(name).cast( - high ? lstats.getHighValue() : lstats.getLowValue()) : null); - break; + long bound = high ? lstats.getHighValue() : lstats.getLowValue(); + oneRow.add(isSet ? LongSubType.valueOf(name).cast(bound) : null); } - case Double: { + case Double -> { DoubleColumnStatsData dstats = statData.getDoubleStats(); boolean isSet = high ? dstats.isSetHighValue() : dstats.isSetLowValue(); - oneRow.add(isSet ? DoubleSubType.valueOf(name).cast( - high ? dstats.getHighValue() : dstats.getLowValue()) : null); - break; + double bound = high ? dstats.getHighValue() : dstats.getLowValue(); + oneRow.add(isSet ? DoubleSubType.valueOf(name).cast(bound) : null); } - case Date: { + case Date -> { DateColumnStatsData dstats = statData.getDateStats(); boolean isSet = high ? dstats.isSetHighValue() : dstats.isSetLowValue(); - oneRow.add(isSet ? DateSubType.DAYS.cast((high ? - dstats.getHighValue() : dstats.getLowValue()).getDaysSinceEpoch()) : null); - break; + Date bound = high ? dstats.getHighValue() : dstats.getLowValue(); + oneRow.add(isSet ? DateSubType.DAYS.cast(bound.getDaysSinceEpoch()) : null); } - default: + default -> { Logger.debug("Unsupported type: " + colDesc.getTypeString() + " encountered in " + "metadata optimizer for column : " + colName); return null; + } } } else { // Unsupported aggregation. Logger.debug("Unsupported aggregation for metadata optimizer: " @@ -592,15 +594,16 @@ else if (udaf instanceof GenericUDAFCount) { /** The columns the aggregates read, which are the ones statistics have to be fetched for. */ private static List aggregateColumns(GroupByOperator pgbyOp, Map exprMap) { - return pgbyOp.getConf().getAggregators().stream() - .filter(aggr -> !aggr.getParameters().isEmpty()) - .map(aggr -> aggr.getParameters().get(0)) - .filter(ExprNodeColumnDesc.class::isInstance) - .map(desc -> exprMap.get(((ExprNodeColumnDesc) desc).getColumn())) - .filter(ExprNodeColumnDesc.class::isInstance) - .map(desc -> ((ExprNodeColumnDesc) desc).getColumn()) - .distinct() - .collect(Collectors.toList()); + Set columns = new LinkedHashSet<>(); + for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) { + List params = aggr.getParameters(); + if (!params.isEmpty() + && params.getFirst() instanceof ExprNodeColumnDesc param + && exprMap.get(param.getColumn()) instanceof ExprNodeColumnDesc column) { + columns.add(column.getColumn()); + } + } + return List.copyOf(columns); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index fde920352147..74e1e8813595 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -2095,7 +2095,7 @@ public static boolean areColumnStatsUptoDateForQueryAnswering(Table table, Map