fix(pivot-table): blank mixed-metric totals in exports and reports

UPDATING.md

@@ -24,6 +24,15 @@ assists people when migrating to a new version.
 
 ## Next
 
+- **Pivot Table mixed-metric export totals.** CSV/Excel exports and scheduled
+  CSV/text reports leave Total and Subtotal cells blank when they combine different
+  metrics, including two SUM metrics. Metric-specific values and totals remain
+  available, including with Combine metrics and Transpose pivot enabled. Previously
+  these cells could show an arbitrary metric's value or a sum across unrelated metrics.
+  Downstream consumers must handle empty cells or use a single metric that explicitly
+  computes the desired combined value. This change does not restore result-aggregation
+  parity between exports and the browser.
+
 - **Impala SQL Lab stops no longer call the coordinator's HTTP cancel.** A SQL
   Lab stop on Impala is delivered to the worker that is polling the query, which
   cancels it with `cancel_operation()`, as Hive and Presto already do. The HTTP

superset/charts/client_processing.py

@@ -157,23 +157,20 @@ def get_metric_rollup_reducers(
     return reducers
 
 
-def _collapsed_metric(present: list[Any], metrics: list[str]) -> Any:
+def _collapsed_metric(present: list[Any] | set[Any]) -> Any:
     """
-    The metric a rollup spanning `present` stands for.
-
-    A total that collapses the metric axis is undefined in the renderer, which
-    resolves it to the last metric pushed into the shared slot (see the
-    ``metricAxis`` handling in ``react-pivottable/utilities.ts``). Mirror that
-    by taking the last metric in the configured order, so exported percentages
-    match the chart rather than summing metrics that share no unit.
+    The metric a rollup spanning `present` stands for, or `None` when `present`
+    spans more than one distinct metric.
+
+    A total that collapses the metric axis has no single meaningful value once
+    it mixes distinct metrics -- e.g. a "Total" cell combining `MAX(sales)`
+    with `MEDIAN(msrp)`. The browser change in #44657 blanks that case instead
+    of picking one arbitrarily (see ``metricAxis`` handling in
+    ``react-pivottable/utilities.ts``); mirror it here so exported/scheduled
+    values agree with what the chart shows.
     """
-    distinct = set(present)
-    if len(distinct) == 1:
-        return distinct.pop()
-    for metric in reversed(metrics):
-        if metric in distinct:
-            return metric
-    return None
+    distinct = present if isinstance(present, set) else set(present)
+    return next(iter(distinct)) if len(distinct) == 1 else None
 
 
 def _broadcast(total: pd.Series, block: pd.DataFrame, axis: int) -> pd.DataFrame:
@@ -194,14 +191,30 @@ def _metric_of_column(column: Any, metric_level: int) -> Any:
     return column[metric_level] if isinstance(column, tuple) else column
 
 
+def _rollup_metric(column: Any, metric_level: int, metrics: set[str]) -> Any:
+    """The metric to read from a rollup record for a cell in ``column``.
+
+    A column carrying a total label instead of a metric name collapses the metric
+    axis; see ``_collapsed_metric``.
+    """
+    name = _metric_of_column(column, metric_level)
+    return name if name in metrics else _collapsed_metric(metrics)
+
+
 def _reduce(
     data: Union[pd.DataFrame, pd.Series],
     reducer: str,
     axis: Optional[int] = None,
 ) -> Any:
-    """Apply a rollup reducer (``sum``/``min``/``max``), skipping empty cells."""
+    """Apply a rollup reducer (``sum``/``min``/``max``), skipping empty cells.
+
+    A rollup with no non-null observations stays blank:
+    ``sum`` otherwise defaults an all-NaN input to 0, treating missing values
+    as a measured zero.
+    """
     method = getattr(data, reducer)
-    return method(axis=axis) if axis is not None else method()
+    kwargs = {"min_count": 1} if reducer == "sum" else {}
+    return method(axis=axis, **kwargs) if axis is not None else method(**kwargs)
 
 
 def _rollup_index(
@@ -271,12 +284,6 @@ def _apply_rollup_totals(  # pylint: disable=too-many-arguments,too-many-locals
     keyed = _rollup_index(rollup_levels)
     metric_names = set(metrics)
 
-    def metric_of(column: Any) -> Any:
-        name = _metric_of_column(column, metric_level)
-        return (
-            name if name in metric_names else _collapsed_metric(list(metrics), metrics)
-        )
-
     def lookup(row: Any, column: Any) -> tuple[bool, Any]:
         row_depth = row_prefix_depth.get(row, len(rows))
         column_depth = column_prefix_depth.get(column, len(columns))
@@ -287,7 +294,7 @@ def lookup(row: Any, column: Any) -> tuple[bool, Any]:
         record = keyed(grouped).get(key)
         if record is None:
             return False, None
-        return True, record.get(metric_of(column))
+        return True, record.get(_rollup_metric(column, metric_level, metric_names))
 
     # Index positionally: a tuple label on a MultiIndex is ambiguous to `.loc`.
     for column_position, column in enumerate(df.columns):
@@ -331,12 +338,6 @@ def _rollup_denominators(  # pylint: disable=too-many-arguments,too-many-locals
     keyed = _rollup_index(rollup_levels)
     metric_names = set(metrics)
 
-    def metric_of(column: Any) -> Any:
-        name = _metric_of_column(column, metric_level)
-        return (
-            name if name in metric_names else _collapsed_metric(list(metrics), metrics)
-        )
-
     def denominator(row: Any, column: Any) -> tuple[bool, Any]:
         if mode == ShowValuesAs.PERCENT_OF_TOTAL:
             grouped: list[str] = []
@@ -356,7 +357,7 @@ def denominator(row: Any, column: Any) -> tuple[bool, Any]:
         record = keyed(grouped).get(key)
         if record is None:
             return False, None
-        return True, record.get(metric_of(column))
+        return True, record.get(_rollup_metric(column, metric_level, metric_names))
 
     resolved = [[denominator(row, column) for column in df.columns] for row in df.index]
     values = pd.DataFrame(
@@ -397,8 +398,8 @@ def _apply_show_values_as(  # pylint: disable=too-many-arguments
     - A total is rolled up within a single metric, so a cell is never divided by
       a total that mixes in another metric, and each metric uses its own
       reducer -- a MIN/MAX metric divides by the row's minimum/maximum rather
-      than its sum. A total that collapses the metric axis resolves to a single
-      metric the way the renderer does; see ``_collapsed_metric``.
+      than its sum. A total that combines distinct metrics stays blank; see
+      ``_collapsed_metric``.
 
     A zero denominator yields NaN (blank) rather than infinity, matching
     ``pandas_postprocessing.pivot``'s ``show_values_as``.
@@ -418,6 +419,8 @@ def _apply_show_values_as(  # pylint: disable=too-many-arguments
     derived = pd.DataFrame(np.nan, index=numeric.index, columns=numeric.columns)
     for metric in dict.fromkeys(metric_of_column):
         selection = np.array([column == metric for column in metric_of_column])
+        # Mixed-metric numerators are blank, so the fallback cross-metric
+        # denominator cannot produce a displayed value.
         denominator_selection = (
             selection if metric is not None else np.ones(len(selection), dtype=bool)
         )
@@ -431,8 +434,7 @@ def _apply_show_values_as(  # pylint: disable=too-many-arguments
                     metric_of_column, denominator_selection, strict=True
                 )
                 if keep and column_metric is not None
-            ],
-            metrics,
+            ]
         )
         reducer = reducers.get(str(denominator_metric), DEFAULT_ROLLUP_REDUCER)
         if denominator_metric is not None:
@@ -606,27 +608,37 @@ def pivot_df(  # pylint: disable=too-many-locals, too-many-arguments, too-many-s
     # still divides by itself: a MAX metric's row total is the row's maximum,
     # and dividing that maximum by itself reads 100%.
     totals_metric_level = df.columns.nlevels - 1 if combine_metrics else 0
-    # A column carrying a total label rather than a metric name rolls up
-    # everything; with no metric to resolve it to, it sums.
+    # Single-metric totals retain their reducer. Mixed-metric totals are
+    # blanked before reduction; the fallback cannot make them meaningful.
     cross_metric_reducer = (
         reducers.get(metrics[0], DEFAULT_ROLLUP_REDUCER)
         if len(set(metrics)) == 1
         else DEFAULT_ROLLUP_REDUCER
     )
 
-    def collapse(block: pd.DataFrame) -> tuple[pd.DataFrame, str]:
-        """Narrow a total's source columns to one metric, and pick its reducer."""
-        metric_names = set(metrics)
+    metric_names = set(metrics)
+
+    def collapse(block: pd.DataFrame) -> tuple[Optional[pd.DataFrame], str]:
+        """Narrow a total's source columns to one metric, and pick its reducer.
+
+        Returns `(None, reducer)` when `block` spans more than one distinct
+        metric: such a total has no single meaningful value, so the caller
+        blanks it instead of summing across metrics that share no unit
+        (matches the browser change proposed in #44657).
+        """
         present = [
             _metric_of_column(column, totals_metric_level) for column in block.columns
         ]
         known = [metric for metric in present if metric in metric_names]
-        metric = _collapsed_metric(known, metrics) if known else None
-        if metric is None:
+        if not known:
             return block, cross_metric_reducer
+        metric = _collapsed_metric(known)
+        if metric is None:
+            return None, DEFAULT_ROLLUP_REDUCER
         keep = [column == metric for column in present]
         return block.loc[:, keep], reducers.get(str(metric), DEFAULT_ROLLUP_REDUCER)
 
+    mixed_metric_totals: list[Any] = []
     if show_rows_total:
         # add subtotal for each group and overall total; we start from the
         # overall group, and iterate deeper into subgroups
@@ -654,8 +666,13 @@ def collapse(block: pd.DataFrame) -> tuple[pd.DataFrame, str]:
                     # to a number. The row totals below already coerce; do the
                     # same here so a total means the same thing on both axes.
                     block = block.apply(pd.to_numeric, errors="coerce")
-                if percent_mode:
-                    source, reducer = collapse(block)
+                source, reducer = collapse(block)
+                mixed_metrics = (
+                    source is None and aggfunc != CURRENCY_CONTEXT_AGGREGATION
+                )
+                if mixed_metrics:
+                    subtotal = pd.Series(np.nan, index=block.index)
+                elif percent_mode:
                     subtotal = _reduce(source, reducer, axis=1)
                 else:
                     subtotal = summarize_frame(aggfunc, block, axis=1)
@@ -665,6 +682,8 @@ def collapse(block: pd.DataFrame) -> tuple[pd.DataFrame, str]:
                 # insert column after subgroup
                 df.insert(int(slice_.stop), subtotal_name, subtotal)
                 inserted_columns.append(subtotal_name)
+                if mixed_metrics:
+                    mixed_metric_totals.append(subtotal_name)
                 column_prefix_depth[subtotal_name] = (
                     level if combine_metrics else max(0, level - 1)
                 )
@@ -686,7 +705,9 @@ def collapse(block: pd.DataFrame) -> tuple[pd.DataFrame, str]:
                     )
                     raise
 
-                subtotal_values = df.iloc[slice_, :]
+                # These labels were inserted above; no columns are removed
+                # before this point. Keep strict lookup to catch broken invariants.
+                subtotal_values = df.iloc[slice_, :].drop(columns=mixed_metric_totals)
                 if aggfunc != CURRENCY_CONTEXT_AGGREGATION:
                     subtotal_values = subtotal_values.apply(
                         pd.to_numeric, errors="coerce"
@@ -702,7 +723,11 @@ def collapse(block: pd.DataFrame) -> tuple[pd.DataFrame, str]:
                 subtotal.name = tuple([*subgroup, total, *([""] * depth)])  # noqa: C409
                 # insert row after subgroup
                 df = pd.concat(
-                    [df[: slice_.stop], subtotal.to_frame().T, df[slice_.stop :]]
+                    [
+                        df[: slice_.stop],
+                        subtotal.to_frame().T.reindex(columns=df.columns),
+                        df[slice_.stop :],
+                    ]
                 )
                 inserted_rows.append(subtotal.name)
                 row_prefix_depth[subtotal.name] = level
@@ -752,13 +777,13 @@ def collapse(block: pd.DataFrame) -> tuple[pd.DataFrame, str]:
             else None,
         )
 
-    # if we want to apply the metrics on the rows we need to pivot the
-    # dataframe back
+    # Replace placeholders before transposing, while column dtypes are intact.
+    df.replace("SUPERSET_PANDAS_NAN", np.nan, inplace=True)
     if apply_metrics_on_rows:
-        df = df.T
+        # Mixed blank columns must not coerce integer cells through float64
+        # during transposition (which would round integers above 2**53).
+        df = df.astype(object).T if mixed_metric_totals else df.T
 
-    # replace the remaining temporary placeholder string for np.nan after pivoting
-    df.replace("SUPERSET_PANDAS_NAN", np.nan, inplace=True)
     df.rename(
         index={"SUPERSET_PANDAS_NAN": np.nan},
         columns={"SUPERSET_PANDAS_NAN": np.nan},

superset/utils/excel.py

@@ -219,7 +219,12 @@ def apply_column_types(
         series = df.iloc[:, idx]
         if column_type == GenericDataType.NUMERIC:
             try:
-                series = pd.to_numeric(series)
+                # Nullable integers retain exact values alongside blank totals.
+                # Convert to object before apply, which would otherwise coerce
+                # nullable integers through float64 during scalar mapping.
+                series = pd.to_numeric(series, dtype_backend="numpy_nullable").astype(
+                    object
+                )
                 # if the number is too large, convert it to a string
                 # Excel does not support numbers larger than 10^15
                 series = series.apply(