fix: 140x optimizer speedup — lazy stats, in-place updates, skip redundant Train()

coverlet.runsettings

@@ -16,7 +16,17 @@
             File exclusions mirror sonar.coverage.exclusions in sonarcloud.yml — Sonar discards
             these anyway, so there is no point instrumenting or serializing them.
           -->
-          <ExcludeByFile>**/ProgramSynthesis/Models/**/*.cs,**/ProgramSynthesis/Execution/**/*.cs,**/ProgramSynthesis/Requests/**/*.cs,**/ProgramSynthesis/Results/**/*.cs,**/ProgramSynthesis/Interfaces/**/*.cs,**/ProgramSynthesis/Enums/**/*.cs,**/AiDotNet.Serving/Configuration/**/*.cs,**/AiDotNet.Serving/Models/**/*.cs,**/AiDotNet.Serving/Persistence/Entities/**/*.cs,**/AiDotNet.Serving/Persistence/Migrations/**/*.cs,**/AiDotNet.Serving/**/I*.cs,**/Interfaces/**/*.cs</ExcludeByFile>
+          <!--
+            Do NOT add a broad **/Interfaces/**/*.cs exclusion here: several files under
+            src/Interfaces are not pure interfaces (e.g. SubModel.cs contains a concrete
+            class; IParameterizable.cs has default interface method bodies). Excluding
+            the whole directory would silently hide executable code from coverage.
+            Configuration/DTO directories with no executable code are listed explicitly.
+            For the AiDotNet.Serving project we use the more aggressive **/I*.cs pattern
+            (per master's narrower-than-the-folder convention) to catch I-prefixed
+            interface files anywhere under that project tree.
+          -->
+          <ExcludeByFile>**/ProgramSynthesis/Models/**/*.cs,**/ProgramSynthesis/Execution/**/*.cs,**/ProgramSynthesis/Requests/**/*.cs,**/ProgramSynthesis/Results/**/*.cs,**/ProgramSynthesis/Interfaces/**/*.cs,**/ProgramSynthesis/Enums/**/*.cs,**/AiDotNet.Serving/Configuration/**/*.cs,**/AiDotNet.Serving/Models/**/*.cs,**/AiDotNet.Serving/Persistence/Entities/**/*.cs,**/AiDotNet.Serving/Persistence/Migrations/**/*.cs,**/AiDotNet.Serving/**/I*.cs</ExcludeByFile>
           <!-- SingleHit: record only whether each line was hit, not the hit count. Smaller XML, faster write. -->
           <SingleHit>true</SingleHit>
           <!-- SkipAutoProps: don't instrument trivial compiler-generated property accessors. -->

src/ActivationFunctions/BinarySpikingActivation.cs

@@ -238,20 +238,48 @@ public override Matrix<T> Derivative(Vector<T> input)
     /// </remarks>
     public override Tensor<T> Activate(Tensor<T> input)
     {
-        // Binary spiking is non-differentiable by design (Heaviside step), so
-        // surrogate-gradient learning approximates the forward with a steep
-        // sigmoid. This is the standard approach in SNN literature (Neftci et
-        // al., "Surrogate Gradient Learning in SNNs", 2019) and lets every op
-        // land on the gradient tape.
+        // Straight-through estimator (STE) per Neftci et al. 2019,
+        // "Surrogate Gradient Learning in Spiking Neural Networks":
+        //   * Forward value = Heaviside(input - threshold) — true binary spike.
+        //   * Backward value = sigmoid'(slope·(input - threshold)) — smooth surrogate.
         //
-        // f(x) ≈ sigmoid(slope * (x - threshold))
+        // The trick: build the output as
+        //     output = sigmoid + (heaviside - sigmoid_detached)
+        // where the sigmoid term is a tape-tracked Engine.Sigmoid so the gradient
+        // flows through the sigmoid derivative, and the (heaviside - sigmoid_detached)
+        // correction is a tape-detached constant offset that simply subtracts the
+        // sigmoid value and adds the Heaviside value, yielding the correct binary
+        // forward output. Because the correction tensors have no GradFn, the tape
+        // never tries to backprop through them — so the gradient of `output` w.r.t.
+        // `input` is exactly the sigmoid derivative, the surrogate prescribed by
+        // the SNN literature.
         //
-        // Larger `derivativeSlope` sharpens the approximation toward the true
-        // Heaviside and narrows the non-zero gradient window, matching the
-        // intent of the scalar Derivative(T) rectangle.
+        // This preserves the paper-faithful Heaviside output (true binary spikes
+        // at inference and on the forward pass during training) while keeping every
+        // step on the gradient tape so optimizer backward works automatically.
         var shifted = Engine.TensorSubtractScalar(input, _threshold);
         var scaled = Engine.TensorMultiplyScalar(shifted, _derivativeSlope);
-        return Engine.Sigmoid(scaled);
+        var sigmoidValue = Engine.Sigmoid(scaled);  // tape-tracked
+
+        // Heaviside snapshot — detached from tape, plain Tensor<T>.
+        var heaviside = new Tensor<T>(input._shape);
+        for (int i = 0; i < input.Length; i++)
+            heaviside[i] = NumOps.GreaterThanOrEquals(input[i], _threshold) ? NumOps.One : NumOps.Zero;
+
+        // Detached snapshot of sigmoidValue — copy the values out so the resulting
+        // tensor has no GradFn link back to `scaled` / `input`. The cheapest way to
+        // do this without engine support is to materialize a fresh Tensor<T> from
+        // the value buffer.
+        var sigmoidDetached = new Tensor<T>(input._shape);
+        for (int i = 0; i < sigmoidValue.Length; i++)
+            sigmoidDetached[i] = sigmoidValue[i];
+
+        // correction = heaviside - sigmoidDetached  (both detached → result detached)
+        // output = sigmoidValue + correction        (sigmoidValue is tape-tracked)
+        // Forward value collapses to: sigmoid + heaviside - sigmoid = heaviside  ✓
+        // Backward gradient: only sigmoidValue contributes               → sigmoid' ✓
+        var correction = Engine.TensorSubtract(heaviside, sigmoidDetached);
+        return Engine.TensorAdd(sigmoidValue, correction);
     }
 
     /// <summary>

src/ActivationFunctions/ScaledTanhActivation.cs

@@ -40,6 +40,12 @@ public class ScaledTanhActivation<T> : ActivationFunctionBase<T>
     private readonly T _beta;
     private readonly double _saturationThreshold;
 
+    // Precomputed constants for the tensor hot path to avoid repeated NumOps.FromDouble
+    // conversions and scalar arithmetic per Activate(Tensor<T>) call.
+    private readonly T _halfBeta;
+    private readonly T _negSaturationThreshold;
+    private readonly T _posSaturationThreshold;
+
     /// <summary>
     /// Initializes a new instance of the ScaledTanhActivation class with the specified steepness parameter.
     /// </summary>
@@ -64,6 +70,9 @@ public ScaledTanhActivation(double beta = 1.0, double saturationThreshold = 20.0
     {
         _beta = NumOps.FromDouble(beta);
         _saturationThreshold = saturationThreshold;
+        _halfBeta = NumOps.Multiply(_beta, NumOps.FromDouble(0.5));
+        _negSaturationThreshold = NumOps.FromDouble(-saturationThreshold);
+        _posSaturationThreshold = NumOps.FromDouble(saturationThreshold);
     }
 
     /// <summary>
@@ -119,9 +128,10 @@ public override T Activate(T input)
     /// </summary>
     public override Tensor<T> Activate(Tensor<T> input)
     {
-        var halfBeta = NumOps.Multiply(_beta, NumOps.FromDouble(0.5));
-        var scaled = Engine.TensorMultiplyScalar(input, halfBeta);
-        return Engine.Tanh(scaled);
+        var scaled = Engine.TensorMultiplyScalar(input, _halfBeta);
+        // Match scalar path: clamp to saturation threshold to prevent overflow
+        var clamped = Engine.TensorClamp(scaled, _negSaturationThreshold, _posSaturationThreshold);
+        return Engine.Tanh(clamped);
     }
 
     /// <summary>

src/ActivationFunctions/SphericalSoftmaxActivation.cs

@@ -35,6 +35,14 @@ namespace AiDotNet.ActivationFunctions;
 [ActivationProperty(IsMonotonic = false, ZeroPreserving = false, IsBounded = true, IsVectorActivation = true, Cost = ComputeCost.High)]
 public class SphericalSoftmaxActivation<T> : ActivationFunctionBase<T>
 {
+    /// <summary>
+    /// Numerical-stability epsilon added to the squared-norm before the sqrt to avoid
+    /// division-by-zero when the input vector is exactly zero. The value 1e-12 is well
+    /// below single-precision rounding error and is the standard guard used in
+    /// L2-normalization layers (e.g. PyTorch's <c>F.normalize(eps=1e-12)</c>).
+    /// </summary>
+    private static readonly T NormalizationEpsilon = MathHelper.GetNumericOperations<T>().FromDouble(1e-12);
+
     /// <summary>
     /// Indicates whether this activation function supports scalar operations.
     /// </summary>
@@ -111,10 +119,17 @@ public override Vector<T> Activate(Vector<T> input)
     /// </summary>
     public override Tensor<T> Activate(Tensor<T> input)
     {
+        if (input is null)
+            throw new ArgumentNullException(nameof(input));
+        if (input.Shape.Length == 0)
+            throw new ArgumentException("SphericalSoftmax requires at least one tensor dimension.", nameof(input));
+
         int lastAxis = input.Shape.Length - 1;
         var squared = Engine.TensorMultiply(input, input);
         var sumSquared = Engine.ReduceSum(squared, new[] { lastAxis }, keepDims: true);
-        var norm = Engine.TensorSqrt(sumSquared);
+        var eps = Tensor<T>.CreateDefault(sumSquared._shape, NormalizationEpsilon);
+        var safeSum = Engine.TensorAdd(sumSquared, eps);
+        var norm = Engine.TensorSqrt(safeSum);
         var normalized = Engine.TensorBroadcastDivide(input, norm);
         return Engine.Softmax(normalized, axis: lastAxis);
     }

src/AiModelBuilder.cs

@@ -2469,7 +2469,13 @@ T ObjectiveFunction(Dictionary<string, object> trialHyperparameters)
             // train/test split) since cluster structure depends on having all data points.
             // Note: preprocessedX/preprocessedY may only contain the training split in the
             // standard (non-federated) path. For clustering we need the complete dataset.
-            bool useFullData = model is Clustering.Base.ClusteringBase<T>;
+            //
+            // Probe the *unwrapped* model: by this point the local `model` variable may be a
+            // DDP/FSDP/ZeRO* distributed-training wrapper (see lines ~1921-1953), and a wrapper
+            // is never a ClusteringBase<T> so the check would silently flip to false and route
+            // clustering training back through the train/test split — which is exactly the bug
+            // this clustering-data path was added to prevent.
+            bool useFullData = _model is Clustering.Base.ClusteringBase<T>;
             // Clustering models need ALL data points for correct density estimation.
             // Use preparedX/preparedY (the full dataset before train/test split) when
             // the preprocessing pipeline is not configured. When it IS configured,
@@ -2497,6 +2503,23 @@ T ObjectiveFunction(Dictionary<string, object> trialHyperparameters)
             }
             var directX = fullX;
             var directY = fullY;
+
+            // Override the earlier split-based sample counts logged at lines ~2222-2224 so
+            // experiment run metadata reflects the dataset actually fed to Train(). Without
+            // this, clustering runs would be reported with a train/val/test split that was
+            // never used (we just trained on the full dataset).
+            if (useFullData && experimentRun is not null)
+            {
+                int effectiveSamples = InputHelper<T, TInput>.GetInputSize(directX);
+                experimentRun.LogParameters(new Dictionary<string, object>
+                {
+                    ["training_samples"] = effectiveSamples,
+                    ["validation_samples"] = 0,
+                    ["test_samples"] = 0,
+                    ["full_data_used"] = true,
+                });
+            }
+
             model.Train(directX, directY);
 
             // Compute evaluation metrics

src/Audio/Generation/VoiceCraft.cs

@@ -247,6 +247,7 @@ protected override void InitializeLayers()
         if (!_useNativeMode) return;
         if (Architecture.Layers is not null && Architecture.Layers.Count > 0) Layers.AddRange(Architecture.Layers);
         else Layers.AddRange(LayerHelper<T>.CreateDefaultVoiceCraftLayers(
+            Architecture,
             hiddenDim: _options.HiddenDim, numLayers: _options.NumLayers,
             numHeads: _options.NumHeads, codebookSize: _options.CodebookSize,
             dropoutRate: _options.DropoutRate));

src/Augmentation/Image/ImageTensor.cs

@@ -266,8 +266,9 @@ public ImageTensor(int batchSize, int height, int width, int channels = 3, Chann
     /// <returns>A new ImageTensor with copied data.</returns>
     public ImageTensor<T> Clone()
     {
-        // Create a copy of the tensor data
-        var clonedData = new Tensor<T>((int[])_data._shape);
+        // Create a copy of the tensor data — defensive-copy the shape array
+        // so the clone never aliases the original's mutable shape metadata.
+        var clonedData = new Tensor<T>((int[])_data._shape.Clone());
         for (int i = 0; i < _data.Length; i++)
         {
             clonedData[i] = _data[i];
@@ -419,7 +420,7 @@ public ImageTensor<T> Crop(int x, int y, int width, int height)
     /// <returns>The dimensions array.</returns>
     public int[] GetDimensions()
     {
-        return (int[])_data._shape;
+        return (int[])_data._shape.Clone();
     }
 
     /// <summary>

src/Autodiff/Testing/TensorOperationsVerification.cs

@@ -111,13 +111,21 @@ public NumericalGradient<T>.ComparisonResult VerifyUnaryOperation(
     {
         // Compute autodiff gradient via ComputationNode backward
         Tensor<T> autodiffGradient;
+        int[] outputShape;
         {
             var inputNode = TensorOperations<T>.Variable(input.Clone(), "input", requiresGradient: true);
 
             var outputNode = operation(inputNode);
+            if (outputNode.Value is null)
+            {
+                throw new InvalidOperationException(
+                    $"Operation '{operationName}' produced a ComputationNode with a null Value. " +
+                    "Cannot verify gradients against an unevaluated node.");
+            }
+            outputShape = outputNode.Value._shape;
 
             // Create output gradient (ones)
-            var outputGradient = CreateOnes(outputNode.Value._shape);
+            var outputGradient = CreateOnes(outputShape);
             outputNode.Gradient = outputGradient;
 
             // Run backward pass
@@ -126,8 +134,8 @@ public NumericalGradient<T>.ComparisonResult VerifyUnaryOperation(
             autodiffGradient = inputNode.Gradient ?? new Tensor<T>(input._shape);
         }
 
-        // Compute numerical gradient
-        var outputGrad = CreateOnes(input._shape);
+        // Compute numerical gradient using the operation's output shape for the seed gradient
+        var outputGrad = CreateOnes(outputShape);
         var numericalGradient = NumericalGradient<T>.ComputeForOperation(
             input.Clone(),
             outputGrad,
@@ -182,14 +190,22 @@ public NumericalGradient<T>.ComparisonResult VerifyUnaryOperation(
     {
         // Compute autodiff gradients via ComputationNode backward
         Tensor<T> autodiffGrad1, autodiffGrad2;
+        int[] outputShape;
         {
             var node1 = TensorOperations<T>.Variable(input1.Clone(), "input1", requiresGradient: true);
             var node2 = TensorOperations<T>.Variable(input2.Clone(), "input2", requiresGradient: true);
 
             var outputNode = operation(node1, node2);
+            if (outputNode.Value is null)
+            {
+                throw new InvalidOperationException(
+                    $"Operation '{operationName}' produced a ComputationNode with a null Value. " +
+                    "Cannot verify gradients against an unevaluated node.");
+            }
+            outputShape = outputNode.Value._shape;
 
             // Create output gradient (ones)
-            var outputGradient = CreateOnes(outputNode.Value._shape);
+            var outputGradient = CreateOnes(outputShape);
             outputNode.Gradient = outputGradient;
 
             // Run backward pass
@@ -199,8 +215,8 @@ public NumericalGradient<T>.ComparisonResult VerifyUnaryOperation(
             autodiffGrad2 = node2.Gradient ?? new Tensor<T>(input2._shape);
         }
 
-        // Compute numerical gradients
-        var outputGrad = CreateOnes(input1._shape);
+        // Compute numerical gradients using the operation's output shape for the seed gradient
+        var outputGrad = CreateOnes(outputShape);
         var (numericalGrad1, numericalGrad2) = NumericalGradient<T>.ComputeForBinaryOperation(
             input1.Clone(),
             input2.Clone(),