diff --git a/src/Interfaces/IGpuOptimizerConfig.cs b/src/Interfaces/IGpuOptimizerConfig.cs index 1dd2b2373f..eef5969c74 100644 --- a/src/Interfaces/IGpuOptimizerConfig.cs +++ b/src/Interfaces/IGpuOptimizerConfig.cs @@ -56,7 +56,7 @@ public interface IGpuOptimizerConfig /// how to apply its own update, so adding new optimizers doesn't require modifying layer code. /// /// - void ApplyUpdate(IDirectGpuBackend backend, IGpuBuffer param, IGpuBuffer gradient, + void ApplyUpdate(IDirectGpuBackend backend, IGpuBuffer param, IGpuBuffer gradient, GpuOptimizerState state, int size); } diff --git a/src/LossFunctions/DiceLoss.cs b/src/LossFunctions/DiceLoss.cs index 483a4c0a88..b189ccd77e 100644 --- a/src/LossFunctions/DiceLoss.cs +++ b/src/LossFunctions/DiceLoss.cs @@ -78,21 +78,21 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + T intersection = NumOps.Zero; T sumPredicted = NumOps.Zero; T sumActual = NumOps.Zero; - + for (int i = 0; i < predicted.Length; i++) { intersection = NumOps.Add(intersection, NumOps.Multiply(predicted[i], actual[i])); sumPredicted = NumOps.Add(sumPredicted, predicted[i]); sumActual = NumOps.Add(sumActual, actual[i]); } - + T denominator = NumOps.Add(sumPredicted, sumActual); T twoIntersection = NumOps.Multiply(NumOps.FromDouble(2.0), intersection); - + var result = new T[predicted.Length]; for (int i = 0; i < predicted.Length; i++) { @@ -109,9 +109,9 @@ public override Vector CalculateDerivative(Vector predicted, Vector act ) ); } - + return new Vector(result); } - + } diff --git a/src/LossFunctions/FocalLoss.cs b/src/LossFunctions/FocalLoss.cs index 0bb8c48316..fb6bfb5818 100644 --- a/src/LossFunctions/FocalLoss.cs +++ b/src/LossFunctions/FocalLoss.cs @@ -101,30 +101,30 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + var result = new T[predicted.Length]; - + for (int i = 0; i < predicted.Length; i++) { T p = NumericalStabilityHelper.ClampProbability(predicted[i], NumericalStabilityHelper.SmallEpsilon); T y = actual[i]; - + T pt = NumOps.Equals(y, NumOps.One) ? p : NumOps.Subtract(NumOps.One, p); T alphaT = NumOps.Equals(y, NumOps.One) ? _alpha : NumOps.Subtract(NumOps.One, _alpha); - + T focusingTerm = NumOps.Power(NumOps.Subtract(NumOps.One, pt), _gamma); T logPt = NumericalStabilityHelper.SafeLog(pt, NumericalStabilityHelper.SmallEpsilon); - + // Derivative of focal loss with respect to p T gammaFactor = NumOps.Multiply(_gamma, NumOps.Power(NumOps.Subtract(NumOps.One, pt), NumOps.Subtract(_gamma, NumOps.One))); T term1 = NumOps.Multiply(gammaFactor, logPt); T term2 = NumOps.Divide(focusingTerm, pt); - + T grad = NumOps.Multiply(alphaT, NumOps.Subtract(term1, term2)); - + result[i] = NumOps.Equals(y, NumOps.One) ? grad : NumOps.Negate(grad); } - + return new Vector(result).Divide(NumOps.FromDouble(predicted.Length)); } diff --git a/src/LossFunctions/HingeLoss.cs b/src/LossFunctions/HingeLoss.cs index 172f46cf95..f0b6338211 100644 --- a/src/LossFunctions/HingeLoss.cs +++ b/src/LossFunctions/HingeLoss.cs @@ -62,13 +62,13 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + var result = new T[predicted.Length]; - + for (int i = 0; i < predicted.Length; i++) { T margin = NumOps.Subtract(NumOps.One, NumOps.Multiply(actual[i], predicted[i])); - + if (NumOps.GreaterThan(margin, NumOps.Zero)) { // Derivative is -y when margin > 0 @@ -80,7 +80,7 @@ public override Vector CalculateDerivative(Vector predicted, Vector act result[i] = NumOps.Zero; } } - + return new Vector(result).Divide(NumOps.FromDouble(predicted.Length)); } diff --git a/src/LossFunctions/HuberLoss.cs b/src/LossFunctions/HuberLoss.cs index 1e3945048a..c18ffc8a0b 100644 --- a/src/LossFunctions/HuberLoss.cs +++ b/src/LossFunctions/HuberLoss.cs @@ -92,14 +92,14 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + var result = new T[predicted.Length]; - + for (int i = 0; i < predicted.Length; i++) { T diff = NumOps.Subtract(predicted[i], actual[i]); T absDiff = NumOps.Abs(diff); - + if (NumOps.LessThanOrEquals(absDiff, _delta)) { // Quadratic region: derivative is diff @@ -111,7 +111,7 @@ public override Vector CalculateDerivative(Vector predicted, Vector act result[i] = NumOps.Multiply(_delta, NumOps.SignOrZero(diff)); } } - + return new Vector(result).Divide(NumOps.FromDouble(predicted.Length)); } diff --git a/src/LossFunctions/LogCoshLoss.cs b/src/LossFunctions/LogCoshLoss.cs index f061cf8c3b..db8a4f7ad2 100644 --- a/src/LossFunctions/LogCoshLoss.cs +++ b/src/LossFunctions/LogCoshLoss.cs @@ -77,9 +77,9 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + var result = new T[predicted.Length]; - + for (int i = 0; i < predicted.Length; i++) { T diff = NumOps.Subtract(predicted[i], actual[i]); @@ -91,7 +91,7 @@ public override Vector CalculateDerivative(Vector predicted, Vector act NumOps.Add(expPos, expNeg) ); } - + return new Vector(result).Divide(NumOps.FromDouble(predicted.Length)); } diff --git a/src/LossFunctions/LossFunctionBase.cs b/src/LossFunctions/LossFunctionBase.cs index 894b9ad6cf..2ef659eb56 100644 --- a/src/LossFunctions/LossFunctionBase.cs +++ b/src/LossFunctions/LossFunctionBase.cs @@ -54,17 +54,17 @@ public virtual (T Loss, IGpuTensor Gradient) CalculateLossAndGradientGpu(IGpu // Default: fall back to CPU var predictedCpu = predicted.ToTensor(); var actualCpu = actual.ToTensor(); - + var loss = CalculateLoss(predictedCpu.ToVector(), actualCpu.ToVector()); var gradientCpu = CalculateDerivative(predictedCpu.ToVector(), actualCpu.ToVector()); - + var gradientTensor = new Tensor(predictedCpu.Shape); Array.Copy(gradientCpu.ToArray(), gradientTensor.Data, gradientCpu.Length); - + var engine = AiDotNetEngine.Current as DirectGpuTensorEngine; var backend = engine?.GetBackend() ?? throw new InvalidOperationException("GPU backend not available"); var gradientGpu = new GpuTensor(backend, gradientTensor, GpuTensorRole.Gradient); - + return (loss, gradientGpu); } diff --git a/src/LossFunctions/QuantileLoss.cs b/src/LossFunctions/QuantileLoss.cs index a4d7e99d09..646441ebb2 100644 --- a/src/LossFunctions/QuantileLoss.cs +++ b/src/LossFunctions/QuantileLoss.cs @@ -90,13 +90,13 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + var result = new T[predicted.Length]; - + for (int i = 0; i < predicted.Length; i++) { T diff = NumOps.Subtract(actual[i], predicted[i]); - + if (NumOps.GreaterThan(diff, NumOps.Zero)) { // Underestimation: derivative is -quantile @@ -108,7 +108,7 @@ public override Vector CalculateDerivative(Vector predicted, Vector act result[i] = NumOps.Subtract(NumOps.One, _quantile); } } - + return new Vector(result).Divide(NumOps.FromDouble(predicted.Length)); } diff --git a/src/LossFunctions/RootMeanSquaredErrorLoss.cs b/src/LossFunctions/RootMeanSquaredErrorLoss.cs index b289678f22..d0dbba5456 100644 --- a/src/LossFunctions/RootMeanSquaredErrorLoss.cs +++ b/src/LossFunctions/RootMeanSquaredErrorLoss.cs @@ -42,11 +42,11 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + var diff = predicted - actual; var mse = diff.PointwiseMultiply(diff).Average(); var rmse = NumOps.Sqrt(mse); - + if (NumOps.Equals(rmse, NumOps.Zero)) { var zeros = new T[predicted.Length]; @@ -54,10 +54,10 @@ public override Vector CalculateDerivative(Vector predicted, Vector act zeros[i] = NumOps.Zero; return new Vector(zeros); } - + var n = NumOps.FromDouble(predicted.Length); return diff.Divide(NumOps.Multiply(rmse, n)); } - + } diff --git a/src/LossFunctions/SquaredHingeLoss.cs b/src/LossFunctions/SquaredHingeLoss.cs index 38333e263e..0312177100 100644 --- a/src/LossFunctions/SquaredHingeLoss.cs +++ b/src/LossFunctions/SquaredHingeLoss.cs @@ -68,13 +68,13 @@ public override T CalculateLoss(Vector predicted, Vector actual) public override Vector CalculateDerivative(Vector predicted, Vector actual) { ValidateVectorLengths(predicted, actual); - + var result = new T[predicted.Length]; - + for (int i = 0; i < predicted.Length; i++) { T margin = NumOps.Subtract(NumOps.One, NumOps.Multiply(actual[i], predicted[i])); - + if (NumOps.GreaterThan(margin, NumOps.Zero)) { // Derivative is -2 * margin * y when margin > 0 @@ -89,7 +89,7 @@ public override Vector CalculateDerivative(Vector predicted, Vector act result[i] = NumOps.Zero; } } - + return new Vector(result).Divide(NumOps.FromDouble(predicted.Length)); } diff --git a/src/NeuralNetworks/BigGAN.cs b/src/NeuralNetworks/BigGAN.cs index d4336b2d9a..74ddc98315 100644 --- a/src/NeuralNetworks/BigGAN.cs +++ b/src/NeuralNetworks/BigGAN.cs @@ -998,17 +998,17 @@ public Vector CalculateDerivative(Vector predicted, Vector // Fall back to CPU for now var predictedCpu = predicted.ToTensor(); var actualCpu = actual.ToTensor(); - + var loss = CalculateLoss(predictedCpu.ToVector(), actualCpu.ToVector()); var gradientCpu = CalculateDerivative(predictedCpu.ToVector(), actualCpu.ToVector()); - + var gradientTensor = new Tensor(predictedCpu.Shape); Array.Copy(gradientCpu.ToArray(), gradientTensor.Data, gradientCpu.Length); - + var engine = AiDotNetEngine.Current as DirectGpuTensorEngine; var backend = engine?.GetBackend() ?? throw new InvalidOperationException("GPU backend not available"); var gradientGpu = new GpuTensor(backend, gradientTensor, GpuTensorRole.Gradient); - + return (loss, gradientGpu); } } diff --git a/src/NeuralNetworks/NeuralNetworkBase.cs b/src/NeuralNetworks/NeuralNetworkBase.cs index 2517cfab92..2442ebee3a 100644 --- a/src/NeuralNetworks/NeuralNetworkBase.cs +++ b/src/NeuralNetworks/NeuralNetworkBase.cs @@ -797,7 +797,7 @@ public virtual T TrainBatchGpuDeferred( }; T lossValue = NumOps.Zero; - + var backend = gpuEngine.GetBackend() as IAsyncGpuBackend; if (backend == null) { @@ -861,7 +861,7 @@ public virtual async Task TrainBatchGpuDeferredAsync( }; T lossValue = NumOps.Zero; - + var backend = gpuEngine.GetBackend() as IAsyncGpuBackend; if (backend == null) { diff --git a/src/NeuralNetworks/SAGAN.cs b/src/NeuralNetworks/SAGAN.cs index 2d726e6b61..36df74138a 100644 --- a/src/NeuralNetworks/SAGAN.cs +++ b/src/NeuralNetworks/SAGAN.cs @@ -683,17 +683,17 @@ public Vector CalculateDerivative(Vector predicted, Vector // Fall back to CPU for now var predictedCpu = predicted.ToTensor(); var actualCpu = actual.ToTensor(); - + var loss = CalculateLoss(predictedCpu.ToVector(), actualCpu.ToVector()); var gradientCpu = CalculateDerivative(predictedCpu.ToVector(), actualCpu.ToVector()); - + var gradientTensor = new Tensor(predictedCpu.Shape); Array.Copy(gradientCpu.ToArray(), gradientTensor.Data, gradientCpu.Length); - + var engine = AiDotNetEngine.Current as DirectGpuTensorEngine; var backend = engine?.GetBackend() ?? throw new InvalidOperationException("GPU backend not available"); var gradientGpu = new GpuTensor(backend, gradientTensor, GpuTensorRole.Gradient); - + return (loss, gradientGpu); } } diff --git a/src/Optimizers/AMSGradOptimizer.cs b/src/Optimizers/AMSGradOptimizer.cs index 0b9bf7e053..d54abad389 100644 --- a/src/Optimizers/AMSGradOptimizer.cs +++ b/src/Optimizers/AMSGradOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/AdaDeltaOptimizer.cs b/src/Optimizers/AdaDeltaOptimizer.cs index 9416b0abba..bcb56f4be4 100644 --- a/src/Optimizers/AdaDeltaOptimizer.cs +++ b/src/Optimizers/AdaDeltaOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/AdaMaxOptimizer.cs b/src/Optimizers/AdaMaxOptimizer.cs index 5c76889b0b..4733823bac 100644 --- a/src/Optimizers/AdaMaxOptimizer.cs +++ b/src/Optimizers/AdaMaxOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/AdagradOptimizer.cs b/src/Optimizers/AdagradOptimizer.cs index 7b10409546..6c13b2be5b 100644 --- a/src/Optimizers/AdagradOptimizer.cs +++ b/src/Optimizers/AdagradOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/AdamOptimizer.cs b/src/Optimizers/AdamOptimizer.cs index c9f598e21f..ac9df42a73 100644 --- a/src/Optimizers/AdamOptimizer.cs +++ b/src/Optimizers/AdamOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/AdamWOptimizer.cs b/src/Optimizers/AdamWOptimizer.cs index 37877b15cd..5f868a040b 100644 --- a/src/Optimizers/AdamWOptimizer.cs +++ b/src/Optimizers/AdamWOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/FTRLOptimizer.cs b/src/Optimizers/FTRLOptimizer.cs index ffbde01617..ff74b99492 100644 --- a/src/Optimizers/FTRLOptimizer.cs +++ b/src/Optimizers/FTRLOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/GradientDescentOptimizer.cs b/src/Optimizers/GradientDescentOptimizer.cs index c170f010ac..4be399f5cf 100644 --- a/src/Optimizers/GradientDescentOptimizer.cs +++ b/src/Optimizers/GradientDescentOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/LAMBOptimizer.cs b/src/Optimizers/LAMBOptimizer.cs index 2c5be72826..298ad943f1 100644 --- a/src/Optimizers/LAMBOptimizer.cs +++ b/src/Optimizers/LAMBOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/LARSOptimizer.cs b/src/Optimizers/LARSOptimizer.cs index 9a887e9bb3..27d4b5f0d3 100644 --- a/src/Optimizers/LARSOptimizer.cs +++ b/src/Optimizers/LARSOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/LionOptimizer.cs b/src/Optimizers/LionOptimizer.cs index 2ef332bce3..e9d43e9cf1 100644 --- a/src/Optimizers/LionOptimizer.cs +++ b/src/Optimizers/LionOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/MiniBatchGradientDescentOptimizer.cs b/src/Optimizers/MiniBatchGradientDescentOptimizer.cs index 0a03f46901..315b7370db 100644 --- a/src/Optimizers/MiniBatchGradientDescentOptimizer.cs +++ b/src/Optimizers/MiniBatchGradientDescentOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/MomentumOptimizer.cs b/src/Optimizers/MomentumOptimizer.cs index 88c1ce3772..8155095b68 100644 --- a/src/Optimizers/MomentumOptimizer.cs +++ b/src/Optimizers/MomentumOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/NadamOptimizer.cs b/src/Optimizers/NadamOptimizer.cs index 445c3b1a4e..d692d7661c 100644 --- a/src/Optimizers/NadamOptimizer.cs +++ b/src/Optimizers/NadamOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/NesterovAcceleratedGradientOptimizer.cs b/src/Optimizers/NesterovAcceleratedGradientOptimizer.cs index 10e07f6851..f9fcb13e22 100644 --- a/src/Optimizers/NesterovAcceleratedGradientOptimizer.cs +++ b/src/Optimizers/NesterovAcceleratedGradientOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers; diff --git a/src/Optimizers/StochasticGradientDescentOptimizer.cs b/src/Optimizers/StochasticGradientDescentOptimizer.cs index 75403c0757..68d769e3d6 100644 --- a/src/Optimizers/StochasticGradientDescentOptimizer.cs +++ b/src/Optimizers/StochasticGradientDescentOptimizer.cs @@ -1,5 +1,5 @@ -using Newtonsoft.Json; using AiDotNet.Tensors.Engines.DirectGpu; +using Newtonsoft.Json; namespace AiDotNet.Optimizers;