diff --git a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu
index b15c82ac9a1bc..946d24f6b7d58 100644
--- a/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu
+++ b/onnxruntime/core/providers/cuda/object_detection/non_max_suppression_impl.cu
@@ -16,11 +16,14 @@ limitations under the License.
 
 #include "core/providers/cuda/cu_inc/common.cuh"
 
+#include <limits>
+
 #include <thrust/count.h>
 #include <thrust/device_vector.h>
 #include <thrust/execution_policy.h>
 
 #include "non_max_suppression_impl.h"
+#include "core/common/safeint.h"
 #include "core/providers/cpu/object_detection/non_max_suppression_helper.h"
 #include "core/providers/cuda/cuda_common.h"
 
@@ -241,15 +244,18 @@ Status NmsGpu(cudaStream_t stream,
   auto iptr = reinterpret_cast<std::uintptr_t>(d_sorted_boxes_float_ptr);
   ORT_ENFORCE((iptr & 15) == 0);
 
-  const int bit_mask_len =
-      (num_boxes + kNmsBoxesPerThread - 1) / kNmsBoxesPerThread;
-  int max_nms_mask_size = num_boxes * bit_mask_len;
+  const int bit_mask_len = num_boxes == 0 ? 0 : 1 + (num_boxes - 1) / kNmsBoxesPerThread;
+  const size_t max_nms_mask_size = SafeInt<size_t>(num_boxes) * bit_mask_len;
+  ORT_RETURN_IF_NOT(max_nms_mask_size <= static_cast<size_t>(std::numeric_limits<int>::max()),
+                    "CUDA NonMaxSuppression mask size exceeds the int index range.");
+  const int max_nms_mask_size_int = static_cast<int>(max_nms_mask_size);
 
-  IAllocatorUniquePtr<void> d_nms_mask_ptr{allocator(max_nms_mask_size * sizeof(int))};
+  IAllocatorUniquePtr<void> d_nms_mask_ptr{allocator(SafeInt<size_t>(max_nms_mask_size) * sizeof(int))};
   auto* d_nms_mask = static_cast<int*>(d_nms_mask_ptr.get());
 
-  int blocksPerGrid = (int)(ceil(static_cast<float>(max_nms_mask_size) / GridDim::maxThreadsPerBlock));
-  SetZero<int><<<blocksPerGrid, GridDim::maxThreadsPerBlock, 0, stream>>>(max_nms_mask_size, d_nms_mask);
+  int blocksPerGrid = static_cast<int>((max_nms_mask_size + GridDim::maxThreadsPerBlock - 1) /
+                                       GridDim::maxThreadsPerBlock);
+  SetZero<int><<<blocksPerGrid, GridDim::maxThreadsPerBlock, 0, stream>>>(max_nms_mask_size_int, d_nms_mask);
 
   int* d_delete_mask = d_nms_mask;
   int* h_selected_count = h_nkeep;
diff --git a/onnxruntime/test/providers/cpu/object_detection/non_max_suppression_test.cc b/onnxruntime/test/providers/cpu/object_detection/non_max_suppression_test.cc
index e642f92ee9728..dfa7f8235f7dc 100644
--- a/onnxruntime/test/providers/cpu/object_detection/non_max_suppression_test.cc
+++ b/onnxruntime/test/providers/cpu/object_detection/non_max_suppression_test.cc
@@ -4,6 +4,10 @@
 #include "gtest/gtest.h"
 #include "test/providers/provider_test_utils.h"
 
+#ifdef USE_CUDA
+#include "test/util/include/default_providers.h"
+#endif
+
 namespace onnxruntime {
 namespace test {
 
@@ -404,5 +408,28 @@ TEST(NonMaxSuppressionOpTest, WithIOUThresholdOpset11) {
   test.Run();
 }
 
+#ifdef USE_CUDA
+TEST(NonMaxSuppressionOpTest, CudaRejectsMaskSizeOutsideIntRange) {
+  auto cuda_provider = DefaultCudaExecutionProvider();
+  if (cuda_provider == nullptr) {
+    GTEST_SKIP() << "CUDA execution provider is not available.";
+  }
+
+  constexpr int64_t num_boxes = 262144;
+  OpTester test("NonMaxSuppression", 11, kOnnxDomain);
+  test.AddInput<float>("boxes", {1, num_boxes, 4}, std::vector<float>(num_boxes * 4));
+  test.AddInput<float>("scores", {1, 1, num_boxes}, std::vector<float>(num_boxes));
+  test.AddInput<int64_t>("max_output_boxes_per_class", {}, {1});
+  test.AddInput<float>("iou_threshold", {}, {0.5f});
+  test.AddOptionalInputEdge<float>();
+  test.AddOutput<int64_t>("selected_indices", {0, 3}, {});
+
+  std::vector<std::unique_ptr<IExecutionProvider>> execution_providers;
+  execution_providers.push_back(std::move(cuda_provider));
+  test.Run(OpTester::ExpectResult::kExpectFailure, "mask size exceeds the int index range",
+           {}, nullptr, &execution_providers);
+}
+#endif
+
 }  // namespace test
 }  // namespace onnxruntime