diff --git a/onnxruntime/core/framework/session_state_utils.cc b/onnxruntime/core/framework/session_state_utils.cc
index 343d634b44691..9d45ec38e5a32 100644
--- a/onnxruntime/core/framework/session_state_utils.cc
+++ b/onnxruntime/core/framework/session_state_utils.cc
@@ -81,6 +81,11 @@ static common::Status ExtDataTensorProtoToTensor(const Env& env,
   ORT_RETURN_IF_ERROR(utils::GetExtDataFromTensorProto(env, proto_path.c_str(), tensor_proto,
                                                        ext_data_buf, ext_data_len, ext_data_deleter,
                                                        buffered_tensor, &prepacked_for_graph));
+  if constexpr (endian::native != endian::little) {
+    if (!proto_path.empty() && (proto_path.compare(onnxruntime::utils::kTensorProtoMemoryAddressTag) != 0)) {
+      utils::ConvertRawDataInTensorProto(const_cast<ONNX_NAMESPACE::TensorProto*>(&tensor_proto), ext_data_buf, ext_data_len);
+    }
+  }
 
   // NB: creating a do-nothing allocator per tensor is wasteful; can perhaps be
   // avoided if the Tensor class implements the do-nothing behavior when given a
diff --git a/onnxruntime/core/framework/tensorprotoutils.cc b/onnxruntime/core/framework/tensorprotoutils.cc
index ae1ec2e53bd7c..94a2a6677358e 100644
--- a/onnxruntime/core/framework/tensorprotoutils.cc
+++ b/onnxruntime/core/framework/tensorprotoutils.cc
@@ -270,10 +270,15 @@ void SetRawDataInTensorProto(ONNX_NAMESPACE::TensorProto& tensor_proto, std::str
   tensor_proto.set_raw_data(std::move(param));
 }
 
-void ConvertRawDataInTensorProto(TensorProto* tensor) {
+void ConvertRawDataInTensorProto(TensorProto* tensor,
+                                 void* ext_data_buf,
+                                 size_t ext_data_len) {
   size_t element_size = 1;
   char* bytes = NULL;
   size_t num_elements = 0;
+  if (ext_data_buf && !ext_data_len) {
+    return;
+  }
   switch (tensor->data_type()) {
     case TensorProto_DataType_FLOAT:
       bytes = reinterpret_cast<char*>(tensor->mutable_float_data()->mutable_data());
@@ -337,6 +342,15 @@ void ConvertRawDataInTensorProto(TensorProto* tensor) {
     num_elements = (tensor->raw_data().size()) / element_size;
     bytes = const_cast<char*>(tensor->mutable_raw_data()->c_str());
   }
+
+  if (element_size == 1) {
+    return;
+  }
+  if (ext_data_buf) {
+    ORT_ENFORCE(ext_data_len % element_size == 0);
+    num_elements = ext_data_len / element_size;
+    bytes = reinterpret_cast<char*>(ext_data_buf);
+  }
   for (size_t i = 0; i < num_elements; ++i) {
     char* start_byte = bytes + i * element_size;
     char* end_byte = start_byte + element_size - 1;
diff --git a/onnxruntime/core/framework/tensorprotoutils.h b/onnxruntime/core/framework/tensorprotoutils.h
index f5dec7ae988f2..79eae48c10411 100644
--- a/onnxruntime/core/framework/tensorprotoutils.h
+++ b/onnxruntime/core/framework/tensorprotoutils.h
@@ -41,12 +41,18 @@ Status GetExternalDataInfo(const ONNX_NAMESPACE::TensorProto& tensor_proto,
                            ExternalDataInfo::PrepackedInfos* prepacked_infos = nullptr);
 /**
  * This function is used to convert the endianess of Tensor data.
+ * If ext_data_buf is provided, then this buffer content's endianess
+ * will be changed.
  * Mostly, will be used in big endian system to support the model file
  * generated on little endian system.
- * @param initializer       given initializer tensor
+ * @param tensor_proto given initializer tensor
+ * @param ext_data_buf optional externl data buffer
+ * @param ext_data_len optional externl data buffer lengeh
  * @returns                 None
  */
-void ConvertRawDataInTensorProto(ONNX_NAMESPACE::TensorProto* initializer);
+void ConvertRawDataInTensorProto(ONNX_NAMESPACE::TensorProto* tensor_proto,
+                                 void* ext_data_buf = NULL,
+                                 size_t ext_data_len = 0);
 
 /**
  * Wrapper function for set_raw_data.