Use System.Text.Json for NativeAOT-critical NuGet JSON APIs

build/Shared/Utf8JsonReaderExtensions.cs

@@ -1,16 +1,15 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-#nullable disable
-
 using System;
+using System.Text;
 using System.Text.Json;
 
 namespace NuGet.Shared
 {
     internal static class Utf8JsonReaderExtensions
     {
-        internal static string ReadTokenAsString(this ref Utf8JsonReader reader)
+        internal static string? ReadTokenAsString(this ref Utf8JsonReader reader)
         {
             switch (reader.TokenType)
             {
@@ -32,11 +31,11 @@ internal static string ReadTokenAsString(this ref Utf8JsonReader reader)
 
         private static string ReadNumberAsString(this ref Utf8JsonReader reader)
         {
-            if (reader.TryGetInt64(out long value))
-            {
-                return value.ToString();
-            }
-            return reader.GetDouble().ToString();
+#if NET5_0_OR_GREATER
+            return Encoding.UTF8.GetString(reader.ValueSpan);
+#else
+            return Encoding.UTF8.GetString(reader.ValueSpan.ToArray());
+#endif
         }
     }
 }

build/Shared/Utf8JsonStreamReader.cs

@@ -16,7 +16,7 @@
 namespace NuGet.Shared
 {
     /// <summary>
-    /// This struct is used to read over a memeory stream in parts, in order to avoid reading the entire stream into memory.
+    /// This struct is used to read over a memory stream in parts, in order to avoid reading the entire stream into memory.
     /// It functions as a wrapper around <see cref="Utf8JsonStreamReader"/>, while maintaining a stream and a buffer to read from.
     /// </summary>
     internal ref struct Utf8JsonStreamReader
@@ -40,7 +40,18 @@ internal ref struct Utf8JsonStreamReader
         private bool _disposed;
         private ArrayPool<byte> _bufferPool;
         private int _bufferUsed = 0;
+        private int _bufferStartLineNumber;
+        private int _bufferStartBytePositionInLine;
 
+        /// <summary>
+        /// A buffered reader that reads from a stream in chunks, and uses a Utf8JsonReader to read the json content from the buffer.
+        /// The reader will advance the underlying stream, but will not dispose it.
+        /// </summary>
+        /// <param name="stream"></param>
+        /// <param name="bufferSize"></param>
+        /// <param name="arrayPool"></param>
+        /// <exception cref="ArgumentNullException"></exception>
+        /// <exception cref="ArgumentException"></exception>
         internal Utf8JsonStreamReader(Stream stream, int bufferSize = BufferSizeDefault, ArrayPool<byte> arrayPool = null)
         {
             if (stream is null)
@@ -57,6 +68,8 @@ internal Utf8JsonStreamReader(Stream stream, int bufferSize = BufferSizeDefault,
             _buffer = _bufferPool.Rent(bufferSize);
             _disposed = false;
             _stream = stream;
+            _bufferStartLineNumber = 0;
+            _bufferStartBytePositionInLine = 0;
 
             if (_stream.Read(_buffer, offset: 0, count: 1) == 1 &&
                 _stream.Read(_buffer, offset: ++_bufferUsed, count: 1) == 1 &&
@@ -82,6 +95,24 @@ internal Utf8JsonStreamReader(Stream stream, int bufferSize = BufferSizeDefault,
 
         internal JsonTokenType TokenType => _reader.TokenType;
 
+        internal int LineNumber
+        {
+            get
+            {
+                GetTokenStartPosition(out int lineNumber, out _);
+                return lineNumber + 1;
+            }
+        }
+
+        internal int ColumnNumber
+        {
+            get
+            {
+                GetTokenStartPosition(out _, out int bytePositionInLine);
+                return bytePositionInLine + 1;
+            }
+        }
+
         internal bool ValueTextEquals(ReadOnlySpan<byte> utf8Text) => _reader.ValueTextEquals(utf8Text);
 
         internal bool TryGetInt32(out int value) => _reader.TryGetInt32(out value);
@@ -92,6 +123,8 @@ internal Utf8JsonStreamReader(Stream stream, int bufferSize = BufferSizeDefault,
 
         internal int GetInt32() => _reader.GetInt32();
 
+        internal string ReadTokenAsString() => _reader.ReadTokenAsString();
+
         internal int CurrentDepth => _reader.CurrentDepth;
 
         internal bool Read()
@@ -334,11 +367,17 @@ internal IReadOnlyList<string> ReadStringArrayAsReadOnlyListFromArrayStart()
         // This function is called when Read() returns false and we're not already in the final block
         private void GetMoreBytesFromStream()
         {
-            if (_reader.BytesConsumed < _bufferUsed)
+            int bytesConsumed = checked((int)_reader.BytesConsumed);
+            AdvancePosition(
+                _buffer.AsSpan(start: 0, length: bytesConsumed),
+                ref _bufferStartLineNumber,
+                ref _bufferStartBytePositionInLine);
+
+            if (bytesConsumed < _bufferUsed)
             {
                 // If the number of bytes consumed by the reader is less than the amount set in the buffer then we have leftover bytes
                 var oldBuffer = _buffer;
-                ReadOnlySpan<byte> leftover = oldBuffer.AsSpan((int)_reader.BytesConsumed);
+                ReadOnlySpan<byte> leftover = oldBuffer.AsSpan(bytesConsumed);
                 _bufferUsed = leftover.Length;
 
                 // If the leftover bytes are the same as the buffer size then we are at capacity and need to double the buffer size
@@ -361,6 +400,33 @@ private void GetMoreBytesFromStream()
             ReadStreamIntoBuffer(_reader.CurrentState);
         }
 
+        private void GetTokenStartPosition(out int lineNumber, out int bytePositionInLine)
+        {
+            lineNumber = _bufferStartLineNumber;
+            bytePositionInLine = _bufferStartBytePositionInLine;
+
+            AdvancePosition(
+                _buffer.AsSpan(start: 0, length: checked((int)_reader.TokenStartIndex)),
+                ref lineNumber,
+                ref bytePositionInLine);
+        }
+
+        private static void AdvancePosition(
+            ReadOnlySpan<byte> bytes,
+            ref int lineNumber,
+            ref int bytePositionInLine)
+        {
+            int newlineIndex;
+            while ((newlineIndex = bytes.IndexOf((byte)'\n')) >= 0)
+            {
+                lineNumber++;
+                bytePositionInLine = 0;
+                bytes = bytes.Slice(newlineIndex + 1);
+            }
+
+            bytePositionInLine += bytes.Length;
+        }
+
         /// <summary>
         /// Loops through the stream and reads it into the buffer until the buffer is full or the stream is empty, creates the Utf8JsonReader.
         /// </summary>

docs/aot-compatibility.md

@@ -1,11 +1,14 @@
 # AOT and Trimming Compatibility
 
-NuGet has `IsAotCompatible` enabled for all NuGet.Core libraries and the code itself is AOT compatible.
-However, NuGet still utilizes Newtonsoft.Json for deserialization, which uses reflection and is not AOT/trim compatible.
+NuGet enables `IsAotCompatible` for all NuGet.Core libraries.
+Some NuGet readers and writers still use Newtonsoft.Json, which uses reflection and is not trimming compatible.
 
-We are in the process of migrating to System.Text.Json source-generated deserialization.
-Until the migration is complete, both deserialization paths coexist, gated under a feature switch.
-Enabling the feature switch ensures NuGet.Protocol uses System.Text.Json instead of Newtonsoft.Json, allowing the linker to trim the Newtonsoft.Json code path entirely.
+NuGet is migrating its JSON readers and selected writers to System.Text.Json.
+During this migration, some Newtonsoft.Json and System.Text.Json implementations coexist behind a feature switch.
+The switch selects available System.Text.Json readers in NuGet.Protocol, NuGet.Packaging, and NuGet.ProjectModel.
+Readers without a System.Text.Json implementation continue to use Newtonsoft.Json.
+The `global.json` and runtime graph readers, and the `packages.lock.json` file, stream, and string APIs use System.Text.Json directly.
+The obsolete `packages.lock.json` `TextWriter` API continues to use Newtonsoft.Json for compatibility.
 
 ## Using NuGet in a Native AOT Application
 
@@ -19,4 +22,6 @@ If you consume NuGet libraries in a native AOT app, add the following feature sw
 </ItemGroup>
 ```
 
-This tells NuGet to use the AOT-safe System.Text.Json path and tells the linker the value is constant so it can eliminate the Newtonsoft.Json code path from the binary.
+This option selects the AOT-compatible System.Text.Json readers that remain behind the feature switch.
+The `Trim` value tells the linker that the switch value is constant.
+The linker can then remove Newtonsoft.Json paths that the application does not use.