fix: compare non-ASCII digits by numeric value in NaturalStringComparer [patch]

Sorting.Test/NaturalStringComparerTests.cs

@@ -101,6 +101,48 @@ public void Compare_StringsWithLeadingZeros_HandledCorrectly()
 		Assert.IsLessThan(0, _comparer.Compare("file005", "file06")); // 5 < 6 numerically
 	}
 
+	[TestMethod]
+	public void Compare_NonAsciiDigits_ComparedByNumericValue()
+	{
+		// Arabic-Indic five is numerically less than ASCII nine, despite the far greater code point
+		Assert.IsLessThan(0, _comparer.Compare("٥", "9"));
+		Assert.IsGreaterThan(0, _comparer.Compare("9", "٥"));
+
+		// The same holds within a single non-ASCII script
+		Assert.IsLessThan(0, _comparer.Compare("٥", "٩")); // Arabic-Indic 5 < 9
+
+		// Devanagari digits order by value too
+		Assert.IsLessThan(0, _comparer.Compare("५", "३०")); // 5 < 30
+	}
+
+	[TestMethod]
+	public void Compare_NonAsciiDigits_EqualValuesAreEqual()
+	{
+		// Arabic-Indic five and ASCII five spell the same number
+		Assert.AreEqual(0, _comparer.Compare("٥", "5"));
+		Assert.AreEqual(0, _comparer.Compare("file٥", "file5"));
+	}
+
+	[TestMethod]
+	public void Compare_NonAsciiLeadingZeros_NormalizedLikeAsciiZeros()
+	{
+		// An Arabic-Indic zero is a leading zero, so both chunks reduce to the single digit 0
+		Assert.AreEqual(0, _comparer.Compare("٠0", "0"));
+		Assert.AreEqual(0, _comparer.Compare("٠٥", "5"));
+
+		// An all-zeros chunk compares as zero, whatever the script
+		Assert.AreEqual(0, _comparer.Compare("٠٠", "0"));
+		Assert.IsLessThan(0, _comparer.Compare("٠٠", "1"));
+	}
+
+	[TestMethod]
+	public void Compare_MixedScriptDigits_ComparedByNumericValue()
+	{
+		// A single chunk may mix scripts; it still spells one number
+		Assert.IsLessThan(0, _comparer.Compare("file1٥", "file20")); // 15 < 20
+		Assert.AreEqual(0, _comparer.Compare("file1٥", "file15"));
+	}
+
 	[TestMethod]
 	public void Compare_DifferentLengthStrings_ShorterComesFirst()
 	{

Sorting/NaturalStringComparer.cs

@@ -2,18 +2,23 @@
 
 namespace ktsu.Sorting;
 
+using System.Globalization;
 using System.Text.RegularExpressions;
 
 /// <summary>
 /// Comparer that performs a natural comparison between strings, correctly comparing embedded numbers.
 /// </summary>
+/// <remarks>
+/// A run of Unicode decimal digits (category <c>Nd</c>) is a number, whatever script it is written
+/// in, and is compared by the value it spells rather than by its code points.
+/// </remarks>
 public partial class NaturalStringComparer : IComparer<string?>
 {
 	/// <summary>
 	/// Regular expression to match alphanumeric chunks in a string.
 	/// </summary>
 	[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "SYSLIB1045:Convert to 'GeneratedRegexAttribute'.", Justification = "<Pending>")]
 	private static Regex CreateNaturalChunkRegex() => new(@"(\d+)|(\D+)");
 
 	/// <summary>
 	/// Compares two strings using natural sorting, where embedded numbers are compared as numeric values.
@@ -25,7 +30,7 @@
 	/// Zero if x equals y.
 	/// Greater than zero if x is greater than y.
 	/// </returns>
 	public int Compare(string? x, string? y)
 	{
 		if (x == null && y == null)
 		{
@@ -80,27 +85,70 @@
 		return xMatches.Length.CompareTo(yMatches.Length);
 	}
 
+	/// <summary>
+	/// Compares two chunks of decimal digits by the numeric value they spell, rather than by their
+	/// code points.
+	/// </summary>
+	/// <remarks>
+	/// Both chunks come from the <c>\d+</c> alternative of the chunk regex, so every character is a
+	/// Unicode decimal digit (category <c>Nd</c>) and has a decimal value of 0-9. Comparing those
+	/// values, rather than the raw UTF-16 code points, is what keeps non-ASCII digit scripts
+	/// ordering by magnitude: <c>'٥'</c> (Arabic-Indic five) is numerically less than <c>'9'</c>,
+	/// even though its code point is far greater.
+	/// </remarks>
+	/// <param name="xChunk">First digit chunk to compare.</param>
+	/// <param name="yChunk">Second digit chunk to compare.</param>
+	/// <returns>A negative number, zero, or a positive number, as for <see cref="Compare"/>.</returns>
 	private static int CompareNumericChunks(string xChunk, string yChunk)
 	{
-		string xTrimmed = xChunk.TrimStart('0');
-		string yTrimmed = yChunk.TrimStart('0');
+		int xStart = SkipLeadingZeros(xChunk);
+		int yStart = SkipLeadingZeros(yChunk);
 
-		if (xTrimmed.Length == 0)
+		// With leading zeros gone, the chunk spelling more digits is the larger number
+		int xDigits = xChunk.Length - xStart;
+		int yDigits = yChunk.Length - yStart;
+		int lengthComparison = xDigits.CompareTo(yDigits);
+		if (lengthComparison != 0)
 		{
-			xTrimmed = "0";
+			return lengthComparison;
 		}
 
-		if (yTrimmed.Length == 0)
+		// Same digit count, so the first differing digit decides
+		for (int offset = 0; offset < xDigits; offset++)
 		{
-			yTrimmed = "0";
+			int digitComparison = DigitValue(xChunk[xStart + offset]).CompareTo(DigitValue(yChunk[yStart + offset]));
+			if (digitComparison != 0)
+			{
+				return digitComparison;
+			}
 		}
 
-		int lengthComparison = xTrimmed.Length.CompareTo(yTrimmed.Length);
-		if (lengthComparison != 0)
+		return 0;
+	}
+
+	/// <summary>
+	/// Returns the index of the first digit in <paramref name="chunk"/> that is not a zero, or
+	/// <c>chunk.Length - 1</c> when the chunk is all zeros, so a chunk of zeros compares as a
+	/// single zero digit.
+	/// </summary>
+	/// <param name="chunk">The digit chunk to scan.</param>
+	/// <returns>The index at which the chunk's significant digits begin.</returns>
+	private static int SkipLeadingZeros(string chunk)
+	{
+		int index = 0;
+		while (index < chunk.Length - 1 && DigitValue(chunk[index]) == 0)
 		{
-			return lengthComparison;
+			index++;
 		}
 
-		return string.Compare(xTrimmed, yTrimmed, StringComparison.Ordinal);
+		return index;
 	}
+
+	/// <summary>
+	/// Returns the decimal value of a Unicode decimal digit, so that digits from any script compare
+	/// by magnitude.
+	/// </summary>
+	/// <param name="digit">The digit character, which the chunk regex guarantees is category <c>Nd</c>.</param>
+	/// <returns>The digit's value of 0-9.</returns>
+	private static int DigitValue(char digit) => CharUnicodeInfo.GetDecimalDigitValue(digit);
 }