Keep NaturalStringComparer transitive across digit scripts

Sorting.Test/NaturalStringComparerTests.cs

@@ -229,4 +229,72 @@
 
 		CollectionAssert.AreEqual(sorted, unsorted);
 	}
+
+	[TestMethod]
+	public void Compare_NonAsciiDigitAgainstText_OrdersLikeTheAsciiDigit()
+	{
+		// A digit chunk against a text chunk used to compare code points, so the Arabic-Indic five
+		// sorted after letters while the ASCII five, which it equals, sorted before them.
+		Assert.IsLessThan(0, _comparer.Compare("\u0665", "10"));
+		Assert.IsLessThan(0, _comparer.Compare("10", "z"));
+		Assert.IsLessThan(0, _comparer.Compare("\u0665", "z"));
+		Assert.IsGreaterThan(0, _comparer.Compare("z", "\u0665"));
+		Assert.IsGreaterThan(0, _comparer.Compare("\u0665", "-"));
+		Assert.IsGreaterThan(0, _comparer.Compare("5", "-"));
+		Assert.IsLessThan(0, _comparer.Compare("\u0665", "~"));
+		Assert.IsLessThan(0, _comparer.Compare("5", "~"));
+	}
+
+	[TestMethod]
+	public void Compare_SortOfMixedScriptDigitsAndText_DoesNotDependOnInputOrder()
+	{
+		string[] expected = ["\u0665", "10", "z"];
+
+		string[] forward = ["\u0665", "10", "z"];
+		string[] backward = ["z", "10", "\u0665"];
+		Array.Sort(forward, _comparer);
+		Array.Sort(backward, _comparer);
+
+		CollectionAssert.AreEqual(expected, forward);
+		CollectionAssert.AreEqual(expected, backward);
+	}
+
+	[TestMethod]
+	public void Compare_IsTransitiveOverMixedScriptDigitsAndText()
+	{
+		string[] values =
+		[
+			"", "0", "5", "05", "10", "\u0665", "\u0661\u0660", "\U0001D7D7", "\U0001D7CF\U0001D7CE",
+			"\u0969", "a", "z", "Z", " ", "-", "_", "~", "\u00e9", "\U0001F600",
+			"a5", "a\u0665", "a10", "a-", "a z", "5a", "\u0665a", "10a", "-5", "~5",
+		];
+
+		foreach (string a in values)
+		{
+			foreach (string b in values)
+			{
+				Assert.AreEqual(Math.Sign(_comparer.Compare(a, b)), -Math.Sign(_comparer.Compare(b, a)), $"antisymmetry: '{a}' vs '{b}'");
+
+				foreach (string c in values)
+				{
+					int ab = Math.Sign(_comparer.Compare(a, b));
+					int bc = Math.Sign(_comparer.Compare(b, c));
+					if (ab <= 0 && bc <= 0)
+					{
+						Assert.IsLessThanOrEqualTo(0, _comparer.Compare(a, c), $"transitivity: '{a}' <= '{b}' <= '{c}'");
+					}
+				}
+			}
+		}
+	}
+
+	[TestMethod]
+	public void Compare_DigitsOutsideTheBasicMultilingualPlane_ComparedByNumericValue()
+	{
+		// Mathematical bold digits nine, and one followed by zero, are surrogate pairs in UTF-16.
+		Assert.IsLessThan(0, _comparer.Compare("file\U0001D7D7", "file\U0001D7CF\U0001D7CE"));
+		Assert.IsLessThan(0, _comparer.Compare("file\U0001D7D7", "file10"));
+		Assert.IsGreaterThan(0, _comparer.Compare("file\U0001D7D7", "file8"));
+		Assert.AreEqual(0, _comparer.Compare("file\U0001D7D7", "file9"));
+	}
 }

Sorting/NaturalStringComparer.cs

@@ -3,23 +3,19 @@
 namespace ktsu.Sorting;
 
 using System.Globalization;
-using System.Text.RegularExpressions;
+using System.Text;
 
 /// <summary>
 /// Comparer that performs a natural comparison between strings, correctly comparing embedded numbers.
 /// </summary>
 /// <remarks>
 /// A run of Unicode decimal digits (category <c>Nd</c>) is a number, whatever script it is written
-/// in, and is compared by the value it spells rather than by its code points.
+/// in and whether or not it lies outside the Basic Multilingual Plane, and is compared by the value
+/// it spells rather than by its code points. A number compared with text orders exactly as the
+/// equivalent ASCII digits would, so every digit script sorts in the same place relative to text.
 /// </remarks>
 public partial class NaturalStringComparer : IComparer<string?>
 {
-	/// <summary>
-	/// Regular expression to match alphanumeric chunks in a string.
-	/// </summary>
-	[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "SYSLIB1045:Convert to 'GeneratedRegexAttribute'.", Justification = "<Pending>")]
-	private static Regex CreateNaturalChunkRegex() => new(@"(\d+)|(\D+)");
-
 	/// <summary>
 	/// Compares two strings using natural sorting, where embedded numbers are compared as numeric values.
 	/// </summary>
@@ -30,7 +26,7 @@
 	/// Zero if x equals y.
 	/// Greater than zero if x is greater than y.
 	/// </returns>
 	public int Compare(string? x, string? y)
 	{
 		if (x == null && y == null)
 		{
@@ -52,28 +48,31 @@
 			return 0;
 		}
 
-		Regex regex = CreateNaturalChunkRegex();
-		Match[] xMatches = [.. regex.Matches(x).Cast<Match>()];
-		Match[] yMatches = [.. regex.Matches(y).Cast<Match>()];
+		List<Chunk> xChunks = SplitIntoChunks(x);
+		List<Chunk> yChunks = SplitIntoChunks(y);
 
 		int i = 0, j = 0;
-		while (i < xMatches.Length && j < yMatches.Length)
+		while (i < xChunks.Count && j < yChunks.Count)
 		{
-			Match xMatch = xMatches[i++];
-			Match yMatch = yMatches[j++];
+			Chunk xChunk = xChunks[i++];
+			Chunk yChunk = yChunks[j++];
 
 			// If both chunks are numeric, compare them as numbers
-			if (char.IsDigit(xMatch.Value[0]) && char.IsDigit(yMatch.Value[0]))
+			if (xChunk.IsNumeric && yChunk.IsNumeric)
 			{
-				int numComparison = CompareNumericChunks(xMatch.Value, yMatch.Value);
+				int numComparison = CompareNumericChunks(xChunk.Value, yChunk.Value);
 				if (numComparison != 0)
 				{
 					return numComparison;
 				}
 			}
 			else // Otherwise, compare them as strings
 			{
-				int stringComparison = string.Compare(xMatch.Value, yMatch.Value, StringComparison.Ordinal);
+				// A numeric chunk holds ASCII digits and a text chunk never starts with a digit, so a
+				// number against text is decided by the first character, the same way for every
+				// digit script. Comparing a non-ASCII digit's own code point here would sort it after
+				// letters while its ASCII equal sorts before them, which makes the order intransitive.
+				int stringComparison = string.Compare(xChunk.Value, yChunk.Value, StringComparison.Ordinal);
 				if (stringComparison != 0)
 				{
 					return stringComparison;
@@ -82,19 +81,65 @@
 		}
 
 		// If we've exhausted one sequence but not the other, the shorter one comes first
-		return xMatches.Length.CompareTo(yMatches.Length);
+		return xChunks.Count.CompareTo(yChunks.Count);
+	}
+
+	/// <summary>
+	/// Splits a string into alternating runs of decimal digits and of other text, walking it by code
+	/// point so that a digit encoded as a surrogate pair still counts as a digit.
+	/// </summary>
+	/// <remarks>
+	/// A numeric chunk's value is rewritten in ASCII digits, so that numbers from every script compare
+	/// alike both with each other and with text. A text chunk keeps its original characters.
+	/// </remarks>
+	/// <param name="value">The string to split.</param>
+	/// <returns>The chunks of <paramref name="value"/>, in order.</returns>
+	private static List<Chunk> SplitIntoChunks(string value)
+	{
+		List<Chunk> chunks = [];
+		StringBuilder current = new();
+		bool currentIsNumeric = false;
+		int index = 0;
+		while (index < value.Length)
+		{
+			bool isDigit = CharUnicodeInfo.GetUnicodeCategory(value, index) == UnicodeCategory.DecimalDigitNumber;
+			if (current.Length > 0 && isDigit != currentIsNumeric)
+			{
+				chunks.Add(new Chunk(currentIsNumeric, current.ToString()));
+				current.Clear();
+			}
+
+			currentIsNumeric = isDigit;
+			int width = char.IsSurrogatePair(value, index) ? 2 : 1;
+			if (isDigit)
+			{
+				current.Append((char)('0' + CharUnicodeInfo.GetDecimalDigitValue(value, index)));
+			}
+			else
+			{
+				current.Append(value, index, width);
+			}
+
+			index += width;
+		}
+
+		if (current.Length > 0)
+		{
+			chunks.Add(new Chunk(currentIsNumeric, current.ToString()));
+		}
+
+		return chunks;
 	}
 
 	/// <summary>
 	/// Compares two chunks of decimal digits by the numeric value they spell, rather than by their
 	/// code points.
 	/// </summary>
 	/// <remarks>
-	/// Both chunks come from the <c>\d+</c> alternative of the chunk regex, so every character is a
-	/// Unicode decimal digit (category <c>Nd</c>) and has a decimal value of 0-9. Comparing those
-	/// values, rather than the raw UTF-16 code points, is what keeps non-ASCII digit scripts
-	/// ordering by magnitude: <c>'٥'</c> (Arabic-Indic five) is numerically less than <c>'9'</c>,
-	/// even though its code point is far greater.
+	/// Both chunks come from <see cref="SplitIntoChunks"/>, which has already rewritten every Unicode
+	/// decimal digit (category <c>Nd</c>) as the ASCII digit of the same value. That is what keeps
+	/// non-ASCII digit scripts ordering by magnitude: <c>'٥'</c> (Arabic-Indic five) is numerically
+	/// less than <c>'9'</c>, even though its code point is far greater.
 	/// </remarks>
 	/// <param name="xChunk">First digit chunk to compare.</param>
 	/// <param name="yChunk">Second digit chunk to compare.</param>
@@ -116,7 +161,7 @@
 		// Same digit count, so the first differing digit decides
 		for (int offset = 0; offset < xDigits; offset++)
 		{
-			int digitComparison = DigitValue(xChunk[xStart + offset]).CompareTo(DigitValue(yChunk[yStart + offset]));
+			int digitComparison = xChunk[xStart + offset].CompareTo(yChunk[yStart + offset]);
 			if (digitComparison != 0)
 			{
 				return digitComparison;
@@ -136,7 +181,7 @@
 	private static int SkipLeadingZeros(string chunk)
 	{
 		int index = 0;
-		while (index < chunk.Length - 1 && DigitValue(chunk[index]) == 0)
+		while (index < chunk.Length - 1 && chunk[index] == '0')
 		{
 			index++;
 		}
@@ -145,10 +190,9 @@
 	}
 
 	/// <summary>
-	/// Returns the decimal value of a Unicode decimal digit, so that digits from any script compare
-	/// by magnitude.
+	/// A run of decimal digits, held as ASCII digits, or a run of other text.
 	/// </summary>
-	/// <param name="digit">The digit character, which the chunk regex guarantees is category <c>Nd</c>.</param>
-	/// <returns>The digit's value of 0-9.</returns>
-	private static int DigitValue(char digit) => CharUnicodeInfo.GetDecimalDigitValue(digit);
+	/// <param name="IsNumeric">Whether the chunk is a run of decimal digits.</param>
+	/// <param name="Value">The chunk's ASCII digits when numeric, otherwise its original text.</param>
+	private readonly record struct Chunk(bool IsNumeric, string Value);
 }