From 999e89c2be0b0961eaaeca8e4eed42a1dfa22831 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Mon, 14 Sep 2026 17:01:31 +0300 Subject: [PATCH 1/2] feat: read and write delimited text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CSV file is a list of rows, which the mapper already understands — a list walking "" and a list-shaped output are both features a JSON bare array needed. So the engine is untouched: this is a reader and a writer. Delimiter and header row are asked per side rather than sniffed, because one client sends comma, semicolon and pipe, two of them with no header at all. Every cell is read as text, so 041800 and 0.100 arrive as written. A file holding several record types needs no feature of its own: the filter a list already has selects them by their first field. Writing a partner's file also needed two things the mapper had no way to say, both useful beyond this format: entries written after the walked rows, and a source that counts the rows produced — which is what a trailer record carries. --- .../Resources/MappingPreviews/Preview.cs | 4 +- .../Mapper/DocumentMapper.cs | 85 +++- .../Mapper/Formats/CsvFormat.cs | 303 +++++++++++ .../Mapper/Formats/CsvOptions.cs | 53 ++ .../Mapper/Formats/DocumentFormats.cs | 29 +- .../Mapper/MappingRules.cs | 43 ++ .../Mapper/NativeMapper.cs | 8 +- .../SW.Bitween.NativeAdapters.csproj | 1 + .../NativeMapper/CsvFormatTests.cs | 470 ++++++++++++++++++ .../NativeMapper/CsvMappingTests.cs | 439 ++++++++++++++++ .../ClientApp/e2e/mapper-csv.spec.ts | 268 ++++++++++ SW.Bitween.Web/ClientApp/package.json | 2 + .../nativeMapper/BuildFromSample.tsx | 4 +- .../src/components/nativeMapper/EntryRow.tsx | 18 +- .../nativeMapper/NativeMapperEditor.tsx | 95 +++- .../src/components/nativeMapper/OutputRow.tsx | 7 +- .../nativeMapper/OutputTreeView.tsx | 42 +- .../src/components/nativeMapper/ValueCell.tsx | 12 + .../__tests__/csvSampleTree.test.ts | 176 +++++++ .../__tests__/documentTree.test.ts | 4 +- .../nativeMapper/__tests__/outputTree.test.ts | 33 ++ .../lib/nativeMapper/__tests__/rules.test.ts | 78 +++ .../src/lib/nativeMapper/documentTree.ts | 102 ++++ .../src/lib/nativeMapper/outputTree.ts | 41 +- .../src/lib/nativeMapper/rulesReducer.ts | 53 +- .../src/lib/nativeMapper/serialize.ts | 18 +- .../ClientApp/src/lib/nativeMapper/types.ts | 85 +++- SW.Bitween.Web/ClientApp/yarn.lock | 36 ++ 28 files changed, 2438 insertions(+), 71 deletions(-) create mode 100644 SW.Bitween.NativeAdapters/Mapper/Formats/CsvFormat.cs create mode 100644 SW.Bitween.NativeAdapters/Mapper/Formats/CsvOptions.cs create mode 100644 SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs create mode 100644 SW.Bitween.UnitTests/NativeMapper/CsvMappingTests.cs create mode 100644 SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts create mode 100644 SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/csvSampleTree.test.ts diff --git a/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs b/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs index f18d96b9..4d34e577 100644 --- a/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs +++ b/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs @@ -83,10 +83,10 @@ public async Task Handle(MappingPreviewRequest request) $"Bitween understands up to version {MappingRules.CurrentVersion}.", }; - if (!DocumentFormats.TryGet(rules.SourceFormat, out var source)) + if (!DocumentFormats.TryGet(rules.SourceFormat, out var source, rules.SourceCsv)) return new MappingPreviewResponse { Error = DocumentFormats.Unsupported(rules.SourceFormat, "source") }; - if (!DocumentFormats.TryGet(rules.TargetFormat, out var target)) + if (!DocumentFormats.TryGet(rules.TargetFormat, out var target, rules.TargetCsv)) return new MappingPreviewResponse { Error = DocumentFormats.Unsupported(rules.TargetFormat, "target") }; ValueNode input; diff --git a/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs b/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs index 0f9e5f8f..1864a05e 100644 --- a/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs +++ b/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs @@ -98,7 +98,8 @@ private static void MapInto( MappingContext context, SourceTraits traits, List errors, - string path) + string path, + int? written = null) { foreach (var field in fields) { @@ -110,7 +111,7 @@ private static void MapInto( continue; } - if (!TryResolveField(field, scope, context, traits, out var value, out var reason)) + if (!TryResolveField(field, scope, context, traits, written, out var value, out var reason)) { errors.Add(new MappingError(target, reason!)); continue; @@ -134,12 +135,20 @@ private static void MapInto( } /// - /// Builds the list a rule produces: its fixed entries, then one per entry of the source list - /// it walks. + /// Builds the list a rule produces: the entries written before it, one per entry of the source + /// list it walks, then the entries written after. /// /// - /// Fixed entries come first because that is where a header line belongs, and because it is - /// the order the previous mapper produced for the same configuration. + /// + /// The order is the feature. A header line belongs at the top and a trailer at the bottom, and + /// a partner's file routinely ends with a record saying how many came before it. + /// + /// + /// Which is why the source entries are matched before anything is built. The number of rows is + /// then the same wherever is read from — the header, a row, + /// or the trailer — rather than a running total that means something different depending on + /// where it sits. + /// /// private static ListNode BuildList( ListRule rule, @@ -150,19 +159,51 @@ private static ListNode BuildList( string path) { var target = Describe(path, rule.Target); + var rows = Matching(rule, scope, traits, errors, target); var list = ValueNode.List(); - // Read against the scope the list sits in, since a fixed entry has no entry of its own. + // Read against the scope the list sits in, since a written entry has no entry of its own. foreach (var entry in rule.Fixed) - AddEntry(list, entry.Item, entry.Fields, entry.Lists, scope, context, traits, errors, target); + AddEntry(list, entry.Item, entry.Fields, entry.Lists, scope, context, traits, errors, + target, rows.Count); + + foreach (var item in rows) + AddEntry(list, rule.Item, rule.Fields, rule.Lists, scope.Enter(item), context, traits, + errors, target, rows.Count); + + // A trailer belongs on the file whether the source list had a thousand entries, none, or + // was never there at all — a partner expecting a record count still expects to be told + // that it is zero. + foreach (var entry in rule.After) + AddEntry(list, entry.Item, entry.Fields, entry.Lists, scope, context, traits, errors, + target, rows.Count); - // No source list to walk: the list is whatever its fixed entries produced. - if (rule.Over is null) return list; + return list; + } + + /// + /// The entries of the source list that this rule's condition lets through. + /// + /// + /// Separated from building them so that how many there are is known first. Nothing is produced + /// here; a rule that fails does so when its entry is built, as it always did. + /// + private static List Matching( + ListRule rule, + Scope scope, + SourceTraits traits, + List errors, + string target) + { + var matched = new List(); + + // No source list to walk: the list is whatever its written entries produce. + if (rule.Over is null) return matched; // A path that is absent adds nothing rather than failing. An order with no lines is // ordinary; so is an optional section. var over = Values.Resolve(scope.Current, rule.Over); - if (over is null) return list; + if (over is null) return matched; // XML makes a list by repeating a name, so an order with one is the same document as // one whose `line` was never a list. Reading that as no lines would drop the only line @@ -191,10 +232,10 @@ private static ListNode BuildList( break; } - AddEntry(list, rule.Item, rule.Fields, rule.Lists, scope.Enter(item), context, traits, errors, target); + matched.Add(item); } - return list; + return matched; } /// @@ -214,11 +255,12 @@ private static void AddEntry( MappingContext context, SourceTraits traits, List errors, - string target) + string target, + int written) { if (item is not null) { - if (TryResolveField(item, scope, context, traits, out var value, out var reason)) + if (TryResolveField(item, scope, context, traits, written, out var value, out var reason)) list.Add(ValueNode.Value(value)); else errors.Add(new MappingError(target, reason!)); @@ -226,7 +268,7 @@ private static void AddEntry( } var row = ValueNode.Object(); - MapInto(row, fields, lists, scope, context, traits, errors, target); + MapInto(row, fields, lists, scope, context, traits, errors, target, written); list.Add(row); } @@ -235,11 +277,21 @@ private static bool TryResolveField( Scope scope, MappingContext context, SourceTraits traits, + int? written, out object? value, out string? reason) { reason = null; + // Outside a list there is nothing to count, and answering zero would be a number the + // partner would act on rather than a mistake anyone would notice. + if (field.From.Kind == ValueSourceKind.Count && written is null) + { + value = null; + reason = "counting entries only means something inside a list"; + return false; + } + value = field.From.Kind switch { ValueSourceKind.Fixed => field.From.Value, @@ -247,6 +299,7 @@ private static bool TryResolveField( ValueSourceKind.RootPath => Values.ResolveScalar(scope.Root, field.From.Path), ValueSourceKind.Partner => context.PartnerValue(field.From.Key), ValueSourceKind.Global => context.GlobalValue(field.From.SetId, field.From.Key), + ValueSourceKind.Count => (decimal)(written ?? 0), _ => null, }; diff --git a/SW.Bitween.NativeAdapters/Mapper/Formats/CsvFormat.cs b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvFormat.cs new file mode 100644 index 00000000..90b89653 --- /dev/null +++ b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvFormat.cs @@ -0,0 +1,303 @@ +using System.Globalization; +using System.Text; +using CsvHelper; +using CsvHelper.Configuration; + +namespace SW.Bitween.NativeAdapters.Mapper.Formats; + +/// +/// Reads and writes delimited text — comma, semicolon, pipe or tab. +/// +/// +/// +/// A file in this format is a list of rows, and that is the whole reason it needs so little +/// here. A list rule walking "" already means "the document is itself the list", and +/// Root already means "the whole output is a list" — both written for a JSON document that is +/// a bare array, both already tested. So the mapper is untouched: this is a reader and a writer, and +/// everything between them already exists. +/// +/// +/// Every field is read as text and left alone. Nothing looks at 041800 and decides it is a +/// number, because a tracking reference that loses its leading zero is rejected by the partner and +/// nothing here would ever say why. Where the output wants a real number the rule's own type does +/// it, and it does it because someone asked rather than because something guessed. +/// +/// +/// An object output — not a list — is written as a header and exactly one row. That falls out of +/// treating a row as an object rather than being a case of its own. +/// +/// +public class CsvFormat(CsvOptions? options = null) : IDocumentFormat +{ + private readonly CsvOptions _options = options ?? new CsvOptions(); + + public string Id => "csv"; + + public string ContentType => "text/csv"; + + /// No: a file of one row is a list of one, and says so. + /// + /// Unlike XML, there is no ambiguity to resolve — a row is a row whether there is one of them or + /// a thousand, so nothing has to be tolerated after the fact. + /// + public bool SingleValueIsAList => false; + + /// + /// The separator between a field's name and the name of a field inside it. + /// + /// + /// A row is flat and a mapping is not, so a rule writing to destination.city has to land + /// somewhere. It becomes a column of exactly that name — which is how the editor already shows + /// the path, and the only way to produce such a column at all, since the output-field name box + /// splits what is typed into it on dots. + /// + public const char PathSeparator = '.'; + + public ValueNode Read(string text) + { + if (string.IsNullOrWhiteSpace(text)) + throw new DocumentFormatException("The document is empty, so there is nothing to map."); + + var list = ValueNode.List(); + + try + { + using var reader = new StringReader(StripByteOrderMark(text)); + using var parser = new CsvParser(reader, Configuration(forWriting: false)); + + var names = Array.Empty(); + if (_options.HasHeader) + { + // A file that is nothing but a header is a list of no rows, which is a true answer + // and not an error — a carrier with nothing to report sends exactly that. + if (!parser.Read()) return list; + names = ColumnNames(parser.Record ?? []); + } + + while (parser.Read()) + { + var record = parser.Record; + if (record is null || IsBlank(record)) continue; + + var row = ValueNode.Object(); + for (var at = 0; at < record.Length; at++) + row.Set(NameAt(names, at), ValueNode.Value(record[at])); + + list.Add(row); + } + } + catch (CsvHelperException ex) + { + throw new DocumentFormatException($"The document could not be read as delimited text: {ex.Message}"); + } + + return list; + } + + public string Write(ValueNode root) + { + var rows = root switch + { + ListNode list => list.Items, + // An object is one row. A mapping whose output is a single record has no reason to be + // written as a list of one just to reach this format. + ObjectNode => [root], + _ => throw new DocumentFormatException( + "A delimited file is rows of fields, so the output has to be a list or an object. " + + "This mapping produced a single value."), + }; + + // Every column any row has, in the order they first appear. Rows can legitimately differ — + // entries written into a list carry their own rules — and taking the first row's columns + // would drop the rest without a word. + var columns = new List(); + var seen = new HashSet(StringComparer.Ordinal); + var flattened = new List>(); + + foreach (var row in rows) + { + var cells = new Dictionary(StringComparer.Ordinal); + Flatten(row, prefix: "", cells); + foreach (var column in cells.Keys) + if (seen.Add(column)) + columns.Add(column); + flattened.Add(cells); + } + + var output = new StringWriter(); + using (var writer = new CsvWriter(output, Configuration(forWriting: true))) + { + if (_options.HasHeader) + { + foreach (var column in columns) writer.WriteField(column); + writer.NextRecord(); + } + + foreach (var cells in flattened) + { + // A column this row does not have is written empty rather than skipped: a short row + // would shift every field after it into the wrong column. + foreach (var column in columns) + writer.WriteField(cells.TryGetValue(column, out var value) ? value : ""); + writer.NextRecord(); + } + } + + // Prepended rather than written through the writer, which has no notion of one. Reading + // strips it again, so a file we produce and then read back is unchanged by it. + return _options.ByteOrderMark ? '\ufeff' + output.ToString() : output.ToString(); + } + + /// + /// One row's cells, with nested fields folded into dotted column names. + /// + /// + /// A list inside a row is refused rather than folded. There is no column name that would make + /// ["A1","B7"] fit into one cell, and inventing one — joining them, taking the first — + /// would lose data quietly, which is the one outcome worth refusing over. + /// + private static void Flatten(ValueNode node, string prefix, Dictionary cells) + { + switch (node) + { + case ObjectNode obj: + foreach (var (key, child) in obj.Children()) + Flatten(child, prefix.Length == 0 ? key : prefix + PathSeparator + key, cells); + break; + + // A row that is a single value — a list of plain values, which is a perfectly ordinary + // one-column file of tracking numbers. It has no name to take, so it takes the name any + // column has when nothing names it: its position. + case ScalarNode scalar: + cells[prefix.Length == 0 ? Position(0) : prefix] = AsText(scalar.Value); + break; + + case ListNode: + throw new DocumentFormatException( + prefix.Length == 0 + ? "A row of a delimited file cannot itself be a list." + : $"'{prefix}' is a list, and a single cell of a delimited file cannot hold " + + "one. Write its entries into named fields instead."); + } + } + + /// + /// The parser and writer settings. + /// + /// + /// Whether to pin the line ending. CsvHelper applies NewLine on reading only when it has + /// been set explicitly, and setting it there would refuse a file that separates its rows with + /// bare newlines — which most of them do. So it is pinned for writing, where RFC 4180 asks for + /// CRLF, and left alone for reading, where whatever the partner sent has to be accepted. + /// + private CsvConfiguration Configuration(bool forWriting) + { + var configuration = new CsvConfiguration(CultureInfo.InvariantCulture) + { + Delimiter = _options.Delimiter, + // Read by hand through the parser rather than mapped onto a class, so CsvHelper is never + // asked to find a header; the names are taken from the first record here instead. + HasHeaderRecord = false, + // A partner file is not a well-formed file. A stray quote in the middle of a field is + // common enough that refusing the whole document over one would stop a day's shipments, + // and the field still arrives — as the characters that are actually there. + BadDataFound = null, + MissingFieldFound = null, + DetectColumnCountChanges = false, + // Whitespace is data. A name written as `s ramanan` and a postcode with a leading space + // both turned up in the first files anyone sent. + TrimOptions = TrimOptions.None, + IgnoreBlankLines = true, + // RFC 4180's set. A field carrying the delimiter, a quote or a line break has to be + // quoted or the file it lands in no longer says what it meant to say. + ShouldQuote = args => + args.Field is not null && + (args.Field.Contains(_options.Delimiter, StringComparison.Ordinal) || + args.Field.Contains('"') || + args.Field.Contains('\r') || + args.Field.Contains('\n')), + }; + + // Assigned only for writing, because assigning it at all is what counts as setting it — + // and a reader pinned to one line ending refuses every file that uses the other. + if (forWriting) configuration.NewLine = "\r\n"; + + return configuration; + } + + /// + /// The column names a header record gives, with every column left addressable. + /// + /// + /// A blank name and a repeated one both fall back to the column's position, which is the name + /// that column would have had with no header at all. The alternative is two columns sharing a + /// name, and since a row is built by setting keys on an object, the second would silently + /// replace the first — a column of the partner's file simply missing, with nothing to say so. + /// + private static string[] ColumnNames(string[] header) + { + var names = new string[header.Length]; + var used = new HashSet(StringComparer.Ordinal); + + for (var at = 0; at < header.Length; at++) + { + var name = header[at]; + names[at] = name.Length > 0 && used.Add(name) ? name : Position(at); + used.Add(names[at]); + } + + return names; + } + + /// + /// What the field at is called. + /// + /// + /// Past the end of the header — a row with more fields than the header named — the position is + /// used, so the extra field is still readable. Dropping it would be the same silent loss as a + /// duplicate name. + /// + private static string NameAt(string[] names, int at) => + at < names.Length ? names[at] : Position(at); + + /// A field's name when it has none: its position, counting from one. + private static string Position(int at) => (at + 1).ToString(CultureInfo.InvariantCulture); + + /// + /// Whether a record is a blank line rather than a row. + /// + /// + /// IgnoreBlankLines covers an empty line, but a line holding only delimiters — which the + /// samples have between blocks of records — parses as a row of empty fields. Mapping that would + /// produce a row of nothing for every gap in the file. + /// + private static bool IsBlank(string[] record) => record.All(field => field.Length == 0); + + /// + /// Drops the byte-order mark, if the text still carries one. + /// + /// + /// Excel writes one, and left in place it becomes part of the first column's name — so a header + /// of ShipmentNumber arrives as ShipmentNumber and every rule reading it + /// resolves to nothing, with the editor showing a name that looks exactly right. + /// + private static string StripByteOrderMark(string text) => + text.Length > 0 && text[0] == '' ? text[1..] : text; + + /// + /// A value as a field's text. + /// + /// + /// Invariant throughout: a decimal written under a French locale uses a comma, which in a + /// comma-delimited file would silently become an extra column. A decimal keeps the scale it was + /// given, so a weight of 0.100 is written back as 0.100. + /// + private static string AsText(object? value) => value switch + { + null => "", + bool b => b ? "true" : "false", + decimal m => m.ToString(CultureInfo.InvariantCulture), + string s => s, + _ => Convert.ToString(value, CultureInfo.InvariantCulture) ?? "", + }; +} diff --git a/SW.Bitween.NativeAdapters/Mapper/Formats/CsvOptions.cs b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvOptions.cs new file mode 100644 index 00000000..8aa0afc9 --- /dev/null +++ b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvOptions.cs @@ -0,0 +1,53 @@ +namespace SW.Bitween.NativeAdapters.Mapper.Formats; + +/// +/// How one side of a mapping reads or writes delimited text. +/// +/// +/// +/// Kept per side rather than per mapping: a partner's semicolon file is routinely turned into a +/// comma file for somebody else, and one client alone has been seen sending all three of comma, +/// semicolon and pipe. +/// +/// +/// There is no standard worth the name for any of this, which is why none of it is inferred. +/// Sniffing the delimiter from a sample gets it wrong the first time a field legitimately contains +/// a comma, and by then the mapping is in production. +/// +/// +public class CsvOptions +{ + /// The characters between one field and the next. + /// + /// A string rather than a char because a tab travels through JSON as \t, and because + /// multi-character delimiters exist in the wild. + /// + public string Delimiter { get; set; } = ","; + + /// + /// Whether the first line names the columns rather than carrying data. + /// + /// + /// False is not the unusual case. Two of the three files a single client sends start straight + /// into data, so the fields have no names at all and paths are positions instead — see + /// . + /// + public bool HasHeader { get; set; } = true; + + /// + /// Whether to start a written file with a byte-order mark. + /// + /// + /// + /// Three bytes that tell Excel the text is UTF-8. Without them a name like + /// BEAUTRAIT Raphaël opens as Raphaël, and whoever opens it has no way to + /// correct that after the fact — which is why this has to be decided when the file is written. + /// + /// + /// Off by default, because a partner's own parser can just as easily choke on three bytes it + /// did not expect at the start of the file. Reading always strips one, whatever this says: + /// left in place it becomes part of the first column's name. + /// + /// + public bool ByteOrderMark { get; set; } +} diff --git a/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs b/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs index 97792ab3..e0c3340e 100644 --- a/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs +++ b/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs @@ -12,20 +12,39 @@ namespace SW.Bitween.NativeAdapters.Mapper.Formats; /// public static class DocumentFormats { - private static readonly Dictionary ById = + private static readonly JsonFormat Json = new(); + private static readonly XmlFormat Xml = new(); + + /// + /// How each id is built. + /// + /// + /// Factories rather than instances because delimited text is configured per side — a partner's + /// semicolon file routinely becomes somebody else's comma file — so its reader and its writer + /// cannot be the same shared object. The formats that need no configuration stay singletons. + /// + private static readonly Dictionary> ById = new(StringComparer.OrdinalIgnoreCase) { - ["json"] = new JsonFormat(), - ["xml"] = new XmlFormat(), + ["json"] = _ => Json, + ["xml"] = _ => Xml, + ["csv"] = csv => new CsvFormat(csv), }; /// Format ids, for the editor's dropdown and for error messages. public static IReadOnlyList Ids { get; } = ById.Keys.OrderBy(k => k).ToList(); - public static bool TryGet(string? id, [NotNullWhen(true)] out IDocumentFormat? format) + /// The format for an id, configured with where it applies. + public static bool TryGet( + string? id, + [NotNullWhen(true)] out IDocumentFormat? format, + CsvOptions? csv = null) { format = null; - return id is not null && ById.TryGetValue(id, out format); + if (id is null || !ById.TryGetValue(id, out var build)) return false; + + format = build(csv); + return true; } /// diff --git a/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs b/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs index 39c29aef..a22d24fe 100644 --- a/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs +++ b/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs @@ -47,6 +47,17 @@ public class MappingRules /// public DateOrder SourceDateOrder { get; set; } = DateOrder.YearFirst; + /// How the incoming document is delimited, when it is delimited text. + /// + /// Null for every other format, and null here means the defaults. Two options rather than one + /// because the two sides are genuinely independent: reading a partner's semicolon file and + /// writing a comma file for someone else is one mapping. + /// + public Formats.CsvOptions? SourceCsv { get; set; } + + /// How the produced document is delimited, when it is delimited text. + public Formats.CsvOptions? TargetCsv { get; set; } + public List Fields { get; set; } = new(); public List Lists { get; set; } = new(); @@ -143,6 +154,27 @@ public enum ValueSourceKind /// A key in one of the global values sets. Global, + + /// + /// How many entries the source list produced for the list this rule sits in. + /// + /// + /// + /// For a trailer record: a partner's file ends with a line carrying the number of records + /// above it, and nothing else in a mapping knows that number. + /// + /// + /// Entries written by hand are not counted, which is the whole point of the distinction — a + /// trailer says how many records there are, and the header line above it is not one. + /// Counting them would also mean adding a header later quietly changed the trailer, with + /// nothing on screen to say the number had moved. + /// + /// + /// The same number wherever it is read in one list, so a format that puts its count in the + /// header rather than the trailer works without anything special. + /// + /// + Count, } /// How a document writes dates that are not year-first. @@ -261,6 +293,17 @@ public class ListRule /// /// public List Fixed { get; set; } = new(); + + /// + /// Entries put into the list after the walked ones. + /// + /// + /// The other end of , and the reason it exists: a delimited file from a + /// carrier ends with a trailer record carrying the number of records above it. Built exactly + /// the same way, reading against the same scope — the only difference is where they land, and + /// that can see the whole list by the time they run. + /// + public List After { get; set; } = new(); } /// One entry of a list that no source list produced. diff --git a/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs b/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs index 8ac31897..15c90b7d 100644 --- a/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs +++ b/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs @@ -69,8 +69,8 @@ public void InitializeStartupValues(IDictionary settings) public Task Handle(XchangeFile xchangeFile) { - var source = ResolveFormat(_rules.SourceFormat, "source"); - var target = ResolveFormat(_rules.TargetFormat, "target"); + var source = ResolveFormat(_rules.SourceFormat, "source", _rules.SourceCsv); + var target = ResolveFormat(_rules.TargetFormat, "target", _rules.TargetCsv); var input = source.Read(xchangeFile.Data); var output = DocumentMapper.Map(_rules, input, _context, source.SingleValueIsAList); @@ -136,8 +136,8 @@ private static MappingContext ReadContext(IDictionary settings) } } - private static IDocumentFormat ResolveFormat(string id, string role) => - DocumentFormats.TryGet(id, out var format) + private static IDocumentFormat ResolveFormat(string id, string role, CsvOptions? csv) => + DocumentFormats.TryGet(id, out var format, csv) ? format : throw new InvalidOperationException(DocumentFormats.Unsupported(id, role)); } diff --git a/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj b/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj index 761c8fdb..3f0e3f41 100644 --- a/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj +++ b/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj @@ -15,6 +15,7 @@ + diff --git a/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs b/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs new file mode 100644 index 00000000..67ccab3e --- /dev/null +++ b/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs @@ -0,0 +1,470 @@ +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.NativeAdapters.Mapper; +using SW.Bitween.NativeAdapters.Mapper.Formats; + +namespace SW.Bitween.UnitTests.NativeMapper; + +/// +/// Reading delimited text into the neutral tree. +/// +/// +/// The three shapes here are the three files a single client actually sends, with nothing +/// smoothed over: a comma file with a header, a pipe file with three record types and no header, +/// and a semicolon file with no header, blank lines between blocks and an accented name in it. +/// Between them they are the reason none of the delimiter, the header or the encoding is inferred. +/// +[TestClass] +public class CsvFormatReadTests +{ + /// Comma, with a header naming seven columns. + private const string Commas = + """ + ShipmentNumber,Reference,TrackingCode,Date,Time,Comment1,Comment2 + 6G61965126082,202493482,SHOR020,2026-09-14,08:29:49,, + 8G49824171336,202340914,SHOR020,2026-09-14,08:34:34,, + """; + + /// Pipe, no header, three record types told apart by the first field. + private const string Pipes = + """ + H|FFSTAT|1|0||||||||||202609141313|1309981|N + D|1309981172|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03831|3800351262|202609141307|20260911|NTE|CDG|NTE||BRIAN MATIAS CASTRO PENA|GLOBAL LOGTICS NETWORK|||||||222998693|Clementine Sandri| + D|1309981174|CC|AWAITING CONSIGNEE COLLECTION|0.100|KGM|1|||E824836443|4472825486|202609141305|20260911|MRS|CDG|MRS||CHRISTOPHER GERGES|GLOBAL LOGISTIC NETWORK|||||||222998693|| + T|9|1309981| + """; + + /// Semicolon, no header, blank lines between blocks, an accented name. + private const string Semicolons = + "track;ZT6090386150DE;CDGSF1;SHTW001;20260914;123938;CDG;M+;;;TS;;;BEAUTRAIT Raphaël;ZT6090386150DE;;041800\n" + + "\n" + + "\n" + + "track;ZT6090386205DE;CDGSF1;SHTW001;20260914;123938;CDG;M+;;;TS;;;BEAUTRAIT Raphaël;ZT6090386205DE;;041800\n"; + + private static CsvFormat Format(string delimiter = ",", bool header = true) => + new(new CsvOptions { Delimiter = delimiter, HasHeader = header }); + + /// The rows, as a rule walking the document would see them. + private static ListNode Rows(string text, string delimiter = ",", bool header = true) + { + var read = Format(delimiter, header).Read(text); + return read as ListNode ?? throw new AssertFailedException("a delimited file reads as a list"); + } + + /// The value at of one row, the way a rule reads it. + private static object At(ValueNode row, string path) => Values.ResolveScalar(row, path); + + [TestMethod] + public void A_file_is_a_list_of_rows() + { + // Not an object with a list inside it: the document *is* the list, which is what lets a + // rule walk it with the same empty path a JSON root array already uses. + var read = Format().Read(Commas); + + Assert.IsInstanceOfType(read); + Assert.AreEqual(2, ((ListNode)read).Items.Count, "the header is not a row"); + } + + [TestMethod] + public void A_header_names_the_columns() + { + var rows = Rows(Commas); + + Assert.AreEqual("6G61965126082", At(rows.Items[0], "ShipmentNumber")); + Assert.AreEqual("SHOR020", At(rows.Items[0], "TrackingCode")); + Assert.AreEqual("08:34:34", At(rows.Items[1], "Time")); + } + + [TestMethod] + public void An_empty_field_is_empty_rather_than_missing() + { + // `,,` at the end of every row of the real file. Present and empty is what the file says, + // and it is different from a column that is not there at all. + var rows = Rows(Commas); + + Assert.AreEqual("", At(rows.Items[0], "Comment1")); + Assert.IsNull(At(rows.Items[0], "Comment3"), "a column the file does not have"); + } + + [TestMethod] + public void Without_a_header_the_fields_are_numbered_from_one() + { + var rows = Rows(Pipes, "|", header: false); + + Assert.AreEqual(4, rows.Items.Count, "every line is a row when nothing is a header"); + Assert.AreEqual("H", At(rows.Items[0], "1")); + Assert.AreEqual("D", At(rows.Items[1], "1")); + Assert.AreEqual("1309981172", At(rows.Items[1], "2")); + Assert.AreEqual("BRIAN MATIAS CASTRO PENA", At(rows.Items[1], "18")); + } + + [TestMethod] + public void Rows_of_different_record_types_keep_their_own_lengths() + { + // The reason this matters: the header record has 16 fields, a detail row 28 and the + // trailer 4. Padding them to a common width would invent data; truncating would lose it. + var rows = Rows(Pipes, "|", header: false); + + Assert.AreEqual("N", At(rows.Items[0], "16")); + Assert.IsNull(At(rows.Items[0], "17"), "the header record has no 17th field"); + Assert.AreEqual("1309981", At(rows.Items[3], "3")); + Assert.IsNull(At(rows.Items[3], "5"), "the trailer has four fields"); + } + + [TestMethod] + public void A_leading_zero_survives() + { + // The whole reason nothing is allowed to decide a field looks like a number. 041800 as + // 41800 is a reference the partner rejects, and nothing here would ever say why. + var rows = Rows(Semicolons, ";", header: false); + + Assert.AreEqual("041800", At(rows.Items[0], "17")); + } + + [TestMethod] + public void A_decimal_keeps_the_scale_it_was_written_with() + { + var rows = Rows(Pipes, "|", header: false); + + Assert.AreEqual("0.100", At(rows.Items[1], "5"), "not 0.1"); + } + + [TestMethod] + public void Blank_lines_between_records_are_not_rows() + { + // The semicolon file has runs of them between blocks. Read as rows they would each map to + // an entry of nothing, and a partner would receive a file padded with empty records. + var rows = Rows(Semicolons, ";", header: false); + + Assert.AreEqual(2, rows.Items.Count); + } + + [TestMethod] + public void An_accented_name_is_read_as_written() + { + var rows = Rows(Semicolons, ";", header: false); + + Assert.AreEqual("BEAUTRAIT Raphaël", At(rows.Items[0], "14")); + } + + [TestMethod] + public void A_byte_order_mark_is_not_part_of_the_first_column_name() + { + // Excel writes one. Left in place it becomes part of the first header name, so every rule + // reading that column resolves to nothing while the editor shows a name that looks right. + var rows = Rows("" + Commas); + + Assert.AreEqual("6G61965126082", At(rows.Items[0], "ShipmentNumber")); + } + + [TestMethod] + public void A_value_holding_the_delimiter_is_one_field() + { + var rows = Rows( + """ + sku,address,qty + A1,"Flat 3, Rainbow St",2 + """); + + Assert.AreEqual("Flat 3, Rainbow St", At(rows.Items[0], "address")); + Assert.AreEqual("2", At(rows.Items[0], "qty"), "the quantity has not shifted a column"); + } + + [TestMethod] + public void A_value_holding_a_line_break_is_still_one_row() + { + // Two rows of data written across four lines of text. This is where a reader that splits + // on newlines produces three broken rows and nothing complains. + var rows = Rows("sku,address\nA1,\"Flat 3\nRainbow Street\nAmman\"\n"); + + Assert.AreEqual(1, rows.Items.Count); + Assert.AreEqual("Flat 3\nRainbow Street\nAmman", At(rows.Items[0], "address")); + } + + [TestMethod] + public void A_doubled_quote_is_one_quote() + { + var rows = Rows("sku,note\nA1,\"He said \"\"urgent\"\" twice\"\n"); + + Assert.AreEqual("He said \"urgent\" twice", At(rows.Items[0], "note")); + } + + [TestMethod] + public void Whitespace_inside_a_field_is_kept() + { + // ` s ramanan` and a postcode with a leading space both turned up in real files. Trimming + // is a decision about the partner's data, and not one to take on their behalf. + var rows = Rows("sku,name\nA1, s ramanan \n"); + + Assert.AreEqual(" s ramanan ", At(rows.Items[0], "name")); + } + + [TestMethod] + public void A_repeated_column_name_stays_reachable_by_position() + { + // Two columns cannot share a name — a row is built by setting keys on an object, so the + // second would silently replace the first and a column would simply be gone. + var rows = Rows("code,code,qty\nA1,B7,2"); + + Assert.AreEqual("A1", At(rows.Items[0], "code")); + Assert.AreEqual("B7", At(rows.Items[0], "2"), "the repeat falls back to its position"); + Assert.AreEqual("2", At(rows.Items[0], "qty")); + } + + [TestMethod] + public void A_field_past_the_end_of_the_header_is_still_readable() + { + var rows = Rows("sku,qty\nA1,2,surprise"); + + Assert.AreEqual("surprise", At(rows.Items[0], "3")); + } + + [TestMethod] + public void A_file_of_nothing_but_a_header_is_no_rows_rather_than_an_error() + { + // A carrier with nothing to report sends exactly this, every morning. + var rows = Rows("ShipmentNumber,Reference\n"); + + Assert.AreEqual(0, rows.Items.Count); + } + + [TestMethod] + public void An_empty_document_is_refused() + { + var thrown = Assert.ThrowsException(() => Format().Read(" ")); + + Assert.IsFalse(string.IsNullOrWhiteSpace(thrown.Message)); + } + + [TestMethod] + public void The_content_type_says_what_it_is() + { + // The previous mapper left this unset and the gateway fell back to application/json, so a + // partner was served a delimited file labelled as JSON. + Assert.AreEqual("text/csv", Format().ContentType); + } +} + +/// +/// Writing the neutral tree back out as delimited text. +/// +[TestClass] +public class CsvFormatWriteTests +{ + private static CsvFormat Format(string delimiter = ",", bool header = true) => + new(new CsvOptions { Delimiter = delimiter, HasHeader = header }); + + /// The document, with newlines normalised so an expectation holds on any platform. + private static string Written(ValueNode tree, string delimiter = ",", bool header = true) => + Format(delimiter, header).Write(tree).Replace("\r\n", "\n"); + + private static ObjectNode Obj(params (string Key, ValueNode Node)[] children) + { + var node = ValueNode.Object(); + foreach (var (key, child) in children) node.Set(key, child); + return node; + } + + private static ValueNode V(object? value) => ValueNode.Value(value); + + private static ListNode L(params ValueNode[] items) + { + var list = ValueNode.List(); + foreach (var item in items) list.Add(item); + return list; + } + + private static string Refuses(ValueNode tree) + { + var thrown = Assert.ThrowsException(() => Format().Write(tree)); + Assert.IsFalse(string.IsNullOrWhiteSpace(thrown.Message), "a refusal must explain itself"); + return thrown.Message; + } + + [TestMethod] + public void A_list_of_records_is_a_header_and_a_row_each() + { + var written = Written(L( + Obj(("sku", V("A1")), ("qty", V(2m))), + Obj(("sku", V("B7")), ("qty", V(5m))))); + + Assert.AreEqual("sku,qty\nA1,2\nB7,5\n", written); + } + + [TestMethod] + public void An_object_is_a_single_row() + { + // A mapping whose output is one record has no reason to be made a list of one just to + // reach this format. + Assert.AreEqual("sku,qty\nA1,2\n", Written(Obj(("sku", V("A1")), ("qty", V(2m))))); + } + + [TestMethod] + public void A_nested_field_becomes_a_dotted_column() + { + // The editor already shows the path this way, and the output-field name box splits what is + // typed into it on dots — so this is the only way such a column can exist at all. + var written = Written(L(Obj( + ("orderId", V("A1")), + ("destination", Obj(("city", V("Amman")), ("country", V("JO"))))))); + + Assert.AreEqual("orderId,destination.city,destination.country\nA1,Amman,JO\n", written); + } + + [TestMethod] + public void Columns_are_every_column_any_row_has() + { + // Rows can differ: entries written into a list carry their own rules. Taking the first + // row's columns would drop the rest without a word. + var written = Written(L( + Obj(("sku", V("A1"))), + Obj(("sku", V("B7")), ("note", V("gift"))))); + + Assert.AreEqual("sku,note\nA1,\nB7,gift\n", written); + } + + [TestMethod] + public void A_column_a_row_lacks_is_written_empty_rather_than_skipped() + { + var written = Written(L( + Obj(("a", V("1")), ("b", V("2")), ("c", V("3"))), + Obj(("a", V("9")), ("c", V("8"))))); + + // Not `9,8`, which would put 8 under b and shift everything after it. + Assert.AreEqual("a,b,c\n1,2,3\n9,,8\n", written); + } + + [TestMethod] + public void A_value_holding_the_delimiter_is_quoted() + { + var written = Written(L(Obj(("sku", V("A1")), ("address", V("Flat 3, Rainbow St"))))); + + Assert.AreEqual("sku,address\nA1,\"Flat 3, Rainbow St\"\n", written); + } + + [TestMethod] + public void A_value_holding_a_quote_has_it_doubled() + { + var written = Written(L(Obj(("note", V("He said \"urgent\" twice"))))); + + Assert.AreEqual("note\n\"He said \"\"urgent\"\" twice\"\n", written); + } + + [TestMethod] + public void A_value_holding_a_line_break_is_quoted() + { + var written = Written(L(Obj(("address", V("Flat 3\nAmman"))))); + + Assert.AreEqual("address\n\"Flat 3\nAmman\"\n", written); + } + + [TestMethod] + public void The_delimiter_is_whatever_was_configured() + { + var tree = L(Obj(("a", V("1")), ("b", V("2")))); + + Assert.AreEqual("a;b\n1;2\n", Written(tree, ";")); + Assert.AreEqual("a|b\n1|2\n", Written(tree, "|")); + Assert.AreEqual("a\tb\n1\t2\n", Written(tree, "\t")); + } + + [TestMethod] + public void Without_a_header_only_the_rows_are_written() + { + var written = Written(L(Obj(("1", V("D")), ("2", V("1309981172")))), "|", header: false); + + Assert.AreEqual("D|1309981172\n", written); + } + + [TestMethod] + public void A_whole_number_has_no_decimal_point() + { + Assert.AreEqual("qty\n2\n", Written(L(Obj(("qty", V(2m)))))); + } + + [TestMethod] + public void A_decimal_keeps_its_scale_and_uses_a_point() + { + // A comma here would become an extra column in a comma-delimited file. + Assert.AreEqual("weight\n0.100\n", Written(L(Obj(("weight", V(0.100m)))))); + } + + [TestMethod] + public void Null_is_an_empty_field() + { + Assert.AreEqual("a,b\n1,\n", Written(L(Obj(("a", V("1")), ("b", V(null)))))); + } + + [TestMethod] + public void A_boolean_is_lower_case() + { + Assert.AreEqual("ok\ntrue\n", Written(L(Obj(("ok", V(true)))))); + } + + [TestMethod] + public void A_list_of_plain_values_is_one_column() + { + // A file of tracking numbers and nothing else. There is no name to take, so the column + // takes the name any column has when nothing names it. + Assert.AreEqual("1\nA1\nB7\n", Written(L(V("A1"), V("B7")))); + } + + [TestMethod] + public void A_list_inside_a_row_is_refused_with_a_reason() + { + var message = Refuses(L(Obj(("sku", V("A1")), ("tags", L(V("cold"), V("fragile")))))); + + StringAssert.Contains(message, "tags"); + } + + [TestMethod] + public void A_single_value_output_is_refused() + { + var message = Refuses(V("just this")); + + StringAssert.Contains(message, "list or an object"); + } +} + +/// +/// Reading a file and writing it back, which is the one test that holds both halves to each other. +/// +[TestClass] +public class CsvRoundTripTests +{ + [TestMethod] + public void A_comma_file_survives_being_read_and_written() + { + const string original = + "ShipmentNumber,Reference,Comment1\r\n" + + "6G61965126082,202493482,\r\n" + + "8G49824171336,202340914,\"a, comment\"\r\n"; + + var format = new CsvFormat(new CsvOptions()); + var written = format.Write(format.Read(original)); + + Assert.AreEqual(original, written); + } + + [TestMethod] + public void A_pipe_file_with_no_header_survives_too() + { + // Ragged on purpose: three records of three different widths, which is the real file. + const string original = + "H|FFSTAT|1\r\n" + + "D|1309981172|OK|DELIVERY\r\n" + + "T|9|1309981\r\n"; + + var format = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var written = format.Write(format.Read(original)); + + // Not identical: every row is written to the widest shape, because a column missing from a + // row has to be written empty or every field after it lands in the wrong column. Reading it + // back gives the same values, which is what a mapping actually depends on. + Assert.AreEqual( + "H|FFSTAT|1|\r\n" + + "D|1309981172|OK|DELIVERY\r\n" + + "T|9|1309981|\r\n", + written); + } +} diff --git a/SW.Bitween.UnitTests/NativeMapper/CsvMappingTests.cs b/SW.Bitween.UnitTests/NativeMapper/CsvMappingTests.cs new file mode 100644 index 00000000..8c06066f --- /dev/null +++ b/SW.Bitween.UnitTests/NativeMapper/CsvMappingTests.cs @@ -0,0 +1,439 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.Bitween.NativeAdapters.Mapper; +using SW.Bitween.NativeAdapters.Mapper.Formats; + +namespace SW.Bitween.UnitTests.NativeMapper; + +/// +/// A real client file, mapped end to end. +/// +/// +/// +/// These are the tests that decide whether the design claim holds: that delimited text needs +/// nothing from the engine, because a file is a list of rows and the mapper already walks +/// lists. Nothing here configures a CSV feature — the rules are the same ones a JSON root array +/// uses, and the only thing that changed is which reader produced the tree. +/// +/// +/// The second test is the important one. The pipe file holds three record types in one document, +/// and handling that was supposed to need no new feature: a list's existing "only some entries" +/// filter selects the detail rows and ignores the header and trailer. If that were wrong, the whole +/// shape of the plan would be wrong with it. +/// +/// +[TestClass] +public class CsvMappingTests +{ + /// The client's pipe file: an H record, three D records, a T record. + private const string Tracking = + "H|FFSTAT|1|0||||||||||202609141313|1309981|N\n" + + "D|1309981172|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03831|3800351262|202609141307|20260911|NTE|CDG|NTE||BRIAN MATIAS CASTRO PENA|GLOBAL LOGTICS NETWORK|||||||222998693|Clementine Sandri|\n" + + "D|1309981174|CC|AWAITING CONSIGNEE COLLECTION|0.100|KGM|1|||E824836443|4472825486|202609141305|20260911|MRS|CDG|MRS||CHRISTOPHER GERGES|GLOBAL LOGISTIC NETWORK|||||||222998693||\n" + + "D|1309981175|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03852|4472869306|202609141309|20260911|GVA|CDG|GVA||AGASH RAMANAN|GLOBAL LOGISTIC NETWORK|||||||222998693|s ramanan|\n" + + "T|9|1309981|\n"; + + /// The client's comma file, which names its columns. + private const string Movements = + "ShipmentNumber,Reference,TrackingCode,Date,Time,Comment1,Comment2\n" + + "6G61965126082,202493482,SHOR020,2026-09-14,08:29:49,,\n" + + "8G49824171336,202340914,SHOR020,2026-09-14,08:34:34,,\n"; + + private static ValueSource Path(string path) => new() { Kind = ValueSourceKind.Path, Path = path }; + + private static FieldRule Field(string target, string path) => + new() { Target = [target], From = Path(path) }; + + private static ValueSource Fixed(object? value) => + new() { Kind = ValueSourceKind.Fixed, Value = value }; + + /// How many entries the enclosing list holds at the point the rule runs. + private static ValueSource Counted() => new() { Kind = ValueSourceKind.Count }; + + private static TransformRule Transform(string fn, params (string Name, object Value)[] args) + { + var rule = new TransformRule { Fn = fn }; + foreach (var (name, value) in args) rule.Args[name] = JToken.FromObject(value); + return rule; + } + + private static string Json(ValueNode tree) => new JsonFormat().Write(tree); + + /// Maps a delimited document with the given rules, and returns the output as JSON. + private static JToken Run(MappingRules rules, string document, CsvOptions source) + { + var reader = new CsvFormat(source); + return JToken.Parse(Json(DocumentMapper.Map( + rules, reader.Read(document), MappingContext.Empty, reader.SingleValueIsAList))); + } + + [TestMethod] + public void A_file_with_a_header_maps_by_column_name() + { + // The whole output is the list, walking the document itself — the same two settings a JSON + // bare array already uses, written before delimited text existed. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Fields = + [ + Field("shipment", "ShipmentNumber"), + Field("reference", "Reference"), + Field("at", "Time"), + ], + }, + }; + + var output = Run(rules, Movements, new CsvOptions()); + + Assert.AreEqual(2, output.Count()); + Assert.AreEqual("6G61965126082", output[0]?["shipment"]?.ToString()); + Assert.AreEqual("202340914", output[1]?["reference"]?.ToString()); + Assert.AreEqual("08:34:34", output[1]?["at"]?.ToString()); + } + + [TestMethod] + public void Record_types_are_separated_by_the_filter_a_list_already_has() + { + // The claim: three record types in one file need no new feature. Field 1 says which kind of + // record a line is, and "only some entries" is exactly the tool for that. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule + { + Field = "1", + Operator = FilterOperator.Equal, + Value = "D", + }, + Fields = + [ + Field("tracking", "2"), + Field("status", "3"), + Field("consignee", "18"), + Field("weight", "5"), + ], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + // Three detail rows. The H record and the T record are gone, and nothing had to know that + // they existed. + Assert.AreEqual(3, output.Count()); + Assert.AreEqual("1309981172", output[0]?["tracking"]?.ToString()); + Assert.AreEqual("CC", output[1]?["status"]?.ToString()); + Assert.AreEqual("AGASH RAMANAN", output[2]?["consignee"]?.ToString()); + + // Still text, still exactly as the partner wrote it. + Assert.AreEqual("0.100", output[0]?["weight"]?.ToString()); + } + + [TestMethod] + public void A_field_can_be_made_a_real_number_where_the_output_wants_one() + { + // Reading leaves every cell as text on purpose. Converting is a decision the rule carries, + // so it happens where someone asked for it and nowhere else. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fields = + [ + new FieldRule { Target = ["weight"], From = Path("5"), Type = ValueType.Number }, + // Left alone beside it: a reference that lost its leading characters would be + // rejected by the partner, and this is the same field in the same row. + Field("reference", "10"), + ], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + Assert.AreEqual(JTokenType.Float, output[0]?["weight"]?.Type); + Assert.AreEqual(0.100m, output[0]?["weight"]?.Value()); + Assert.AreEqual("20260908FRACPKT03831", output[0]?["reference"]?.ToString()); + } + + [TestMethod] + public void A_delimited_file_can_be_mapped_into_another_one() + { + // The pipe file becoming the comma file, which is a mapping like any other — one format's + // reader and another's writer, even when they are the same format configured differently. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fields = [Field("ShipmentNumber", "11"), Field("TrackingCode", "3")], + }, + }; + + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var writer = new CsvFormat(new CsvOptions()); + + var written = writer + .Write(DocumentMapper.Map(rules, reader.Read(Tracking), MappingContext.Empty)) + .Replace("\r\n", "\n"); + + Assert.AreEqual( + "ShipmentNumber,TrackingCode\n" + + "3800351262,OK\n" + + "4472825486,CC\n" + + "4472869306,OK\n", + written); + } + + [TestMethod] + public void A_nested_rule_lands_in_a_dotted_column() + { + // Decided rather than discovered: a row is flat, so a nested target has to become a column + // name. The editor already shows the path this way. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fields = + [ + new FieldRule { Target = ["destination", "city"], From = Path("16") }, + new FieldRule { Target = ["destination", "hub"], From = Path("15") }, + ], + }, + }; + + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var written = new CsvFormat(new CsvOptions()) + .Write(DocumentMapper.Map(rules, reader.Read(Tracking), MappingContext.Empty)) + .Replace("\r\n", "\n"); + + StringAssert.StartsWith(written, "destination.city,destination.hub\n"); + StringAssert.Contains(written, "NTE,CDG\n"); + } + + [TestMethod] + public void The_rules_never_mention_a_format_so_the_writer_alone_decides() + { + // What "every pair of formats works" actually rests on. The tree below is built once and + // handed to two different writers; nothing in the rules knows which one is coming. + var fields = new List { Field("tracking", "2"), Field("status", "3") }; + var filter = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }; + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var source = reader.Read(Tracking); + + var asList = DocumentMapper.Map( + new MappingRules { Root = new ListRule { Over = "", Where = filter, Fields = fields } }, + source, MappingContext.Empty); + + Assert.AreEqual("1309981172", JToken.Parse(Json(asList))[0]?["tracking"]?.ToString()); + StringAssert.Contains(new CsvFormat(new CsvOptions()).Write(asList), "1309981172"); + + // XML is the one target that cannot take a list at the top, because a document has exactly + // one root element and no way to repeat it. That is a fact about XML rather than anything + // to do with where the rows came from, and it is refused by name rather than producing a + // document the partner's parser rejects. + var refused = Assert.ThrowsException( + () => new XmlFormat().Write(asList)); + StringAssert.Contains(refused.Message, "one root element"); + + // Named the list instead, and the same rows reach XML too. + var wrapped = DocumentMapper.Map( + new MappingRules + { + Lists = [new ListRule { Over = "", Where = filter, Fields = fields, Target = ["movements", "movement"] }], + }, + source, MappingContext.Empty); + + StringAssert.Contains(new XmlFormat().Write(wrapped), "1309981172"); + } + + [TestMethod] + public void The_whole_file_can_be_produced_header_rows_and_trailer() + { + // The end of it: reading the carrier's file was never the hard half. This writes one — + // an H line, a D line per shipment, and a T line carrying how many there were. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + // The header line, written before anything is walked. + Fixed = + [ + new ListEntry + { + Fields = + [ + new FieldRule { Target = ["1"], From = Fixed("H") }, + new FieldRule { Target = ["2"], From = Fixed("FFSTAT") }, + ], + }, + ], + Fields = [Field("1", "1"), Field("2", "2"), Field("3", "3")], + // The trailer, written after them — and the only place the count is known. + After = + [ + new ListEntry + { + Fields = + [ + new FieldRule { Target = ["1"], From = Fixed("T") }, + new FieldRule { Target = ["2"], From = Counted() }, + ], + }, + ], + }, + }; + + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var written = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }) + .Write(DocumentMapper.Map(rules, reader.Read(Tracking), MappingContext.Empty)) + .Replace("\r\n", "\n"); + + // Three detail rows, so the trailer says 3. The header line is not counted: a trailer + // states how many records there are, and the line announcing the file is not one. + Assert.AreEqual( + "H|FFSTAT|\n" + + "D|1309981172|OK\n" + + "D|1309981174|CC\n" + + "D|1309981175|OK\n" + + "T|3|\n", + written); + } + + [TestMethod] + public void The_count_does_not_move_when_a_header_line_is_added() + { + // The reason it counts rows rather than entries. Building the trailer first and adding + // the header afterwards is the ordinary way round, and the number must not shift under + // it — nothing on screen would say that it had. + var trailer = new ListEntry + { + Fields = [new FieldRule { Target = ["count"], From = Counted() }], + }; + + ListRule ListWith(List before) => new() + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fixed = before, + Fields = [Field("1", "1")], + After = [trailer], + }; + + var csv = new CsvOptions { Delimiter = "|", HasHeader = false }; + var withoutHeader = Run(new MappingRules { Root = ListWith([]) }, Tracking, csv); + var withHeader = Run( + new MappingRules + { + Root = ListWith([new ListEntry { Fields = [new FieldRule { Target = ["1"], From = Fixed("H") }] }]), + }, + Tracking, csv); + + Assert.AreEqual(3, withoutHeader.Last()?["count"]?.Value()); + Assert.AreEqual(3, withHeader.Last()?["count"]?.Value(), "adding a header moved nothing"); + } + + [TestMethod] + public void The_count_is_the_same_number_wherever_it_is_read() + { + // So a format that puts its record count in the header rather than the trailer needs + // nothing special, and a row can carry "1 of 3" without the numbers disagreeing. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fixed = [new ListEntry { Fields = [new FieldRule { Target = ["count"], From = Counted() }] }], + Fields = [new FieldRule { Target = ["count"], From = Counted() }], + After = [new ListEntry { Fields = [new FieldRule { Target = ["count"], From = Counted() }] }], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + foreach (var row in output) + Assert.AreEqual(3, row["count"]?.Value()); + } + + [TestMethod] + public void A_trailer_is_still_written_when_the_source_list_is_empty() + { + // A carrier with nothing to report still sends a file, and the partner still expects to + // be told the count is zero rather than to receive no trailer at all. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "NONE" }, + Fields = [Field("tracking", "2")], + After = + [ + new ListEntry + { + Fields = [new FieldRule { Target = ["count"], From = Counted() }], + }, + ], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + Assert.AreEqual(1, output.Count(), "the trailer, and nothing else"); + Assert.AreEqual(0, output[0]?["count"]?.Value()); + } + + [TestMethod] + public void Counting_outside_a_list_is_refused_rather_than_answered_with_zero() + { + // Zero is a number a partner would act on. There is no list here, so the honest answer + // is that the rule does not mean anything rather than that the answer is none. + var rules = new MappingRules + { + Fields = [new FieldRule { Target = ["total"], From = Counted() }], + }; + + var thrown = Assert.ThrowsException(() => DocumentMapper.Map( + rules, + new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }).Read(Tracking), + MappingContext.Empty)); + + StringAssert.Contains(thrown.Errors[0].Reason, "inside a list"); + } + + [TestMethod] + public void A_written_file_can_carry_the_mark_Excel_needs() + { + // Without it `BEAUTRAIT Raphaël` opens in Excel as mangled text, and nobody downstream + // can put that right afterwards. + var tree = ValueNode.List(); + var row = ValueNode.Object(); + row.Set("name", ValueNode.Value("BEAUTRAIT Raphaël")); + tree.Add(row); + + var plain = new CsvFormat(new CsvOptions()).Write(tree); + var marked = new CsvFormat(new CsvOptions { ByteOrderMark = true }).Write(tree); + + Assert.IsFalse(plain.StartsWith('\ufeff'), "off unless asked for"); + Assert.IsTrue(marked.StartsWith('\ufeff')); + + // And reading strips it again, so a file we produce and read back is unchanged by it. + Assert.AreEqual( + "BEAUTRAIT Raphaël", + Values.ResolveScalar( + ((ListNode)new CsvFormat(new CsvOptions()).Read(marked)).Items[0], "name")); + } +} diff --git a/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts b/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts new file mode 100644 index 00000000..d0f48604 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts @@ -0,0 +1,268 @@ +import { test, expect } from "@playwright/test"; +import { signInAsAdmin } from "./helpers"; +import { + addListField, + createSubscription, + expectPreview, + openMapper, + preview, +} from "./mapperHelpers"; + +/** + * Delimited text, through the real editor against the real backend. + * + * The point of running these end to end rather than in a unit test: the tree the editor + * draws comes from d3-dsv in the browser, and the document the mapping actually reads + * comes from CsvHelper on the server. Two parsers, and nothing but a test like this + * notices when they stop agreeing — a column offered here that resolves to nothing there + * is a mapping that looks complete and quietly writes an empty field. + * + * The files are the three a single client really sends, unaltered. + */ + +/** Pipe, no header, three record types told apart by the first field. */ +const TRACKING = + "H|FFSTAT|1|0||||||||||202609141313|1309981|N\n" + + "D|1309981172|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03831|3800351262|202609141307|20260911|NTE|CDG|NTE||BRIAN MATIAS CASTRO PENA|GLOBAL LOGTICS NETWORK|||||||222998693|Clementine Sandri|\n" + + "D|1309981174|CC|AWAITING CONSIGNEE COLLECTION|0.100|KGM|1|||E824836443|4472825486|202609141305|20260911|MRS|CDG|MRS||CHRISTOPHER GERGES|GLOBAL LOGISTIC NETWORK|||||||222998693||\n" + + "T|9|1309981|\n"; + +/** Comma, with a header naming its columns. */ +const MOVEMENTS = + "ShipmentNumber,Reference,TrackingCode,Date,Time,Comment1,Comment2\n" + + "6G61965126082,202493482,SHOR020,2026-09-14,08:29:49,,\n" + + "8G49824171336,202340914,SHOR020,2026-09-14,08:34:34,,\n"; + +test.beforeEach(async ({ page }) => { + await signInAsAdmin(page); +}); + +/** Opens the editor with the source side reading delimited text. */ +async function openWithCsv(page: import("@playwright/test").Page, sample: string, delimiter: string, header: boolean) { + const subscriptionId = await createSubscription(page); + await openMapper(page, subscriptionId); + + await page.getByLabel("From format").selectOption("csv"); + await page.getByLabel("source delimiter").selectOption(delimiter); + const box = page.getByRole("checkbox", { name: "source header row" }); + if (header) await box.check(); + else await box.uncheck(); + + await page.getByRole("textbox", { name: "Sample source document" }).fill(sample); + return subscriptionId; +} + +/** Makes the whole output a list walking the document, which is what every row-per-row mapping is. */ +async function rootListOverTheDocument(page: import("@playwright/test").Page) { + await page.getByRole("checkbox", { name: /The whole output is a list/ }).check(); + await page.getByRole("combobox", { name: "Source list" }).selectOption("p:"); + return page.getByRole("group", { name: "Rules for the list at the root" }); +} + +test("a header names the columns, and the server reads the same names", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "shipment", "ShipmentNumber"); + await addListField(root, "the root list", "at", "Time"); + + // Produced by the server from its own reading of the file. If the two parsers disagreed + // about where the columns are, this is where it would show. + await expectPreview(page, '"shipment": "6G61965126082"'); + await expect(preview(page)).toContainText('"at": "08:34:34"'); +}); + +test("the columns the editor offers are the ones the server can read", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + + const root = await rootListOverTheDocument(page); + await root.getByRole("button", { name: "Add a field to the root list" }).click(); + await root.getByRole("textbox", { name: "Output field name" }).last().fill("checked"); + + // Every column the source panel offers, tried against the server one at a time. + const field = root.getByRole("combobox", { name: "Source field" }).last(); + for (const column of ["ShipmentNumber", "Reference", "TrackingCode", "Date", "Time"]) { + await field.fill(column); + await expect(preview(page)).not.toContainText('"checked": null', { timeout: 15000 }); + } +}); + +test("three record types in one file are separated by the list's own filter", async ({ page }) => { + await openWithCsv(page, TRACKING, "|", false); + + const root = await rootListOverTheDocument(page); + + // The claim the whole plan rests on: a file holding an H record, D records and a T record + // needs no feature of its own. Field 1 says which kind of line this is. + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + await page.getByRole("checkbox", { name: "Only some entries" }).check(); + await page.getByRole("textbox", { name: "Filter field" }).fill("1"); + await page.getByRole("combobox", { name: "Filter comparison" }).selectOption("equal"); + await page.getByRole("textbox", { name: "Filter value" }).fill("D"); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + + await addListField(root, "the root list", "tracking", "2"); + await addListField(root, "the root list", "status", "3"); + + await expectPreview(page, '"tracking": "1309981172"'); + await expect(preview(page)).toContainText('"status": "CC"'); + + // The header and the trailer are gone, and nothing had to be told they existed. + await expect(preview(page)).not.toContainText("FFSTAT"); + await expect(preview(page)).not.toContainText('"tracking": "9"'); +}); + +test("a leading zero and a trailing scale survive the round trip to the server", async ({ + page, +}) => { + await openWithCsv(page, TRACKING, "|", false); + + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "weight", "5"); + await addListField(root, "the root list", "account", "26"); + + // 0.100 as 0.1, or an account reference losing a character, is a file the partner + // rejects with nothing anywhere to say why. + await expectPreview(page, '"weight": "0.100"'); + await expect(preview(page)).toContainText('"account": "222998693"'); +}); + +test("writing a delimited file takes its delimiter and header from the target side", async ({ + page, +}) => { + await openWithCsv(page, TRACKING, "|", false); + + await page.getByLabel("To format").selectOption("csv"); + await page.getByLabel("target delimiter").selectOption(";"); + await page.getByRole("checkbox", { name: "target header row" }).check(); + + const root = await rootListOverTheDocument(page); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + await page.getByRole("checkbox", { name: "Only some entries" }).check(); + await page.getByRole("textbox", { name: "Filter field" }).fill("1"); + await page.getByRole("textbox", { name: "Filter value" }).fill("D"); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + + await addListField(root, "the root list", "Tracking", "2"); + await addListField(root, "the root list", "Status", "3"); + + await expectPreview(page, "Tracking;Status"); + await expect(preview(page)).toContainText("1309981172;OK"); + await expect(preview(page)).toContainText("1309981174;CC"); +}); + +test("a nested rule becomes a dotted column", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + await page.getByLabel("To format").selectOption("csv"); + + const root = await rootListOverTheDocument(page); + // A row is flat, so the nesting has to land somewhere. The name box splits on dots, so + // this is the only way a column of this name can exist at all. + await addListField(root, "the root list", "shipment.number", "ShipmentNumber"); + + await expectPreview(page, "shipment.number"); + await expect(preview(page)).toContainText("6G61965126082"); +}); + +test("a shape a row cannot hold is refused with a reason", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + await page.getByLabel("To format").selectOption("csv"); + + // A list inside a row. There is no cell that holds one, and inventing a way to fit it — + // joining the entries, taking the first — would lose data without a word. + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "shipment", "ShipmentNumber"); + await root.getByRole("button", { name: "Add a list to the root list" }).click(); + await root.getByRole("textbox", { name: "Output list name" }).last().fill("tags"); + + await expect(page.getByText(/cannot hold one|cannot itself be a list/)).toBeVisible({ + timeout: 15000, + }); +}); + +/** Adds a field inside one of a list's written entries, pointed at a fixed value. */ +async function addEntryField( + scope: import("@playwright/test").Locator, + entry: string, + name: string, + value: string, +) { + await scope.getByRole("button", { name: `Add a field to ${entry}` }).click(); + await scope.getByRole("textbox", { name: "Output field name" }).last().fill(name); + await scope.getByRole("radio", { name: "Fixed" }).last().click(); + await scope.getByRole("textbox", { name: "Fixed value" }).last().fill(value); +} + +test("the carrier's whole file can be produced, trailer count and all", async ({ page }) => { + // The other direction, and the one that needed two features of its own: a file like the + // client's ends with a record carrying how many records came before it. + await openWithCsv(page, TRACKING, "|", false); + + await page.getByLabel("To format").selectOption("csv"); + await page.getByLabel("target delimiter").selectOption("|"); + await page.getByRole("checkbox", { name: "target header row" }).uncheck(); + + const root = await rootListOverTheDocument(page); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + await page.getByRole("checkbox", { name: "Only some entries" }).check(); + await page.getByRole("textbox", { name: "Filter field" }).fill("1"); + await page.getByRole("textbox", { name: "Filter value" }).fill("D"); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + + // The header line, written before anything is walked. + await root.getByRole("button", { name: "Add an entry to the root list" }).click(); + const first = page.getByRole("group", { name: "Rules for entry 1" }); + await addEntryField(first, "entry 1", "1", "H"); + await addEntryField(first, "entry 1", "2", "FFSTAT"); + + // One line per shipment. + await addListField(root, "the root list", "1", "1"); + await addListField(root, "the root list", "2", "2"); + await addListField(root, "the root list", "3", "3"); + + // And the trailer, which is the only place a rule can see what the list ended up holding. + await root.getByRole("button", { name: "Add a closing entry to the root list" }).click(); + const closing = page.getByRole("group", { name: "Rules for closing entry 1" }); + await addEntryField(closing, "closing entry 1", "1", "T"); + await closing.getByRole("button", { name: "Add a field to closing entry 1" }).click(); + await closing.getByRole("textbox", { name: "Output field name" }).last().fill("2"); + await closing.getByRole("radio", { name: "Count" }).last().click(); + + await expectPreview(page, "H|FFSTAT"); + await expect(preview(page)).toContainText("D|1309981172|OK"); + await expect(preview(page)).toContainText("D|1309981174|CC"); + + // Two shipments, so the trailer says 2 — the header line above it is not a record, and + // adding one later cannot move the number. + await expect(preview(page)).toContainText("T|2"); +}); + +test("counting is offered inside a list and nowhere else", async ({ page }) => { + // Outside a list there is nothing to count, so the segment is not there to be chosen. + await openWithCsv(page, MOVEMENTS, ",", true); + + await page.getByRole("button", { name: "Add a field", exact: true }).click(); + await expect(page.getByRole("radio", { name: "Count" })).toHaveCount(0); + + const root = await rootListOverTheDocument(page); + await root.getByRole("button", { name: "Add a field to the root list" }).click(); + await expect(root.getByRole("radio", { name: "Count" })).toHaveCount(1); +}); + +test("a file can be marked so Excel opens accented names correctly", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + await page.getByLabel("To format").selectOption("csv"); + + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "shipment", "ShipmentNumber"); + await expectPreview(page, "shipment"); + + // The mark itself is invisible, so what is checked is that asking for it changes the + // document the server produced rather than that anything looks different. + const before = await preview(page).textContent(); + await page.getByRole("checkbox", { name: "write a byte-order mark" }).check(); + await expect + .poll(async () => (await preview(page).textContent())?.charCodeAt(0), { timeout: 15000 }) + .toBe(0xfeff); + expect(before?.charCodeAt(0)).not.toBe(0xfeff); +}); diff --git a/SW.Bitween.Web/ClientApp/package.json b/SW.Bitween.Web/ClientApp/package.json index c677bb33..2b102b97 100644 --- a/SW.Bitween.Web/ClientApp/package.json +++ b/SW.Bitween.Web/ClientApp/package.json @@ -24,6 +24,7 @@ "@tailwindcss/vite": "^4.3.2", "@tanstack/react-query": "^5.101.2", "@uiw/react-codemirror": "^4.25.11", + "d3-dsv": "^3.0.1", "highlight.js": "^11.12.0", "immer": "^11.1.8", "lucide-react": "^1.24.0", @@ -34,6 +35,7 @@ }, "devDependencies": { "@playwright/test": "^1.61.1", + "@types/d3-dsv": "^3.0.7", "@types/node": "^24.13.2", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", diff --git a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx index 73900b9a..599a7f33 100644 --- a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx +++ b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx @@ -19,8 +19,8 @@ export function BuildFromSample() { const dispatch = useRulesDispatch(); const parsed = useMemo( - () => parseSample(targetSample, rules.targetFormat, "target"), - [targetSample, rules.targetFormat], + () => parseSample(targetSample, rules.targetFormat, "target", rules.targetCsv), + [targetSample, rules.targetFormat, rules.targetCsv], ); return ( diff --git a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx index 9afaf5f9..b2e3ff91 100644 --- a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx +++ b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx @@ -3,9 +3,9 @@ import { ChevronDown, ChevronRight, CornerDownRight, Trash2 } from "lucide-react import type { OutputEntryNode } from "../../lib/nativeMapper/outputTree"; import { useRules, useRulesDispatch } from "../../lib/nativeMapper/RulesEditorContext"; import { - SOURCE_KINDS, TYPE_BADGES, freshSource, + sourceKindsFor, type EditorFieldRule, } from "../../lib/nativeMapper/types"; import { SegmentedControl } from "../ui/SegmentedControl"; @@ -34,6 +34,8 @@ export function EntryRow({ paths: SourcePaths; }) { const { entry, position } = node; + // Both ends number from one, so the words have to say which end this is. + const named = node.placement === "after" ? `closing entry ${position}` : `entry ${position}`; const dispatch = useRulesDispatch(); const { ruleErrors } = useRules(); const [open, setOpen] = useState(false); @@ -54,16 +56,14 @@ export function EntryRow({ // Named, so this entry's own controls can be told apart from the list's and // from the other entries' — the position is the only thing distinguishing them. role="group" - aria-label={`Entry ${position}`} + aria-label={named} className={`rounded border ${ error ? "border-danger-300 bg-danger-50" : "border-warn-200/70 bg-warn-100/20" }`} >
- - entry {position} - + {named} {item && ( <> @@ -72,8 +72,8 @@ export function EntryRow({ updateItem({ from: freshSource(kind) })} /> @@ -87,7 +87,7 @@ export function EntryRow({ type="button" onClick={() => setOpen((o) => !o)} aria-expanded={open} - aria-label={`Details for entry ${position}`} + aria-label={`Details for ${named}`} title={ extras > 0 ? "Has a transform or a table" @@ -109,7 +109,7 @@ export function EntryRow({
@@ -407,6 +427,75 @@ function TestPartnerSelect({ ); } +/** + * How one side's delimited text is separated, and whether its first line names the columns. + * + * Asked rather than sniffed. A guessed delimiter is right until the first field that + * legitimately contains a comma, and a guessed header is right until a file whose first + * data row happens to look like labels — both of which are found in production rather + * than here. One client alone sends comma, semicolon and pipe, two of the three with no + * header at all, so neither of these has a safe default to fall back on. + */ +function CsvControls({ + side, + options, + onChange, +}: { + side: "source" | "target"; + options: CsvOptions; + onChange: (options: CsvOptions) => void; +}) { + const what = side === "source" ? "incoming" : "produced"; + + return ( +
+ onChange({ ...options, hasHeader: e.target.checked })} + className="size-3.5 rounded border-ink-300" + /> + header + + {/* Only worth asking on the side that writes one. A mark on the way in is stripped + whatever anyone thinks about it, because left in place it becomes part of the first + column's name. */} + {side === "target" && ( + + )} +
+ ); +} + function FormatSelect({ label, value, diff --git a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx index 8c9368f9..a5715170 100644 --- a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx +++ b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx @@ -3,7 +3,7 @@ import { ChevronDown, ChevronRight, Trash2 } from "lucide-react"; import { useRules, useRulesDispatch } from "../../lib/nativeMapper/RulesEditorContext"; import { isAssigned, isItemAssigned } from "../../lib/nativeMapper/rulesReducer"; import type { OutputRowNode } from "../../lib/nativeMapper/outputTree"; -import { SOURCE_KINDS, TYPE_BADGES, freshSource } from "../../lib/nativeMapper/types"; +import { TYPE_BADGES, freshSource, sourceKindsFor } from "../../lib/nativeMapper/types"; import { SegmentedControl } from "../ui/SegmentedControl"; import { RuleDetail } from "./RuleDetail"; import { RowInput } from "./rowControls"; @@ -31,10 +31,13 @@ export function OutputRow({ */ prefix, paths, + /** Whether this rule sits inside a list, which is the only place counting means anything. */ + inList, }: { node: OutputRowNode; prefix: string[]; paths: SourcePaths; + inList: boolean; }) { const { rule, errorKey } = node; const dispatch = useRulesDispatch(); @@ -152,7 +155,7 @@ export function OutputRow({ - +
); } @@ -136,13 +136,15 @@ function TreeNode({ } if (node.kind === "entry") { + const named = + node.placement === "after" ? `closing entry ${node.position}` : `entry ${node.position}`; return ( <>
{node.entry.item === undefined && ( -
+
@@ -207,6 +209,18 @@ function TreeNode({ indent={indent + 1} /> )} + {/* Written after everything above them, which is where a trailer goes — and the + only point from which counting can see the whole list. */} + +
dispatch({ type: "ADD_FIXED_ENTRY", listId: node.list.id }) } + onAddClosingEntry={() => + dispatch({ type: "ADD_CLOSING_ENTRY", listId: node.list.id }) + } showPerEntryRules={node.list.over !== undefined && node.list.item === undefined} onAddValue={ stillUndecided(node.list) @@ -297,6 +314,7 @@ export function AddRuleButtons({ canAddList, inside, onAddFixedEntry, + onAddClosingEntry, onAddValue, showPerEntryRules = true, }: { @@ -306,6 +324,13 @@ export function AddRuleButtons({ inside?: string; /** Offered on a list, where an entry can be written into it. */ onAddFixedEntry?: () => void; + /** + * Offered on a list, for an entry written after the walked ones. + * + * Separate from the leading one because a trailer is a different thing from a header, and + * because it is the only place a rule can count what the list ended up holding. + */ + onAddClosingEntry?: () => void; /** * Offered on a list that has not yet been made of anything. * @@ -344,6 +369,17 @@ export function AddRuleButtons({ Entry )} + {onAddClosingEntry && ( + + )} {showPerEntryRules && (
/// - /// A blank name and a repeated one both fall back to the column's position, which is the name - /// that column would have had with no header at all. The alternative is two columns sharing a - /// name, and since a row is built by setting keys on an object, the second would silently - /// replace the first — a column of the partner's file simply missing, with nothing to say so. + /// + /// Two columns cannot share a name: a row is built by setting keys on an object, so the second + /// would silently replace the first and a column of the partner's file would simply be missing. + /// + /// + /// Falling back to the position is not enough on its own, because a header can be a number. A + /// file headed 2, names its first column 2 and then wants 2 again for the + /// blank one beside it, so the fallback has to be checked like any other name. + /// /// - private static string[] ColumnNames(string[] header) + private static string Unique(string wanted, HashSet used) { - var names = new string[header.Length]; - var used = new HashSet(StringComparer.Ordinal); + if (used.Add(wanted)) return wanted; - for (var at = 0; at < header.Length; at++) + for (var n = 2; ; n++) { - var name = header[at]; - names[at] = name.Length > 0 && used.Add(name) ? name : Position(at); - used.Add(names[at]); + var candidate = $"{wanted}_{n}"; + if (used.Add(candidate)) return candidate; } - - return names; } - /// - /// What the field at is called. - /// - /// - /// Past the end of the header — a row with more fields than the header named — the position is - /// used, so the extra field is still readable. Dropping it would be the same silent loss as a - /// duplicate name. - /// - private static string NameAt(string[] names, int at) => - at < names.Length ? names[at] : Position(at); - /// A field's name when it has none: its position, counting from one. private static string Position(int at) => (at + 1).ToString(CultureInfo.InvariantCulture); diff --git a/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs b/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs index 67ccab3e..3a599c73 100644 --- a/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs +++ b/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs @@ -200,17 +200,63 @@ public void Whitespace_inside_a_field_is_kept() } [TestMethod] - public void A_repeated_column_name_stays_reachable_by_position() + public void A_repeated_column_name_stays_reachable_under_a_name_of_its_own() { // Two columns cannot share a name — a row is built by setting keys on an object, so the - // second would silently replace the first and a column would simply be gone. + // second would silently replace the first and a column would simply be gone. Numbered + // rather than moved to its position, which reads better and cannot collide with a header + // that happens to be a number. var rows = Rows("code,code,qty\nA1,B7,2"); Assert.AreEqual("A1", At(rows.Items[0], "code")); - Assert.AreEqual("B7", At(rows.Items[0], "2"), "the repeat falls back to its position"); + Assert.AreEqual("B7", At(rows.Items[0], "code_2")); Assert.AreEqual("2", At(rows.Items[0], "qty")); } + [TestMethod] + public void A_fallback_name_that_a_header_already_took_is_moved_out_of_the_way() + { + // The header's second column has no name, so it wants its position — which the first + // column is already called. Sharing it would mean the second column silently replacing + // the first when the row is built, and a column of the file simply disappearing. + var rows = Rows("2,,qty\na,b,7"); + + Assert.AreEqual("a", At(rows.Items[0], "2")); + Assert.AreEqual("b", At(rows.Items[0], "2_2")); + Assert.AreEqual("7", At(rows.Items[0], "qty")); + } + + [TestMethod] + public void A_header_repeating_a_number_keeps_both_columns() + { + var rows = Rows("2,2\na,b"); + + Assert.AreEqual("a", At(rows.Items[0], "2")); + Assert.AreEqual("b", At(rows.Items[0], "2_2")); + } + + [TestMethod] + public void A_surplus_field_whose_position_is_taken_is_moved_too() + { + // The header names a column `3`, and a row then has a third field wanting the same name. + var rows = Rows("sku,3\nA1,x,surplus"); + + Assert.AreEqual("x", At(rows.Items[0], "3")); + Assert.AreEqual("surplus", At(rows.Items[0], "3_2")); + } + + [TestMethod] + public void Every_row_names_a_column_the_same_way() + { + // Naming runs once for the file rather than per row, so a narrow row followed by a wide + // one cannot end up calling the same column two different things. + var rows = Rows("sku\nA1\nB7,extra"); + + Assert.AreEqual("A1", At(rows.Items[0], "sku")); + Assert.AreEqual("B7", At(rows.Items[1], "sku")); + Assert.AreEqual("extra", At(rows.Items[1], "2")); + } + [TestMethod] public void A_field_past_the_end_of_the_header_is_still_readable() { @@ -409,6 +455,18 @@ public void A_list_of_plain_values_is_one_column() Assert.AreEqual("1\nA1\nB7\n", Written(L(V("A1"), V("B7")))); } + [TestMethod] + public void Two_rules_writing_the_same_column_are_refused() + { + // A literal `a.b` key and a nested `a` → `b` both flatten to the column `a.b`. Keeping + // whichever ran last would drop a field the mapping plainly asks for. + var message = Refuses(L(Obj( + ("a.b", V("literal")), + ("a", Obj(("b", V("nested"))))))); + + StringAssert.Contains(message, "a.b"); + } + [TestMethod] public void A_list_inside_a_row_is_refused_with_a_reason() { diff --git a/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts b/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts index d0f48604..b491f8e0 100644 --- a/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts +++ b/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts @@ -79,11 +79,21 @@ test("the columns the editor offers are the ones the server can read", async ({ await root.getByRole("button", { name: "Add a field to the root list" }).click(); await root.getByRole("textbox", { name: "Output field name" }).last().fill("checked"); - // Every column the source panel offers, tried against the server one at a time. + // Every column the source panel offers, tried against the server one at a time — and each + // asserted on the value it should carry. A "not null" check would pass off the previous + // column's preview before the next one arrived, which is exactly the mismatch being hunted. const field = root.getByRole("combobox", { name: "Source field" }).last(); - for (const column of ["ShipmentNumber", "Reference", "TrackingCode", "Date", "Time"]) { + const firstRow: [string, string][] = [ + ["ShipmentNumber", "6G61965126082"], + ["Reference", "202493482"], + ["TrackingCode", "SHOR020"], + ["Date", "2026-09-14"], + ["Time", "08:29:49"], + ]; + + for (const [column, value] of firstRow) { await field.fill(column); - await expect(preview(page)).not.toContainText('"checked": null', { timeout: 15000 }); + await expect(preview(page)).toContainText(`"checked": "${value}"`, { timeout: 15000 }); } }); diff --git a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/csvSampleTree.test.ts b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/csvSampleTree.test.ts index af6574fe..14dd5aa7 100644 --- a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/csvSampleTree.test.ts +++ b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/csvSampleTree.test.ts @@ -140,10 +140,41 @@ describe("the tree for a delimited sample", () => { ); }); - it("keeps a repeated column name reachable by position", () => { + it("gives a repeated column name one of its own", () => { const columns = itemShapeAt(tree("code,code,qty\nA1,B7,2").root, ""); - expect(columns.map((c) => c.key)).toEqual(["code", "2", "qty"]); + expect(columns.map((c) => c.key)).toEqual(["code", "code_2", "qty"]); + }); + + it("moves a fallback name a header already took", () => { + // The blank second column wants its position, `2`, which the first column is called. Sharing + // it would leave the second column with no path of its own for a rule to read. + const columns = itemShapeAt(tree("2,,qty\na,b,7").root, ""); + + expect(columns.map((c) => c.key)).toEqual(["2", "2_2", "qty"]); + }); + + it("names a surplus field without colliding either", () => { + const columns = itemShapeAt(tree("sku,3\nA1,x,surplus").root, ""); + + expect(columns.map((c) => c.key)).toEqual(["sku", "3", "3_2"]); + }); + + it("names a column the same way on every row", () => { + // Naming runs once for the file rather than per row, so a narrow row followed by a wide one + // cannot end up calling the same column two different things. + const columns = itemShapeAt(tree("sku\nA1\nB7,extra").root, ""); + + expect(columns.map((c) => c.key)).toEqual(["sku", "2"]); + }); + + it("draws a tree even when the stored options are incomplete", () => { + // Rules can be hand-written or saved by an older build. Reading the delimiter off `{}` used + // to throw while the editor was drawing, which is a blank screen rather than a message. + const parsed = parseSample("a,b\n1,2", "csv", "source", {} as CsvOptions); + + expect(parsed.error).toBeNull(); + expect(itemShapeAt(parsed.root, "").map((c) => c.key)).toEqual(["a", "b"]); }); it("drops the byte-order mark Excel writes", () => { diff --git a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/rules.test.ts b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/rules.test.ts index 9c75db35..681178f6 100644 --- a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/rules.test.ts +++ b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/__tests__/rules.test.ts @@ -798,3 +798,32 @@ describe("a list's closing entries", () => { expect(isAssigned({ ...emptyFieldRule(), from: { kind: "count" } })).toBe(true); }); }); + +describe("delimited-text options off the wire", () => { + // Rules reach the editor from storage, so they can be hand-written or saved by an older + // build. Neither is worth refusing a whole mapping over, and neither can be passed on as it + // came: the editor reads the delimiter while it is drawing. + + it("fills in what an incomplete stored options object leaves out", () => { + const loaded = fromWire({ ...emptyRules(), sourceFormat: "csv", sourceCsv: {} as never }); + + expect(loaded.sourceCsv).toEqual({ delimiter: ",", hasHeader: true, byteOrderMark: false }); + }); + + it("replaces a delimiter that is not usable text", () => { + const loaded = fromWire({ + ...emptyRules(), + sourceFormat: "csv", + sourceCsv: { delimiter: "", hasHeader: false } as never, + }); + + expect(loaded.sourceCsv?.delimiter).toBe(","); + expect(loaded.sourceCsv?.hasHeader).toBe(false); + }); + + it("keeps options off a format that has none", () => { + const loaded = fromWire({ ...emptyRules(), sourceFormat: "json", sourceCsv: {} as never }); + + expect(loaded.sourceCsv).toBeUndefined(); + }); +}); diff --git a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/documentTree.ts b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/documentTree.ts index 2e1119a1..00de7049 100644 --- a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/documentTree.ts +++ b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/documentTree.ts @@ -176,7 +176,9 @@ function parseCsvSample(text: string, options: CsvOptions): ParsedSample { // d3-dsv takes one character. The wire format allows more, since the server's parser // does — so a mapping hand-edited to a longer delimiter still runs, it just cannot be // shown here, and saying so is better than drawing a tree that is wrong. - const delimiter = options.delimiter; + // Defended rather than assumed: rules can be written by hand or by an older build, and a + // missing delimiter here would throw while the editor was drawing itself. + const delimiter = typeof options?.delimiter === "string" ? options.delimiter : ","; if (delimiter.length !== 1) { return { root: null, @@ -192,8 +194,12 @@ function parseCsvSample(text: string, options: CsvOptions): ParsedSample { if (rows.length === 0) return { root: null, error: null }; - const names = options.hasHeader ? columnNames(rows[0]) : []; - const body = options.hasHeader ? rows.slice(1) : rows; + const hasHeader = options?.hasHeader !== false; + // One naming run for the whole file, so a column keeps the same name on every row and no two + // columns ever share one. It grows as wider rows turn up. + const used = new Set(); + const names = hasHeader ? rows[0].map((name, at) => unique(name || String(at + 1), used)) : []; + const body = hasHeader ? rows.slice(1) : rows; return { root: { @@ -205,18 +211,19 @@ function parseCsvSample(text: string, options: CsvOptions): ParsedSample { // sample from a row that has something in it — and a row with extra fields still // offers them. children: mergeShape( - body.flatMap((row) => - row.map((field, at) => { - const name = nameAt(names, at); - return { - key: name, - path: name, - kind: "value" as const, - sample: field ?? "", - children: [], - }; - }), - ), + body.flatMap((row) => { + // A row wider than anything seen before: the extra fields are named by position rather + // than dropped, which would be the same silent loss as two columns sharing a name. + while (names.length < row.length) names.push(unique(String(names.length + 1), used)); + + return row.map((field, at) => ({ + key: names[at], + path: names[at], + kind: "value" as const, + sample: field ?? "", + children: [], + })); + }), ), }, error: null, @@ -224,26 +231,30 @@ function parseCsvSample(text: string, options: CsvOptions): ParsedSample { } /** - * The column names a header record gives, with every column left addressable. + * `wanted`, or the first name after it that nothing has taken yet. + * + * Two columns cannot share a name: a rule reads a column by its name, so the second would be + * unreachable and the editor would offer one path for two different fields. Falling back to the + * position is not enough on its own either, because a header can be a number — a file headed + * `2,` names its first column `2` and then wants `2` again for the blank one beside it. * - * A blank name and a repeated one both fall back to the column's position — the name - * that column would have had with no header at all. Two columns sharing a name would - * mean the second silently replacing the first, which on the server is a column of the - * partner's file simply missing. + * The same rule as `CsvFormat.Unique` on the server, and it has to stay that way: a name only + * one side invents is a path the editor offers and the mapping cannot read. */ -function columnNames(header: string[]): string[] { - const used = new Set(); - return header.map((name, at) => { - const taken = name.length === 0 || used.has(name); - const chosen = taken ? String(at + 1) : name; - used.add(name); - used.add(chosen); - return chosen; - }); -} +function unique(wanted: string, used: Set): string { + if (!used.has(wanted)) { + used.add(wanted); + return wanted; + } -/** What the field at `at` is called: its header name, or its position counting from one. */ -const nameAt = (names: string[], at: number) => names[at] ?? String(at + 1); + for (let n = 2; ; n++) { + const candidate = `${wanted}_${n}`; + if (!used.has(candidate)) { + used.add(candidate); + return candidate; + } + } +} /** * Drops the byte-order mark Excel writes. diff --git a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/serialize.ts b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/serialize.ts index 34d41f24..e17527fb 100644 --- a/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/serialize.ts +++ b/SW.Bitween.Web/ClientApp/src/lib/nativeMapper/serialize.ts @@ -6,6 +6,7 @@ import { defaultCsvOptions, emptyRules, newRuleId, + type CsvOptions, type DocumentFormatId, type EditorFieldRule, type EditorListEntry, @@ -110,6 +111,28 @@ export function toWire(rules: EditorRules): MappingRules { return wire; } +/** + * Stored delimited-text options, with anything missing or unusable filled in. + * + * Rules can be written by hand or saved by an older build, so `{}` and a delimiter that is not a + * string both reach here. Neither is worth refusing the whole mapping over — an absent option + * already means the default — but neither can be passed on either: the editor reads the delimiter + * while it is drawing, and would throw rather than show a mapping. + */ +function csvOptionsFrom(stored: CsvOptions | undefined, format: DocumentFormatId) { + if (format !== "csv") return undefined; + + const defaults = defaultCsvOptions(); + return { + delimiter: + typeof stored?.delimiter === "string" && stored.delimiter.length > 0 + ? stored.delimiter + : defaults.delimiter, + hasHeader: typeof stored?.hasHeader === "boolean" ? stored.hasHeader : defaults.hasHeader, + byteOrderMark: stored?.byteOrderMark === true, + }; +} + /** Adds the editor's ids to rules that came off the wire. */ export function fromWire(rules: MappingRules): EditorRules { return { @@ -119,8 +142,8 @@ export function fromWire(rules: MappingRules): EditorRules { // Kept as they were stored. A mapping whose format is delimited text but whose options // are missing — hand-written, or saved before these existed — falls back to the same // defaults the server would apply, rather than to nothing. - sourceCsv: rules.sourceCsv ?? (rules.sourceFormat === "csv" ? defaultCsvOptions() : undefined), - targetCsv: rules.targetCsv ?? (rules.targetFormat === "csv" ? defaultCsvOptions() : undefined), + sourceCsv: csvOptionsFrom(rules.sourceCsv, rules.sourceFormat), + targetCsv: csvOptionsFrom(rules.targetCsv, rules.targetFormat), // Absent means year-first: a mapping saved before this existed could only ever have // read the unambiguous shapes correctly anyway. sourceDateOrder: rules.sourceDateOrder ?? "yearFirst",