diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3a1e12..051b297 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,8 @@ jobs: # BuildTests dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 && \ # Run Tests - dotnet test ./tests/bin/Release/net9.0/Tests.dll && \ + dotnet test ./tests/bin/Release/net10.0/Tests.dll && \ # BuildDataProcessing dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 && \ # Test CLRImports Script - pip install --no-cache-dir clr-loader==0.1.6 && python ./DataProcessing/bin/Release/net9.0/CLRImports.py + pip install --no-cache-dir clr-loader==0.2.9 && python ./DataProcessing/bin/Release/net10.0/CLRImports.py diff --git a/BrainLanguageMetricsEarningsCallsBase.cs b/BrainLanguageMetricsEarningsCallsBase.cs index 6729b19..05ca745 100644 --- a/BrainLanguageMetricsEarningsCallsBase.cs +++ b/BrainLanguageMetricsEarningsCallsBase.cs @@ -3,6 +3,8 @@ using NodaTime; using QuantConnect; using QuantConnect.Data; +using System.Globalization; +using System.Linq; namespace QuantConnect.DataSource { @@ -15,8 +17,44 @@ namespace QuantConnect.DataSource public abstract class BrainLanguageMetricsEarningsCallsBase : BaseData where T : BrainLanguageMetricsEarningsCallsBase, new() { + private static DateTime? ParseNullableDate(string v) + { + if (string.IsNullOrWhiteSpace(v)) + return null; + v = v.Trim(); + if (DateTime.TryParseExact(v, "yyyyMMdd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var d1)) + return d1; + if (DateTime.TryParseExact(v, "yyyy-MM-dd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var d2)) + return d2; + if (DateTime.TryParse(v, CultureInfo.InvariantCulture, DateTimeStyles.None, out var d3)) + return d3; + return null; + } + + private static int? ParseNullableIntSafe(string v) + { + if (string.IsNullOrWhiteSpace(v)) + return null; + + return int.TryParse(v, NumberStyles.Any, CultureInfo.InvariantCulture, out var x) + ? x + : null; + } + + private static decimal? ParseNullableDecimalSafe(string v) + { + if (string.IsNullOrWhiteSpace(v)) + return null; + + return decimal.TryParse(v, NumberStyles.Any, CultureInfo.InvariantCulture, out var x) + ? x + : null; + } + // === Transcript metadata === - public DateTime LastTranscriptDate { get; set; } + public DateTime? LastTranscriptDate { get; set; } public int? LastTranscriptQuarter { get; set; } public int? LastTranscriptYear { get; set; } @@ -149,7 +187,10 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date if (csv.Length < 4) return null; - var dataDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); + //var dataDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); + if (!DateTime.TryParseExact(csv[0], "yyyyMMdd", CultureInfo.InvariantCulture, + DateTimeStyles.None, out var dataDate)) + return null; var data = new T { @@ -158,93 +199,95 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date EndTime = dataDate.AddHours(12) }; - data.LastTranscriptDate = Parse.DateTimeExact(csv[1], "yyyyMMdd"); - data.LastTranscriptQuarter = ParseNullableInt(csv[2]); - data.LastTranscriptYear = ParseNullableInt(csv[3]); + int i = 1; - int i = 4; + // === Transcript metadata === + data.LastTranscriptDate = ParseNullableDate(csv.ElementAtOrDefault(i++)); + data.LastTranscriptQuarter = ParseNullableIntSafe(csv.ElementAtOrDefault(i++)); + data.LastTranscriptYear = ParseNullableIntSafe(csv.ElementAtOrDefault(i++)); // === MD === - data.MdNCharacters = ParseNullableDecimal(csv[i++]); - data.MdSentiment = ParseNullableDecimal(csv[i++]); - data.MdScoreUncertainty = ParseNullableDecimal(csv[i++]); - data.MdScoreLitigious = ParseNullableDecimal(csv[i++]); - data.MdScoreConstraining = ParseNullableDecimal(csv[i++]); - data.MdReadability = ParseNullableDecimal(csv[i++]); - data.MdLexicalRichness = ParseNullableDecimal(csv[i++]); - data.MdLexicalDensity = ParseNullableDecimal(csv[i++]); - data.MdSpecificDensity = ParseNullableDecimal(csv[i++]); + data.MdNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); // === AQ === - data.AqNCharacters = ParseNullableDecimal(csv[i++]); - data.AqSentiment = ParseNullableDecimal(csv[i++]); - data.AqScoreUncertainty = ParseNullableDecimal(csv[i++]); - data.AqScoreLitigious = ParseNullableDecimal(csv[i++]); - data.AqScoreConstraining = ParseNullableDecimal(csv[i++]); + data.AqNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); // === MA === - data.MaNCharacters = ParseNullableDecimal(csv[i++]); - data.MaSentiment = ParseNullableDecimal(csv[i++]); - data.MaScoreUncertainty = ParseNullableDecimal(csv[i++]); - data.MaScoreLitigious = ParseNullableDecimal(csv[i++]); - data.MaScoreConstraining = ParseNullableDecimal(csv[i++]); - data.MaReadability = ParseNullableDecimal(csv[i++]); - data.MaLexicalRichness = ParseNullableDecimal(csv[i++]); - data.MaLexicalDensity = ParseNullableDecimal(csv[i++]); - data.MaSpecificDensity = ParseNullableDecimal(csv[i++]); + data.MaNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); if (csv.Length <= i) return data; // === Prev transcript === - data.PrevTranscriptDate = Parse.DateTimeExact(csv[i++], "yyyyMMdd"); - data.PrevTranscriptQuarter = ParseNullableInt(csv[i++]); - data.PrevTranscriptYear = ParseNullableInt(csv[i++]); + data.PrevTranscriptDate = ParseNullableDate(csv.ElementAtOrDefault(i++)); + data.PrevTranscriptQuarter = ParseNullableIntSafe(csv.ElementAtOrDefault(i++)); + data.PrevTranscriptYear = ParseNullableIntSafe(csv.ElementAtOrDefault(i++)); // === MD deltas & similarities === - data.MdDeltaPercNCharacters = ParseNullableDecimal(csv[i++]); - data.MdDeltaSentiment = ParseNullableDecimal(csv[i++]); - data.MdDeltaScoreUncertainty = ParseNullableDecimal(csv[i++]); - data.MdDeltaScoreLitigious = ParseNullableDecimal(csv[i++]); - data.MdDeltaScoreConstraining = ParseNullableDecimal(csv[i++]); - data.MdDeltaReadability = ParseNullableDecimal(csv[i++]); - data.MdDeltaLexicalRichness = ParseNullableDecimal(csv[i++]); - data.MdDeltaLexicalDensity = ParseNullableDecimal(csv[i++]); - data.MdDeltaSpecificDensity = ParseNullableDecimal(csv[i++]); - - data.MdSimilarityAll = ParseNullableDecimal(csv[i++]); - data.MdSimilarityPositive = ParseNullableDecimal(csv[i++]); - data.MdSimilarityNegative = ParseNullableDecimal(csv[i++]); - data.MdSimilarityUncertainty = ParseNullableDecimal(csv[i++]); - data.MdSimilarityLitigious = ParseNullableDecimal(csv[i++]); - data.MdSimilarityConstraining = ParseNullableDecimal(csv[i++]); + data.MdDeltaPercNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdDeltaSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + + data.MdSimilarityAll = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdSimilarityPositive = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdSimilarityNegative = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdSimilarityUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdSimilarityLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MdSimilarityConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); // === AQ deltas & similarities === - data.AqDeltaPercNCharacters = ParseNullableDecimal(csv[i++]); - data.AqDeltaSentimentDelta = ParseNullableDecimal(csv[i++]); - data.AqDeltaScoreUncertainty = ParseNullableDecimal(csv[i++]); - data.AqDeltaScoreLitigious = ParseNullableDecimal(csv[i++]); - data.AqDeltaScoreConstraining = ParseNullableDecimal(csv[i++]); - data.AqSimilarityAll = ParseNullableDecimal(csv[i++]); - data.AqSimilarityPositive = ParseNullableDecimal(csv[i++]); - data.AqSimilarityNegative = ParseNullableDecimal(csv[i++]); + data.AqDeltaPercNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqDeltaSentimentDelta = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqDeltaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqDeltaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqDeltaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqSimilarityAll = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqSimilarityPositive = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.AqSimilarityNegative = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); // === MA deltas & similarities === - data.MaDeltaPercNCharacters = ParseNullableDecimal(csv[i++]); - data.MaDeltaSentimentDelta = ParseNullableDecimal(csv[i++]); - data.MaDeltaScoreUncertainty = ParseNullableDecimal(csv[i++]); - data.MaDeltaScoreLitigious = ParseNullableDecimal(csv[i++]); - data.MaDeltaScoreConstraining = ParseNullableDecimal(csv[i++]); - data.MaDeltaReadability = ParseNullableDecimal(csv[i++]); - data.MaDeltaLexicalRichness = ParseNullableDecimal(csv[i++]); - data.MaDeltaLexicalDensity = ParseNullableDecimal(csv[i++]); - data.MaDeltaSpecificDensity = ParseNullableDecimal(csv[i++]); - data.MaSimilarityAll = ParseNullableDecimal(csv[i++]); - data.MaSimilarityPositive = ParseNullableDecimal(csv[i++]); - data.MaSimilarityNegative = ParseNullableDecimal(csv[i++]); - data.MaSimilarityUncertainty = ParseNullableDecimal(csv[i++]); - data.MaSimilarityLitigious = ParseNullableDecimal(csv[i++]); - data.MaSimilarityConstraining = ParseNullableDecimal(csv[i++]); + data.MaDeltaPercNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaSentimentDelta = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaDeltaSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + + data.MaSimilarityAll = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaSimilarityPositive = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaSimilarityNegative = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaSimilarityUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaSimilarityLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); + data.MaSimilarityConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++)); return data; } diff --git a/DataProcessing/CLRImports.py b/DataProcessing/CLRImports.py index 824e378..b9857f9 100644 --- a/DataProcessing/CLRImports.py +++ b/DataProcessing/CLRImports.py @@ -6,7 +6,7 @@ # process.runtimeconfig.json is created when we build the DataProcessing Project: # dotnet build .\DataProcessing\DataProcessing.csproj -set_runtime(get_coreclr(join(dirname(realpath(__file__)), "process.runtimeconfig.json"))) +set_runtime(get_coreclr(runtime_config=join(dirname(realpath(__file__)), "process.runtimeconfig.json"))) from AlgorithmImports import * from QuantConnect.Lean.Engine.DataFeeds import * diff --git a/tests/BrainLanguageMetricsEarningsCallsTests.cs b/tests/BrainLanguageMetricsEarningsCallsTests.cs index cd56dbf..987b23e 100644 --- a/tests/BrainLanguageMetricsEarningsCallsTests.cs +++ b/tests/BrainLanguageMetricsEarningsCallsTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using NUnit.Framework; using QuantConnect; using QuantConnect.Data; @@ -26,28 +27,49 @@ public void Setup() ); } + /// + /// Reader layout expects up to index 68 inclusive => 69 columns total (0..68) + /// + private static string BuildRow69(params (int index, string value)[] overrides) + { + var cols = Enumerable.Repeat("1", 69).ToArray(); + cols[0] = "20250910"; // required row date (yyyyMMdd) + + foreach (var (index, value) in overrides) + { + if (index < 0 || index >= cols.Length) + throw new ArgumentOutOfRangeException(nameof(overrides), $"Index {index} out of range for 69-col row."); + cols[index] = value; + } + + return string.Join(",", cols); + } + [Test] - public void Reader_Parses_All_Fields_Correctly() + public void Reader_Parses_All_Fields_Correctly_WithPrevTranscript() { - // FULL CSV reflecting production-format BLMECT output + // FULL CSV reflecting production-format BLMECT output (69 columns) string csv = "20250910,20250731,3,2025," + "20851,0.9062,0.1141,0.0537,0.0201,11.178,0.3696,0.5447,0.0717," + "9032,0.6471,0.4058,0.029,0.0145," + "14849,0.9636,0.2152,0.1013,0.0506,8.481,0.4344,0.4326,0.0619," + - "20250501,2,2025," + + "20250501,2,2025," + // <-- Prev transcript metadata (27..29) "0.0831,0.0124,-0.0298,0.0234,-0.0026,-0.0935,-0.0597,0.0122,0.0026," + "0.659,0.7342,0.7638,0.8332,0.9562,0.2," + "-0.0662,0.6471,-0.0386,-0.0266,0.0145," + "0.7541,0.3921,0.7232," + - "0.0267,0.3549,-0.0495,0.0621,0.0212,-0.4425,-0.0319,-0.0182,-0.0206," + - "0.6139,0.4824,0.9113,0.482,0.75,0"; + "0.0267,0.3549,-0.0495,0.0621,0.0212,-0.4425,-0.0319,-0.0182,-0.0206," + // MA deltas (9) + "0.6139,0.4824,0.9113,0.482,0.75,0"; // MA similarities (6) var data = new BrainLanguageMetricsEarningsCalls() .Reader(_config, csv, DateTime.UtcNow, false) as BrainLanguageMetricsEarningsCalls; Assert.NotNull(data, "Reader returned null"); + // === ROW DATE === + Assert.AreEqual(new DateTime(2025, 9, 10), data.Time.Date); + // === CORE METADATA === Assert.AreEqual(new DateTime(2025, 7, 31), data.LastTranscriptDate); Assert.AreEqual(3, data.LastTranscriptQuarter); @@ -68,7 +90,7 @@ public void Reader_Parses_All_Fields_Correctly() Assert.AreEqual(9032m, data.AqNCharacters); Assert.AreEqual(0.6471m, data.AqSentiment); Assert.AreEqual(0.4058m, data.AqScoreUncertainty); - Assert.AreEqual(0.029m, data.AqScoreLitigious); + Assert.AreEqual(0.029m, data.AqScoreLitigious); Assert.AreEqual(0.0145m, data.AqScoreConstraining); // === MA METRICS === @@ -77,7 +99,7 @@ public void Reader_Parses_All_Fields_Correctly() Assert.AreEqual(0.2152m, data.MaScoreUncertainty); Assert.AreEqual(0.1013m, data.MaScoreLitigious); Assert.AreEqual(0.0506m, data.MaScoreConstraining); - Assert.AreEqual(8.481m, data.MaReadability); + Assert.AreEqual(8.481m, data.MaReadability); Assert.AreEqual(0.4344m, data.MaLexicalRichness); Assert.AreEqual(0.4326m, data.MaLexicalDensity); Assert.AreEqual(0.0619m, data.MaSpecificDensity); @@ -88,54 +110,196 @@ public void Reader_Parses_All_Fields_Correctly() Assert.AreEqual(2025, data.PrevTranscriptYear); // === MD DELTAS (9 fields) === - Assert.AreEqual(0.0831m, data.MdDeltaPercNCharacters); - Assert.AreEqual(0.0124m, data.MdDeltaSentiment); + Assert.AreEqual(0.0831m, data.MdDeltaPercNCharacters); + Assert.AreEqual(0.0124m, data.MdDeltaSentiment); Assert.AreEqual(-0.0298m, data.MdDeltaScoreUncertainty); - Assert.AreEqual(0.0234m, data.MdDeltaScoreLitigious); + Assert.AreEqual(0.0234m, data.MdDeltaScoreLitigious); Assert.AreEqual(-0.0026m, data.MdDeltaScoreConstraining); Assert.AreEqual(-0.0935m, data.MdDeltaReadability); Assert.AreEqual(-0.0597m, data.MdDeltaLexicalRichness); - Assert.AreEqual(0.0122m, data.MdDeltaLexicalDensity); - Assert.AreEqual(0.0026m, data.MdDeltaSpecificDensity); + Assert.AreEqual(0.0122m, data.MdDeltaLexicalDensity); + Assert.AreEqual(0.0026m, data.MdDeltaSpecificDensity); // === MD SIMILARITIES (6 fields) === - Assert.AreEqual(0.659m, data.MdSimilarityAll); + Assert.AreEqual(0.659m, data.MdSimilarityAll); Assert.AreEqual(0.7342m, data.MdSimilarityPositive); Assert.AreEqual(0.7638m, data.MdSimilarityNegative); Assert.AreEqual(0.8332m, data.MdSimilarityUncertainty); Assert.AreEqual(0.9562m, data.MdSimilarityLitigious); - Assert.AreEqual(0.2m, data.MdSimilarityConstraining); + Assert.AreEqual(0.2m, data.MdSimilarityConstraining); // === AQ DELTAS (5 fields) === Assert.AreEqual(-0.0662m, data.AqDeltaPercNCharacters); - Assert.AreEqual(0.6471m, data.AqDeltaSentimentDelta); + Assert.AreEqual(0.6471m, data.AqDeltaSentimentDelta); Assert.AreEqual(-0.0386m, data.AqDeltaScoreUncertainty); Assert.AreEqual(-0.0266m, data.AqDeltaScoreLitigious); - Assert.AreEqual(0.0145m, data.AqDeltaScoreConstraining); + Assert.AreEqual(0.0145m, data.AqDeltaScoreConstraining); // === AQ SIMILARITIES (3 fields) === Assert.AreEqual(0.7541m, data.AqSimilarityAll); Assert.AreEqual(0.3921m, data.AqSimilarityPositive); Assert.AreEqual(0.7232m, data.AqSimilarityNegative); - // === MA DELTAS (10 fields) === - Assert.AreEqual(0.0267m, data.MaDeltaPercNCharacters); - Assert.AreEqual(0.3549m, data.MaDeltaSentimentDelta); + // === MA DELTAS (9 fields) === + Assert.AreEqual(0.0267m, data.MaDeltaPercNCharacters); + Assert.AreEqual(0.3549m, data.MaDeltaSentimentDelta); Assert.AreEqual(-0.0495m, data.MaDeltaScoreUncertainty); - Assert.AreEqual(0.0621m, data.MaDeltaScoreLitigious); - Assert.AreEqual(0.0212m, data.MaDeltaScoreConstraining); + Assert.AreEqual(0.0621m, data.MaDeltaScoreLitigious); + Assert.AreEqual(0.0212m, data.MaDeltaScoreConstraining); Assert.AreEqual(-0.4425m, data.MaDeltaReadability); Assert.AreEqual(-0.0319m, data.MaDeltaLexicalRichness); Assert.AreEqual(-0.0182m, data.MaDeltaLexicalDensity); Assert.AreEqual(-0.0206m, data.MaDeltaSpecificDensity); - + // === MA SIMILARITIES (6 fields) === Assert.AreEqual(0.6139m, data.MaSimilarityAll); Assert.AreEqual(0.4824m, data.MaSimilarityPositive); Assert.AreEqual(0.9113m, data.MaSimilarityNegative); - Assert.AreEqual(0.482m, data.MaSimilarityUncertainty); - Assert.AreEqual(0.75m, data.MaSimilarityLitigious); - Assert.AreEqual(0m, data.MaSimilarityConstraining); + Assert.AreEqual(0.482m, data.MaSimilarityUncertainty); + Assert.AreEqual(0.75m, data.MaSimilarityLitigious); + Assert.AreEqual(0m, data.MaSimilarityConstraining); + } + + [Test] + public void Reader_EmptyOptionalFields_ReturnNulls_AndDoesNotThrow() + { + // Blank out representative optional fields across the row + var csv = BuildRow69( + (1, ""), // LastTranscriptDate + (2, ""), // LastTranscriptQuarter + (3, ""), // LastTranscriptYear + (5, ""), // MdSentiment + (14, ""), // AqSentiment + (23, ""), // MaReadability + (27, ""), // PrevTranscriptDate + (28, ""), // PrevTranscriptQuarter + (29, ""), // PrevTranscriptYear + (39, ""), // MdSimilarityAll + (54, "") // MaDeltaSentimentDelta + ); + + BrainLanguageMetricsEarningsCalls data = null; + + Assert.DoesNotThrow(() => + { + data = new BrainLanguageMetricsEarningsCalls() + .Reader(_config, csv, DateTime.UtcNow, false) as BrainLanguageMetricsEarningsCalls; + }); + + Assert.NotNull(data); + Assert.AreEqual(new DateTime(2025, 9, 10), data.Time.Date); + + Assert.IsNull(data.LastTranscriptDate); + Assert.IsNull(data.LastTranscriptQuarter); + Assert.IsNull(data.LastTranscriptYear); + + Assert.IsNull(data.MdSentiment); + Assert.IsNull(data.AqSentiment); + Assert.IsNull(data.MaReadability); + + Assert.IsNull(data.PrevTranscriptDate); + Assert.IsNull(data.PrevTranscriptQuarter); + Assert.IsNull(data.PrevTranscriptYear); + + Assert.IsNull(data.MdSimilarityAll); + Assert.IsNull(data.MaDeltaSentimentDelta); + } + + [Test] + public void Reader_InvalidOptionalFields_ReturnNulls_AndDoesNotThrow() + { + // Malformed optional fields should become null because Reader uses TryParse-based helpers. + var csv = BuildRow69( + (1, "not-a-date"), // LastTranscriptDate + (2, "xyz"), // LastTranscriptQuarter + (5, "abc"), // MdSentiment + (27, "2025-99-99"),// PrevTranscriptDate + (39, "nan"), // MdSimilarityAll + (54, "oops") // MaDeltaSentimentDelta + ); + + BrainLanguageMetricsEarningsCalls data = null; + + Assert.DoesNotThrow(() => + { + data = new BrainLanguageMetricsEarningsCalls() + .Reader(_config, csv, DateTime.UtcNow, false) as BrainLanguageMetricsEarningsCalls; + }); + + Assert.NotNull(data); + Assert.AreEqual(new DateTime(2025, 9, 10), data.Time.Date); + + Assert.IsNull(data.LastTranscriptDate); + Assert.IsNull(data.LastTranscriptQuarter); + Assert.IsNull(data.MdSentiment); + + Assert.IsNull(data.PrevTranscriptDate); + Assert.IsNull(data.MdSimilarityAll); + Assert.IsNull(data.MaDeltaSentimentDelta); + } + + [Test] + public void Reader_ShortRow_DoesNotThrow_ParsesAvailableFields() + { + // Only 27 columns (0..26) => through MA metrics. + var cols = Enumerable.Repeat("1", 27).ToArray(); + cols[0] = "20250910"; + cols[1] = "20250731"; + cols[2] = "3"; + cols[3] = "2025"; + var csv = string.Join(",", cols); + + BrainLanguageMetricsEarningsCalls data = null; + + Assert.DoesNotThrow(() => + { + data = new BrainLanguageMetricsEarningsCalls() + .Reader(_config, csv, DateTime.UtcNow, false) as BrainLanguageMetricsEarningsCalls; + }); + + Assert.NotNull(data); + + Assert.AreEqual(new DateTime(2025, 9, 10), data.Time.Date); + Assert.AreEqual(new DateTime(2025, 7, 31), data.LastTranscriptDate); + Assert.AreEqual(3, data.LastTranscriptQuarter); + Assert.AreEqual(2025, data.LastTranscriptYear); + + // Not present in short row => should be null + Assert.IsNull(data.PrevTranscriptDate); + Assert.IsNull(data.PrevTranscriptQuarter); + Assert.IsNull(data.PrevTranscriptYear); + } + + [Test] + public void Reader_EmptyLine_ReturnsNull() + { + var data = new BrainLanguageMetricsEarningsCalls() + .Reader(_config, " ", DateTime.UtcNow, false); + + Assert.IsNull(data); + } + + [Test] + public void Reader_TooShortCsv_ReturnsNull() + { + var data = new BrainLanguageMetricsEarningsCalls() + .Reader(_config, "20250910,20250731,3", DateTime.UtcNow, false); + + Assert.IsNull(data); + } + + [Test] + public void Reader_InvalidRowDate_ReturnsNull_AndDoesNotThrow() + { + var csv = BuildRow69((0, "bad-date")); + + Assert.DoesNotThrow(() => + { + var data = new BrainLanguageMetricsEarningsCalls() + .Reader(_config, csv, DateTime.UtcNow, false); + Assert.IsNull(data); + }); } + } } diff --git a/tests/BrainWikipediaPageViewsTests.cs b/tests/BrainWikipediaPageViewsTests.cs index 8e8fa93..a48be26 100644 --- a/tests/BrainWikipediaPageViewsTests.cs +++ b/tests/BrainWikipediaPageViewsTests.cs @@ -29,16 +29,17 @@ public void Setup() [Test] public void Reader_Parses_All_Fields_Correctly() { - // FULL CSV reflecting production-format BWPV output - string csv = - "20250910, 14220, 4.2139, 58460, 0.5318, 218379, -0.6219"; + // 7 columns required + var csv = "20250910,14220,4.2139,58460,0.5318,218379,-0.6219"; var data = new BrainWikipediaPageViews() .Reader(_config, csv, DateTime.UtcNow, false) as BrainWikipediaPageViews; Assert.NotNull(data, "Reader returned null"); - // === METADATA === + // Time may be midnight or midday in some datasets; validate date only + Assert.AreEqual(new DateTime(2025, 9, 10), data.Time.Date); + Assert.AreEqual(14220m, data.NumberViews1); Assert.AreEqual(4.2139m, data.Buzz1); Assert.AreEqual(58460m, data.NumberViews7); @@ -46,5 +47,79 @@ public void Reader_Parses_All_Fields_Correctly() Assert.AreEqual(218379m, data.NumberViews30); Assert.AreEqual(-0.6219m, data.Buzz30); } + + [Test] + public void Reader_Allows_Empty_Numeric_Fields_As_Null() + { + // Keep 7 columns; last value empty -> Buzz30 should be null + var csv = "20250910,14220,4.2139,58460,0.5318,218379,"; + + var data = new BrainWikipediaPageViews() + .Reader(_config, csv, DateTime.UtcNow, false) as BrainWikipediaPageViews; + + Assert.NotNull(data); + + Assert.AreEqual(new DateTime(2025, 9, 10), data.Time.Date); + Assert.AreEqual(14220m, data.NumberViews1); + Assert.IsNull(data.Buzz30); + } + + [Test] + public void Reader_ReturnsNull_On_Whitespace_Line() + { + var data = new BrainWikipediaPageViews() + .Reader(_config, " ", DateTime.UtcNow, false); + + Assert.IsNull(data); + } + + [Test] + public void Reader_ReturnsNull_When_TooFew_Columns() + { + // Less than 7 columns -> null + var csv = "20250910,14220,4.2139,58460,0.5318,218379"; // only 6 columns + + var data = new BrainWikipediaPageViews() + .Reader(_config, csv, DateTime.UtcNow, false); + + Assert.IsNull(data); + } + + [Test] + public void Reader_Throws_On_Invalid_RowDate() + { + var csv = "bad-date,14220,4.2139,58460,0.5318,218379,-0.6219"; + + Assert.Throws(() => + new BrainWikipediaPageViews().Reader(_config, csv, DateTime.UtcNow, false)); + } + + [Test] + public void Reader_Throws_On_Invalid_Numeric_Field() + { + var csv = "20250910,14220,abc,58460,0.5318,218379,-0.6219"; + + Assert.Throws(() => + new BrainWikipediaPageViews().Reader(_config, csv, DateTime.UtcNow, false)); + } + + [Test] + public void Reader_Allows_Empty_Middle_Field_As_Null() + { + // Buzz7 empty (still 7 columns) => null + var csv = "20250910,14220,4.2139,58460,,218379,-0.6219"; + + var data = new BrainWikipediaPageViews() + .Reader(_config, csv, DateTime.UtcNow, false) as BrainWikipediaPageViews; + + Assert.NotNull(data); + + Assert.AreEqual(14220m, data.NumberViews1); + Assert.AreEqual(4.2139m, data.Buzz1); + Assert.AreEqual(58460m, data.NumberViews7); + Assert.IsNull(data.Buzz7); + Assert.AreEqual(218379m, data.NumberViews30); + Assert.AreEqual(-0.6219m, data.Buzz30); + } } }