Skip to content

Handle all data scenarios #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AlexCatarino merged 4 commits into from
Dec 30, 2025
Merged

Handle all data scenarios #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1043c71
Handle all data scenarios
ashutoshrana171 Dec 29, 2025
66e3237
Update build.yml to use net10
ashutoshrana171 Dec 30, 2025
e0570f1
update clr-loader==0.2.9
ashutoshrana171 Dec 30, 2025
47ebb99
Update CLRImports.py
ashutoshrana171 Dec 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ jobs:
# BuildTests
dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 && \
# Run Tests
dotnet test ./tests/bin/Release/net9.0/Tests.dll && \
dotnet test ./tests/bin/Release/net10.0/Tests.dll && \
# BuildDataProcessing
dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 && \
# Test CLRImports Script
pip install --no-cache-dir clr-loader==0.1.6 && python ./DataProcessing/bin/Release/net9.0/CLRImports.py
pip install --no-cache-dir clr-loader==0.2.9 && python ./DataProcessing/bin/Release/net10.0/CLRImports.py
185 changes: 114 additions & 71 deletions BrainLanguageMetricsEarningsCallsBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
using NodaTime;
using QuantConnect;
using QuantConnect.Data;
using System.Globalization;
using System.Linq;

namespace QuantConnect.DataSource
{
Expand All @@ -15,8 +17,44 @@ namespace QuantConnect.DataSource
public abstract class BrainLanguageMetricsEarningsCallsBase<T> : BaseData
where T : BrainLanguageMetricsEarningsCallsBase<T>, new()
{
private static DateTime? ParseNullableDate(string v)
{
if (string.IsNullOrWhiteSpace(v))
return null;
v = v.Trim();
if (DateTime.TryParseExact(v, "yyyyMMdd", CultureInfo.InvariantCulture,
DateTimeStyles.None, out var d1))
return d1;
if (DateTime.TryParseExact(v, "yyyy-MM-dd", CultureInfo.InvariantCulture,
DateTimeStyles.None, out var d2))
return d2;
if (DateTime.TryParse(v, CultureInfo.InvariantCulture, DateTimeStyles.None, out var d3))
return d3;
return null;
}

private static int? ParseNullableIntSafe(string v)
{
if (string.IsNullOrWhiteSpace(v))
return null;

return int.TryParse(v, NumberStyles.Any, CultureInfo.InvariantCulture, out var x)
? x
: null;
}

private static decimal? ParseNullableDecimalSafe(string v)
{
if (string.IsNullOrWhiteSpace(v))
return null;

return decimal.TryParse(v, NumberStyles.Any, CultureInfo.InvariantCulture, out var x)
? x
: null;
}

// === Transcript metadata ===
public DateTime LastTranscriptDate { get; set; }
public DateTime? LastTranscriptDate { get; set; }
public int? LastTranscriptQuarter { get; set; }
public int? LastTranscriptYear { get; set; }

Expand Down Expand Up @@ -149,7 +187,10 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
if (csv.Length < 4)
return null;

var dataDate = Parse.DateTimeExact(csv[0], "yyyyMMdd");
//var dataDate = Parse.DateTimeExact(csv[0], "yyyyMMdd");
if (!DateTime.TryParseExact(csv[0], "yyyyMMdd", CultureInfo.InvariantCulture,
DateTimeStyles.None, out var dataDate))
return null;

var data = new T
{
Expand All @@ -158,93 +199,95 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
EndTime = dataDate.AddHours(12)
};

data.LastTranscriptDate = Parse.DateTimeExact(csv[1], "yyyyMMdd");
data.LastTranscriptQuarter = ParseNullableInt(csv[2]);
data.LastTranscriptYear = ParseNullableInt(csv[3]);
int i = 1;

int i = 4;
// === Transcript metadata ===
data.LastTranscriptDate = ParseNullableDate(csv.ElementAtOrDefault(i++));
data.LastTranscriptQuarter = ParseNullableIntSafe(csv.ElementAtOrDefault(i++));
data.LastTranscriptYear = ParseNullableIntSafe(csv.ElementAtOrDefault(i++));

// === MD ===
data.MdNCharacters = ParseNullableDecimal(csv[i++]);
data.MdSentiment = ParseNullableDecimal(csv[i++]);
data.MdScoreUncertainty = ParseNullableDecimal(csv[i++]);
data.MdScoreLitigious = ParseNullableDecimal(csv[i++]);
data.MdScoreConstraining = ParseNullableDecimal(csv[i++]);
data.MdReadability = ParseNullableDecimal(csv[i++]);
data.MdLexicalRichness = ParseNullableDecimal(csv[i++]);
data.MdLexicalDensity = ParseNullableDecimal(csv[i++]);
data.MdSpecificDensity = ParseNullableDecimal(csv[i++]);
data.MdNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

// === AQ ===
data.AqNCharacters = ParseNullableDecimal(csv[i++]);
data.AqSentiment = ParseNullableDecimal(csv[i++]);
data.AqScoreUncertainty = ParseNullableDecimal(csv[i++]);
data.AqScoreLitigious = ParseNullableDecimal(csv[i++]);
data.AqScoreConstraining = ParseNullableDecimal(csv[i++]);
data.AqNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

// === MA ===
data.MaNCharacters = ParseNullableDecimal(csv[i++]);
data.MaSentiment = ParseNullableDecimal(csv[i++]);
data.MaScoreUncertainty = ParseNullableDecimal(csv[i++]);
data.MaScoreLitigious = ParseNullableDecimal(csv[i++]);
data.MaScoreConstraining = ParseNullableDecimal(csv[i++]);
data.MaReadability = ParseNullableDecimal(csv[i++]);
data.MaLexicalRichness = ParseNullableDecimal(csv[i++]);
data.MaLexicalDensity = ParseNullableDecimal(csv[i++]);
data.MaSpecificDensity = ParseNullableDecimal(csv[i++]);
data.MaNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

if (csv.Length <= i)
return data;

// === Prev transcript ===
data.PrevTranscriptDate = Parse.DateTimeExact(csv[i++], "yyyyMMdd");
data.PrevTranscriptQuarter = ParseNullableInt(csv[i++]);
data.PrevTranscriptYear = ParseNullableInt(csv[i++]);
data.PrevTranscriptDate = ParseNullableDate(csv.ElementAtOrDefault(i++));
data.PrevTranscriptQuarter = ParseNullableIntSafe(csv.ElementAtOrDefault(i++));
data.PrevTranscriptYear = ParseNullableIntSafe(csv.ElementAtOrDefault(i++));

// === MD deltas & similarities ===
data.MdDeltaPercNCharacters = ParseNullableDecimal(csv[i++]);
data.MdDeltaSentiment = ParseNullableDecimal(csv[i++]);
data.MdDeltaScoreUncertainty = ParseNullableDecimal(csv[i++]);
data.MdDeltaScoreLitigious = ParseNullableDecimal(csv[i++]);
data.MdDeltaScoreConstraining = ParseNullableDecimal(csv[i++]);
data.MdDeltaReadability = ParseNullableDecimal(csv[i++]);
data.MdDeltaLexicalRichness = ParseNullableDecimal(csv[i++]);
data.MdDeltaLexicalDensity = ParseNullableDecimal(csv[i++]);
data.MdDeltaSpecificDensity = ParseNullableDecimal(csv[i++]);

data.MdSimilarityAll = ParseNullableDecimal(csv[i++]);
data.MdSimilarityPositive = ParseNullableDecimal(csv[i++]);
data.MdSimilarityNegative = ParseNullableDecimal(csv[i++]);
data.MdSimilarityUncertainty = ParseNullableDecimal(csv[i++]);
data.MdSimilarityLitigious = ParseNullableDecimal(csv[i++]);
data.MdSimilarityConstraining = ParseNullableDecimal(csv[i++]);
data.MdDeltaPercNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaSentiment = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdDeltaSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

data.MdSimilarityAll = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdSimilarityPositive = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdSimilarityNegative = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdSimilarityUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdSimilarityLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MdSimilarityConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

// === AQ deltas & similarities ===
data.AqDeltaPercNCharacters = ParseNullableDecimal(csv[i++]);
data.AqDeltaSentimentDelta = ParseNullableDecimal(csv[i++]);
data.AqDeltaScoreUncertainty = ParseNullableDecimal(csv[i++]);
data.AqDeltaScoreLitigious = ParseNullableDecimal(csv[i++]);
data.AqDeltaScoreConstraining = ParseNullableDecimal(csv[i++]);
data.AqSimilarityAll = ParseNullableDecimal(csv[i++]);
data.AqSimilarityPositive = ParseNullableDecimal(csv[i++]);
data.AqSimilarityNegative = ParseNullableDecimal(csv[i++]);
data.AqDeltaPercNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqDeltaSentimentDelta = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqDeltaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqDeltaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqDeltaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqSimilarityAll = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqSimilarityPositive = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.AqSimilarityNegative = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

// === MA deltas & similarities ===
data.MaDeltaPercNCharacters = ParseNullableDecimal(csv[i++]);
data.MaDeltaSentimentDelta = ParseNullableDecimal(csv[i++]);
data.MaDeltaScoreUncertainty = ParseNullableDecimal(csv[i++]);
data.MaDeltaScoreLitigious = ParseNullableDecimal(csv[i++]);
data.MaDeltaScoreConstraining = ParseNullableDecimal(csv[i++]);
data.MaDeltaReadability = ParseNullableDecimal(csv[i++]);
data.MaDeltaLexicalRichness = ParseNullableDecimal(csv[i++]);
data.MaDeltaLexicalDensity = ParseNullableDecimal(csv[i++]);
data.MaDeltaSpecificDensity = ParseNullableDecimal(csv[i++]);
data.MaSimilarityAll = ParseNullableDecimal(csv[i++]);
data.MaSimilarityPositive = ParseNullableDecimal(csv[i++]);
data.MaSimilarityNegative = ParseNullableDecimal(csv[i++]);
data.MaSimilarityUncertainty = ParseNullableDecimal(csv[i++]);
data.MaSimilarityLitigious = ParseNullableDecimal(csv[i++]);
data.MaSimilarityConstraining = ParseNullableDecimal(csv[i++]);
data.MaDeltaPercNCharacters = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaSentimentDelta = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaScoreUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaScoreLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaScoreConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaReadability = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaLexicalRichness = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaLexicalDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaDeltaSpecificDensity = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

data.MaSimilarityAll = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaSimilarityPositive = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaSimilarityNegative = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaSimilarityUncertainty = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaSimilarityLitigious = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));
data.MaSimilarityConstraining = ParseNullableDecimalSafe(csv.ElementAtOrDefault(i++));

return data;
}
Expand Down
2 changes: 1 addition & 1 deletion DataProcessing/CLRImports.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

# process.runtimeconfig.json is created when we build the DataProcessing Project:
# dotnet build .\DataProcessing\DataProcessing.csproj
set_runtime(get_coreclr(join(dirname(realpath(__file__)), "process.runtimeconfig.json")))
set_runtime(get_coreclr(runtime_config=join(dirname(realpath(__file__)), "process.runtimeconfig.json")))

from AlgorithmImports import *
from QuantConnect.Lean.Engine.DataFeeds import *
Expand Down
Loading