Skip to content
This repository was archived by the owner on Aug 2, 2023. It is now read-only.

Robust dataframe io #2938

Closed
chriss2401 wants to merge 2 commits into from
Closed

Robust dataframe io #2938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b894fd8
Moved main LoadCsv from Data.Analysis to IO project. Removed duplicat…
Jun 15, 2020
d4ac04a
Adding M.D.A.IO test project to .sln file and fixing TestAppendRow un…
Jun 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions corefxlab.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Web.Compatibility.Te
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.Interactive.Tests", "tests\Microsoft.Data.Analysis.Interactive.Tests\Microsoft.Data.Analysis.Interactive.Tests.csproj", "{C4DEF44E-2FAA-4887-9C7F-36D54A4E975D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.IO", "src\Microsoft.Data.Analysis.IO\Microsoft.Data.Analysis.IO.csproj", "{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.IO.Tests", "tests\Microsoft.Data.Analysis.IO.Tests\Microsoft.Data.Analysis.IO.Tests.csproj", "{F9711AFA-36E4-4D90-89B7-D306382CE4F3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -759,6 +763,30 @@ Global
{0C11386C-E3CC-45EB-B932-9297A8DD6649}.Release|x64.Build.0 = Release|Any CPU
{0C11386C-E3CC-45EB-B932-9297A8DD6649}.Release|x86.ActiveCfg = Release|Any CPU
{0C11386C-E3CC-45EB-B932-9297A8DD6649}.Release|x86.Build.0 = Release|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Debug|x64.ActiveCfg = Debug|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Debug|x64.Build.0 = Debug|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Debug|x86.ActiveCfg = Debug|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Debug|x86.Build.0 = Debug|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Release|Any CPU.Build.0 = Release|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Release|x64.ActiveCfg = Release|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Release|x64.Build.0 = Release|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Release|x86.ActiveCfg = Release|Any CPU
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E}.Release|x86.Build.0 = Release|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Debug|x64.ActiveCfg = Debug|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Debug|x64.Build.0 = Debug|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Debug|x86.ActiveCfg = Debug|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Debug|x86.Build.0 = Debug|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Release|Any CPU.Build.0 = Release|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Release|x64.ActiveCfg = Release|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Release|x64.Build.0 = Release|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Release|x86.ActiveCfg = Release|Any CPU
{F9711AFA-36E4-4D90-89B7-D306382CE4F3}.Release|x86.Build.0 = Release|Any CPU
{1051DFD2-A38C-4368-8FF7-CC7046F90161}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1051DFD2-A38C-4368-8FF7-CC7046F90161}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1051DFD2-A38C-4368-8FF7-CC7046F90161}.Debug|x64.ActiveCfg = Debug|Any CPU
Expand Down Expand Up @@ -852,6 +880,8 @@ Global
{6435C875-F526-45E1-97E1-B5BF8C4F9A55} = {3079E458-D0E6-4F99-8CAB-80011D35C7DA}
{6B8DAAD7-6CC3-4138-B719-02AE81BEC535} = {4B000021-5278-4F2A-B734-DE49F55D4024}
{0C11386C-E3CC-45EB-B932-9297A8DD6649} = {4B000021-5278-4F2A-B734-DE49F55D4024}
{7DA30CA2-F08E-43FC-9C35-CF5D0487397E} = {4B000021-5278-4F2A-B734-DE49F55D4024}
{F9711AFA-36E4-4D90-89B7-D306382CE4F3} = {3079E458-D0E6-4F99-8CAB-80011D35C7DA}
{1051DFD2-A38C-4368-8FF7-CC7046F90161} = {4B000021-5278-4F2A-B734-DE49F55D4024}
{904B608D-0D3A-4019-A70A-AAFE0DBC92C4} = {3079E458-D0E6-4F99-8CAB-80011D35C7DA}
{C4DEF44E-2FAA-4887-9C7F-36D54A4E975D} = {3079E458-D0E6-4F99-8CAB-80011D35C7DA}
Expand Down
247 changes: 123 additions & 124 deletions src/Microsoft.Data.Analysis/DataFrame.IO.cs → ...Microsoft.Data.Analysis.IO/DataFrameIO.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,110 +6,16 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using Microsoft.VisualBasic.FileIO;
using System.Globalization;

namespace Microsoft.Data.Analysis
namespace Microsoft.Data.Analysis.IO
{
public partial class DataFrame
public static class DataFrameIO
{
private const int DefaultStreamReaderBufferSize = 1024;

private static Type GuessKind(int col, List<string[]> read)
{
Type res = typeof(string);
int nbline = 0;
foreach (var line in read)
{
if (col >= line.Length)
throw new FormatException(string.Format(Strings.LessColumnsThatExpected, nbline + 1));

string val = line[col];

if (string.Equals(val, "null", StringComparison.OrdinalIgnoreCase))
{
continue;
}

bool boolParse = bool.TryParse(val, out bool boolResult);
if (boolParse)
{
res = DetermineType(nbline == 0, typeof(bool), res);
++nbline;
continue;
}
else
{
if (string.IsNullOrEmpty(val))
{
res = DetermineType(nbline == 0, typeof(bool), res);
continue;
}
}
bool floatParse = float.TryParse(val, out float floatResult);
if (floatParse)
{
res = DetermineType(nbline == 0, typeof(float), res);
++nbline;
continue;
}
else
{
if (string.IsNullOrEmpty(val))
{
res = DetermineType(nbline == 0, typeof(float), res);
continue;
}
}
res = DetermineType(nbline == 0, typeof(string), res);
++nbline;
}
return res;
}

private static Type DetermineType(bool first, Type suggested, Type previous)
{
if (first)
return suggested;
else
return MaxKind(suggested, previous);
}

private static Type MaxKind(Type a, Type b)
{
if (a == typeof(string) || b == typeof(string))
return typeof(string);
if (a == typeof(float) || b == typeof(float))
return typeof(float);
if (a == typeof(bool) || b == typeof(bool))
return typeof(bool);
return typeof(string);
}

/// <summary>
/// Reads a text file as a DataFrame.
/// Follows pandas API.
/// </summary>
/// <param name="filename">filename</param>
/// <param name="separator">column separator</param>
/// <param name="header">has a header or not</param>
/// <param name="columnNames">column names (can be empty)</param>
/// <param name="dataTypes">column types (can be empty)</param>
/// <param name="numRows">number of rows to read</param>
/// <param name="guessRows">number of rows used to guess types</param>
/// <param name="addIndexColumn">add one column with the row index</param>
/// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>
/// <returns>DataFrame</returns>
public static DataFrame LoadCsv(string filename,
char separator = ',', bool header = true,
string[] columnNames = null, Type[] dataTypes = null,
int numRows = -1, int guessRows = 10,
bool addIndexColumn = false, Encoding encoding = null)
private static void SetTextFieldParserProperties(TextFieldParser parser)
{
using (Stream fileStream = new FileStream(filename, FileMode.Open))
{
return LoadCsv(fileStream,
separator: separator, header: header, columnNames: columnNames, dataTypes: dataTypes, numberOfRowsToRead: numRows,
guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding);
}
parser.HasFieldsEnclosedInQuotes = true;
}

private static string GetColumnName(string[] columnNames, int columnIndex)
Expand Down Expand Up @@ -183,29 +89,125 @@ private static DataFrameColumn CreateColumn(Type kind, string[] columnNames, int
return ret;
}

private static Type MaxKind(Type a, Type b)
{
if (a == typeof(string) || b == typeof(string))
return typeof(string);
if (a == typeof(float) || b == typeof(float))
return typeof(float);
if (a == typeof(bool) || b == typeof(bool))
return typeof(bool);
return typeof(string);
}

private static Type DetermineType(bool first, Type suggested, Type previous)
{
if (first)
return suggested;
else
return MaxKind(suggested, previous);
}

private static Type GuessKind(int col, List<string[]> read)
{
Type res = typeof(string);
int nbline = 0;
foreach (var line in read)
{
if (col >= line.Length)
throw new FormatException(string.Format(Strings.LessColumnsThatExpected, nbline + 1));

string val = line[col];

if (string.Equals(val, "null", StringComparison.OrdinalIgnoreCase))
{
continue;
}

bool boolParse = bool.TryParse(val, out bool boolResult);
if (boolParse)
{
res = DetermineType(nbline == 0, typeof(bool), res);
++nbline;
continue;
}
else
{
if (string.IsNullOrEmpty(val))
{
res = DetermineType(nbline == 0, typeof(bool), res);
continue;
}
}
bool floatParse = float.TryParse(val, out float floatResult);
if (floatParse)
{
res = DetermineType(nbline == 0, typeof(float), res);
++nbline;
continue;
}
else
{
if (string.IsNullOrEmpty(val))
{
res = DetermineType(nbline == 0, typeof(float), res);
continue;
}
}
res = DetermineType(nbline == 0, typeof(string), res);
++nbline;
}
return res;
}

/// <summary>
/// Reads a seekable stream of CSV data into a DataFrame.
/// Reads a text file as a DataFrame.
/// Follows pandas API.
/// </summary>
/// <param name="csvStream">stream of CSV data to be read in</param>
/// <param name="filename">filename</param>
/// <param name="separator">column separator</param>
/// <param name="header">has a header or not</param>
/// <param name="columnNames">column names (can be empty)</param>
/// <param name="dataTypes">column types (can be empty)</param>
/// <param name="numberOfRowsToRead">number of rows to read not including the header(if present)</param>
/// <param name="numRows">number of rows to read</param>
/// <param name="guessRows">number of rows used to guess types</param>
/// <param name="addIndexColumn">add one column with the row index</param>
/// <param name="encoding">The character encoding. Defaults to UTF8 if not specified</param>
/// <returns><see cref="DataFrame"/></returns>
public static DataFrame LoadCsv(Stream csvStream,
/// <param name="cultureInfo">culture info for processing floats/doubles</param>
/// <returns>DataFrame</returns>
public static DataFrame LoadCsv(string filename,
char separator = ',', bool header = true,
string[] columnNames = null, Type[] dataTypes = null,
int numRows = -1, int guessRows = 10,
bool addIndexColumn = false, Encoding encoding = null,
CultureInfo cultureInfo = null)

@chriss2401 chriss2401 Jun 21, 2020

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Related to #2926 , here is the CultureInfo exposed to the public API.

{
using (Stream fileStream = new FileStream(filename, FileMode.Open))
{
return LoadCsv(() => fileStream, SetTextFieldParserProperties,
separator: separator, header: header, columnNames: columnNames, dataTypes: dataTypes, numberOfRowsToRead: numRows,
guessRows: guessRows, addIndexColumn: addIndexColumn, encoding: encoding, cultureInfo: cultureInfo);
;
}
}
public static DataFrame LoadCsv(Func<Stream> csvStream,
Action<TextFieldParser> SetTextFieldParserProperties,
char separator = ',', bool header = true,
string[] columnNames = null, Type[] dataTypes = null,
long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false,
Encoding encoding = null)
Encoding encoding = null, CultureInfo cultureInfo = null)
{
if (!csvStream.CanSeek)
// if we have a comma separator and the cultureInfo has not been specified by the user,
// we set it to Invariant Culture (since logically floats/doubles wouldn't be represented with commas in this case)
if(separator.Equals(',') && cultureInfo is null)

@chriss2401 chriss2401 Jun 21, 2020 •
edited
Loading

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the case where the CultureInfo object is null and the file has a comma based separator, we change the CultureInfo to Invariant in order to handle Double/Single values.

@pgovind pgovind Jul 28, 2020

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, for now I think this is ok since it's being more restrictive. I'm wondering if we can lose the separator.Equals(',') part of the if condition safely, but we can do that later too.

{
throw new ArgumentException(Strings.NonSeekableStream, nameof(csvStream));
cultureInfo = CultureInfo.InvariantCulture;
}

Stream stream = csvStream();
if (!stream.CanSeek)
{
throw new ArgumentException(Strings.NonSeekableStream, nameof(stream));
}

if (dataTypes == null && guessRows <= 0)
Expand All @@ -223,19 +225,18 @@ public static DataFrame LoadCsv(Stream csvStream,
}

List<DataFrameColumn> columns;
long streamStart = csvStream.Position;
// First pass: schema and number of rows.
using (var streamReader = new StreamReader(csvStream, encoding ?? Encoding.UTF8, detectEncodingFromByteOrderMarks: true, DefaultStreamReaderBufferSize, leaveOpen: true))
using (TextFieldParser parser = new TextFieldParser(stream, defaultEncoding: encoding ?? Encoding.UTF8))
{
string line = null;
line = streamReader.ReadLine();
while (line != null)
parser.SetDelimiters(separator.ToString());
SetTextFieldParserProperties(parser);
while (!parser.EndOfData)
{
if ((numberOfRowsToRead == -1) || rowline < numberOfRowsToRead)
{
if (linesForGuessType.Count < guessRows || (header && rowline == 0))
{
var spl = line.Split(separator);
var spl = parser.ReadFields();
if (header && rowline == 0)
{
if (columnNames == null)
Expand All @@ -255,7 +256,6 @@ public static DataFrame LoadCsv(Stream csvStream,
{
break;
}
line = streamReader.ReadLine();
}

if (rowline == 0)
Expand All @@ -270,28 +270,27 @@ public static DataFrame LoadCsv(Stream csvStream,
Type kind = dataTypes == null ? GuessKind(i, linesForGuessType) : dataTypes[i];
columns.Add(CreateColumn(kind, columnNames, i));
}
}

using (TextFieldParser parser = new TextFieldParser(csvStream(), defaultEncoding: encoding ?? Encoding.UTF8))
{
parser.SetDelimiters(separator.ToString());
SetTextFieldParserProperties(parser);
DataFrame ret = new DataFrame(columns);
line = null;
streamReader.DiscardBufferedData();
streamReader.BaseStream.Seek(streamStart, SeekOrigin.Begin);

// Fills values.
line = streamReader.ReadLine();
rowline = 0;
while (line != null && (numberOfRowsToRead == -1 || rowline < numberOfRowsToRead))
while (!parser.EndOfData && (numberOfRowsToRead == -1 || rowline < numberOfRowsToRead))
{
var spl = line.Split(separator);
var spl = parser.ReadFields();
if (header && rowline == 0)
{
// Skips.
}
else
{
ret.Append(spl, inPlace: true);
ret.Append(spl, inPlace: true, cultureInfo);
}
++rowline;
line = streamReader.ReadLine();
}

if (addIndexColumn)
Expand Down
Loading