diff --git a/corefxlab.sln b/corefxlab.sln index 1a643924b5e..f28bb791182 100644 --- a/corefxlab.sln +++ b/corefxlab.sln @@ -119,6 +119,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.CaseFolding", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Data.Analysis.Interactive", "Microsoft.Data.Analysis.Interactive\Microsoft.Data.Analysis.Interactive.csproj", "{0C11386C-E3CC-45EB-B932-9297A8DD6649}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.Analysis.IO", "src\Microsoft.Data.Analysis.IO\Microsoft.Data.Analysis.IO.csproj", "{BA66674D-CD08-418F-B2F2-04F942108BFB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Data.Analysis.IO.Tests", "tests\Microsoft.Data.Analysis.IO.Tests\Microsoft.Data.Analysis.IO.Tests\Microsoft.Data.Analysis.IO.Tests.csproj", "{872F67A5-91EA-4A82-8626-C0D2DCD29F5F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -753,6 +757,30 @@ Global {0C11386C-E3CC-45EB-B932-9297A8DD6649}.Release|x64.Build.0 = Release|Any CPU {0C11386C-E3CC-45EB-B932-9297A8DD6649}.Release|x86.ActiveCfg = Release|Any CPU {0C11386C-E3CC-45EB-B932-9297A8DD6649}.Release|x86.Build.0 = Release|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Debug|x64.ActiveCfg = Debug|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Debug|x64.Build.0 = Debug|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Debug|x86.ActiveCfg = Debug|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Debug|x86.Build.0 = Debug|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Release|Any CPU.Build.0 = Release|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Release|x64.ActiveCfg = Release|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Release|x64.Build.0 = Release|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Release|x86.ActiveCfg = Release|Any CPU + {BA66674D-CD08-418F-B2F2-04F942108BFB}.Release|x86.Build.0 = Release|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Debug|x64.ActiveCfg = Debug|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Debug|x64.Build.0 = Debug|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Debug|x86.ActiveCfg = Debug|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Debug|x86.Build.0 = Debug|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Release|Any CPU.Build.0 = Release|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Release|x64.ActiveCfg = Release|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Release|x64.Build.0 = Release|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Release|x86.ActiveCfg = Release|Any CPU + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -810,6 +838,8 @@ Global {6435C875-F526-45E1-97E1-B5BF8C4F9A55} = {3079E458-D0E6-4F99-8CAB-80011D35C7DA} {6B8DAAD7-6CC3-4138-B719-02AE81BEC535} = {4B000021-5278-4F2A-B734-DE49F55D4024} {0C11386C-E3CC-45EB-B932-9297A8DD6649} = {4B000021-5278-4F2A-B734-DE49F55D4024} + {BA66674D-CD08-418F-B2F2-04F942108BFB} = {4B000021-5278-4F2A-B734-DE49F55D4024} + {872F67A5-91EA-4A82-8626-C0D2DCD29F5F} = {3079E458-D0E6-4F99-8CAB-80011D35C7DA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9DD4022C-A010-4A9B-BCC5-171566D4CB17} diff --git a/src/Microsoft.Data.Analysis.IO/DataFrameIO.cs b/src/Microsoft.Data.Analysis.IO/DataFrameIO.cs new file mode 100644 index 00000000000..650dc813c31 --- /dev/null +++ b/src/Microsoft.Data.Analysis.IO/DataFrameIO.cs @@ -0,0 +1,263 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Microsoft.VisualBasic.FileIO; + +namespace Microsoft.Data.Analysis.IO +{ + public static class DataFrameIO + { + private static string GetColumnName(string[] columnNames, int columnIndex) + { + return columnNames == null ? "Column" + columnIndex.ToString() : columnNames[columnIndex]; + } + + private static DataFrameColumn CreateColumn(Type kind, string[] columnNames, int columnIndex) + { + DataFrameColumn ret; + if (kind == typeof(bool)) + { + ret = new BooleanDataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(int)) + { + ret = new Int32DataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(float)) + { + ret = new SingleDataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(string)) + { + ret = new StringDataFrameColumn(GetColumnName(columnNames, columnIndex), 0); + } + else if (kind == typeof(long)) + { + ret = new Int64DataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(decimal)) + { + ret = new DecimalDataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(byte)) + { + ret = new ByteDataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(char)) + { + ret = new CharDataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(double)) + { + ret = new DoubleDataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(sbyte)) + { + ret = new SByteDataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(short)) + { + ret = new Int16DataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(uint)) + { + ret = new UInt32DataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(ulong)) + { + ret = new UInt64DataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else if (kind == typeof(ushort)) + { + ret = new UInt16DataFrameColumn(GetColumnName(columnNames, columnIndex)); + } + else + { + throw new NotSupportedException(nameof(kind)); + } + return ret; + } + + private static Type MaxKind(Type a, Type b) + { + if (a == typeof(string) || b == typeof(string)) + return typeof(string); + if (a == typeof(float) || b == typeof(float)) + return typeof(float); + if (a == typeof(bool) || b == typeof(bool)) + return typeof(bool); + return typeof(string); + } + + private static Type DetermineType(bool first, Type suggested, Type previous) + { + return first ? suggested : MaxKind(suggested, previous); + } + + private static Type GuessKind(int col, List read) + { + Type res = typeof(string); + int nbline = 0; + foreach (var line in read) + { + if (col >= line.Length) + throw new FormatException(string.Format(Strings.LessColumnsThatExpected, nbline + 1)); + + string val = line[col]; + + if (string.Equals(val, "null", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + bool boolParse = bool.TryParse(val, out bool boolResult); + if (boolParse) + { + res = DetermineType(nbline == 0, typeof(bool), res); + ++nbline; + continue; + } + else + { + if (string.IsNullOrEmpty(val)) + { + res = DetermineType(nbline == 0, typeof(bool), res); + continue; + } + } + bool floatParse = float.TryParse(val, out float floatResult); + if (floatParse) + { + res = DetermineType(nbline == 0, typeof(float), res); + ++nbline; + continue; + } + else + { + if (string.IsNullOrEmpty(val)) + { + res = DetermineType(nbline == 0, typeof(float), res); + continue; + } + } + res = DetermineType(nbline == 0, typeof(string), res); + ++nbline; + } + return res; + } + + public static DataFrame LoadCsv(Func csvStream, + Action SetTextFieldParserProperties, + char separator = ',', bool header = true, + string[] columnNames = null, Type[] dataTypes = null, + long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false, + Encoding encoding = null) + { + Stream stream = csvStream(); + if (!stream.CanSeek) + { + throw new ArgumentException(Strings.NonSeekableStream, nameof(stream)); + } + + if (dataTypes == null && guessRows <= 0) + { + throw new ArgumentException(string.Format(Strings.ExpectedEitherGuessRowsOrDataTypes, nameof(guessRows), nameof(dataTypes))); + } + + var linesForGuessType = new List(); + long rowline = 0; + int numberOfColumns = dataTypes?.Length ?? 0; + + if (header == true && numberOfRowsToRead != -1) + { + numberOfRowsToRead++; + } + + List columns; + // First pass: schema and number of rows. + using (TextFieldParser parser = new TextFieldParser(stream, defaultEncoding: encoding ?? Encoding.UTF8)) + { + parser.SetDelimiters(separator.ToString()); + SetTextFieldParserProperties(parser); + while (!parser.EndOfData) + { + if ((numberOfRowsToRead == -1) || rowline < numberOfRowsToRead) + { + if (linesForGuessType.Count < guessRows || (header && rowline == 0)) + { + var spl = parser.ReadFields(); + if (header && rowline == 0) + { + if (columnNames == null) + { + columnNames = spl; + } + } + else + { + linesForGuessType.Add(spl); + numberOfColumns = Math.Max(numberOfColumns, spl.Length); + } + } + } + ++rowline; + if (rowline == guessRows || guessRows == 0) + { + break; + } + } + + if (rowline == 0) + { + throw new FormatException(Strings.EmptyFile); + } + + columns = new List(numberOfColumns); + // Guesses types or looks up dataTypes and adds columns. + for (int i = 0; i < numberOfColumns; ++i) + { + Type kind = dataTypes == null ? GuessKind(i, linesForGuessType) : dataTypes[i]; + columns.Add(CreateColumn(kind, columnNames, i)); + } + } + + using (TextFieldParser parser = new TextFieldParser(csvStream(), defaultEncoding: encoding ?? Encoding.UTF8)) + { + parser.SetDelimiters(separator.ToString()); + SetTextFieldParserProperties(parser); + DataFrame ret = new DataFrame(columns); + // Fills values. + rowline = 0; + while (!parser.EndOfData && (numberOfRowsToRead == -1 || rowline < numberOfRowsToRead)) + { + var spl = parser.ReadFields(); + if (header && rowline == 0) + { + // Skips. + } + else + { + ret.Append(spl, inPlace: true); + } + ++rowline; + } + + if (addIndexColumn) + { + PrimitiveDataFrameColumn indexColumn = new PrimitiveDataFrameColumn("IndexColumn", columns[0].Length); + for (int i = 0; i < columns[0].Length; i++) + { + indexColumn[i] = i; + } + columns.Insert(0, indexColumn); + } + return ret; + } + } + } +} diff --git a/src/Microsoft.Data.Analysis.IO/Microsoft.Data.Analysis.IO.csproj b/src/Microsoft.Data.Analysis.IO/Microsoft.Data.Analysis.IO.csproj new file mode 100644 index 00000000000..e17f7f37dc9 --- /dev/null +++ b/src/Microsoft.Data.Analysis.IO/Microsoft.Data.Analysis.IO.csproj @@ -0,0 +1,37 @@ + + + + netcoreapp3.0;net461 + false + 0.1.0 + false + This package contains IO libraries that work with the Microsoft.Data.Analysis package. + Initial preview of IO routines that import data into a Microsoft.Data.Analysis.DataFrame. + Data Science DataFrame Preparation DataView Analytics Exploration + true + + + + + + + + + + + + + True + True + Strings.resx + + + + + + PublicResXFileCodeGenerator + Strings.Designer.cs + + + + diff --git a/src/Microsoft.Data.Analysis.IO/Strings.Designer.cs b/src/Microsoft.Data.Analysis.IO/Strings.Designer.cs new file mode 100644 index 00000000000..478cde7439a --- /dev/null +++ b/src/Microsoft.Data.Analysis.IO/Strings.Designer.cs @@ -0,0 +1,99 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.Data.Analysis.IO { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public class Strings { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Strings() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Microsoft.Data.Analysis.IO.Strings", typeof(Strings).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to Empty file. + /// + public static string EmptyFile { + get { + return ResourceManager.GetString("EmptyFile", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Expected either {0} or {1} to be provided. + /// + public static string ExpectedEitherGuessRowsOrDataTypes { + get { + return ResourceManager.GetString("ExpectedEitherGuessRowsOrDataTypes", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Line {0} has less columns than expected. + /// + public static string LessColumnsThatExpected { + get { + return ResourceManager.GetString("LessColumnsThatExpected", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Expected a seekable stream. + /// + public static string NonSeekableStream { + get { + return ResourceManager.GetString("NonSeekableStream", resourceCulture); + } + } + } +} diff --git a/src/Microsoft.Data.Analysis.IO/Strings.resx b/src/Microsoft.Data.Analysis.IO/Strings.resx new file mode 100644 index 00000000000..1459c977f5c --- /dev/null +++ b/src/Microsoft.Data.Analysis.IO/Strings.resx @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Empty file + + + Expected either {0} or {1} to be provided + + + Line {0} has less columns than expected + + + Expected a seekable stream + + \ No newline at end of file diff --git a/tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests/DataFrame.IOTests.cs b/tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests/DataFrame.IOTests.cs new file mode 100644 index 00000000000..322bc9c614d --- /dev/null +++ b/tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests/DataFrame.IOTests.cs @@ -0,0 +1,626 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text; +using Microsoft.VisualBasic.FileIO; +using Xunit; + +namespace Microsoft.Data.Analysis.IO.Tests +{ + public partial class DataFrameIOTests + { + private void SetTextFieldParserProperties(TextFieldParser parser) + { + parser.HasFieldsEnclosedInQuotes = true; + } + + internal static void VerifyColumnTypes(DataFrame df, bool testArrowStringColumn = false) + { + foreach (DataFrameColumn column in df.Columns) + { + Type dataType = column.DataType; + if (dataType == typeof(bool)) + { + Assert.IsType(column); + } + else if (dataType == typeof(decimal)) + { + Assert.IsType(column); + } + else if (dataType == typeof(byte)) + { + Assert.IsType(column); + } + else if (dataType == typeof(char)) + { + Assert.IsType(column); + } + else if (dataType == typeof(double)) + { + Assert.IsType(column); + } + else if (dataType == typeof(float)) + { + Assert.IsType(column); + } + else if (dataType == typeof(int)) + { + Assert.IsType(column); + } + else if (dataType == typeof(long)) + { + Assert.IsType(column); + } + else if (dataType == typeof(sbyte)) + { + Assert.IsType(column); + } + else if (dataType == typeof(short)) + { + Assert.IsType(column); + } + else if (dataType == typeof(uint)) + { + Assert.IsType(column); + + } + else if (dataType == typeof(ulong)) + { + Assert.IsType(column); + } + else if (dataType == typeof(ushort)) + { + Assert.IsType(column); + } + else if (dataType == typeof(string)) + { + if (!testArrowStringColumn) + { + Assert.IsType(column); + } + else + { + Assert.IsType(column); + } + } + else + { + throw new NotImplementedException("Unit test has to be updated"); + } + } + } + + [Fact] + public void TestReadCsvWithHeader() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount +CMT,1,1,1271,3.8,CRD,17.5 +CMT,1,1,474,1.5,CRD,8 +CMT,1,1,637,1.4,CRD,8.5 +CMT,1,1,181,0.6,CSH,4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties); + Assert.Equal(4, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + Assert.Equal("CMT", df.Columns["vendor_id"][3]); + VerifyColumnTypes(df); + + DataFrame reducedRows = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, numberOfRowsToRead: 3); + Assert.Equal(3, reducedRows.Rows.Count); + Assert.Equal(7, reducedRows.Columns.Count); + Assert.Equal("CMT", reducedRows.Columns["vendor_id"][2]); + VerifyColumnTypes(df); + } + + [Fact] + public void TestReadCsvNoHeader() + { + string data = @"CMT,1,1,1271,3.8,CRD,17.5 +CMT,1,1,474,1.5,CRD,8 +CMT,1,1,637,1.4,CRD,8.5 +CMT,1,1,181,0.6,CSH,4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, header: false); + Assert.Equal(4, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + Assert.Equal("CMT", df.Columns["Column0"][3]); + VerifyColumnTypes(df); + + DataFrame reducedRows = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, header: false, numberOfRowsToRead: 3); + Assert.Equal(3, reducedRows.Rows.Count); + Assert.Equal(7, reducedRows.Columns.Count); + Assert.Equal("CMT", reducedRows.Columns["Column0"][2]); + VerifyColumnTypes(df); + } + + void VerifyDataFrameWithNamedColumnsAndDataTypes(DataFrame df, bool verifyColumnDataType, bool verifyNames) + { + Assert.Equal(4, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + + if (verifyColumnDataType) + { + Assert.True(typeof(string) == df.Columns[0].DataType); + Assert.True(typeof(short) == df.Columns[1].DataType); + Assert.True(typeof(int) == df.Columns[2].DataType); + Assert.True(typeof(long) == df.Columns[3].DataType); + Assert.True(typeof(float) == df.Columns[4].DataType); + Assert.True(typeof(string) == df.Columns[5].DataType); + Assert.True(typeof(double) == df.Columns[6].DataType); + } + + if (verifyNames) + { + Assert.Equal("vendor_id", df.Columns[0].Name); + Assert.Equal("rate_code", df.Columns[1].Name); + Assert.Equal("passenger_count", df.Columns[2].Name); + Assert.Equal("trip_time_in_secs", df.Columns[3].Name); + Assert.Equal("trip_distance", df.Columns[4].Name); + Assert.Equal("payment_type", df.Columns[5].Name); + Assert.Equal("fare_amount", df.Columns[6].Name); + } + + VerifyColumnTypes(df); + + foreach (var column in df.Columns) + { + Assert.Equal(0, column.NullCount); + } + } + + [Theory] + [InlineData(true, 0)] + [InlineData(false, 0)] + [InlineData(true, 10)] + [InlineData(false, 10)] + public void TestReadCsvWithTypesAndGuessRows(bool header, int guessRows) + { + /* Tests this matrix + * + header GuessRows DataTypes + True 0 NotNull + False 0 NotNull + True 10 NotNull + False 10 NotNull + True 0 Null -----> Throws an exception + False 0 Null -----> Throws an exception + True 10 Null + False 10 Null + * + */ + string headerLine = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount +"; + string dataLines = +@"CMT,1,1,1271,3.8,CRD,17.5 +CMT,1,1,474,1.5,CRD,8 +CMT,1,1,637,1.4,CRD,8.5 +CMT,1,1,181,0.6,CSH,4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + + string data = header ? headerLine + dataLines : dataLines; + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), + SetTextFieldParserProperties, + header: header, + guessRows: guessRows, + dataTypes: new Type[] { typeof(string), typeof(short), typeof(int), typeof(long), typeof(float), typeof(string), typeof(double) } + ); + VerifyDataFrameWithNamedColumnsAndDataTypes(df, verifyColumnDataType: true, verifyNames: header); + + if (guessRows == 10) + { + df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, header: header, + guessRows: guessRows); + VerifyDataFrameWithNamedColumnsAndDataTypes(df, verifyColumnDataType: false, verifyNames: header); + } + else + { + Assert.ThrowsAny(() => DataFrameIO.LoadCsv(() => GetStream(data), + SetTextFieldParserProperties, + header: header, + guessRows: guessRows + )); + } + } + + [Fact] + public void TestReadCsvWithTypes() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount +CMT,1,1,1271,3.8,CRD,17.5 +CMT,1,1,474,1.5,CRD,8 +CMT,1,1,637,1.4,CRD,8.5 +,,,,,, +CMT,1,1,181,0.6,CSH,4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, dataTypes: new Type[] { typeof(string), typeof(short), typeof(int), typeof(long), typeof(float), typeof(string), typeof(double) }); + Assert.Equal(5, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + + Assert.True(typeof(string) == df.Columns[0].DataType); + Assert.True(typeof(short) == df.Columns[1].DataType); + Assert.True(typeof(int) == df.Columns[2].DataType); + Assert.True(typeof(long) == df.Columns[3].DataType); + Assert.True(typeof(float) == df.Columns[4].DataType); + Assert.True(typeof(string) == df.Columns[5].DataType); + Assert.True(typeof(double) == df.Columns[6].DataType); + + Assert.Equal("vendor_id", df.Columns[0].Name); + Assert.Equal("rate_code", df.Columns[1].Name); + Assert.Equal("passenger_count", df.Columns[2].Name); + Assert.Equal("trip_time_in_secs", df.Columns[3].Name); + Assert.Equal("trip_distance", df.Columns[4].Name); + Assert.Equal("payment_type", df.Columns[5].Name); + Assert.Equal("fare_amount", df.Columns[6].Name); + VerifyColumnTypes(df); + + foreach (var column in df.Columns) + { + if (column.DataType != typeof(string)) + { + Assert.Equal(1, column.NullCount); + } + else + { + Assert.Equal(0, column.NullCount); + } + } + var nullRow = df.Rows[3]; + Assert.Equal("", nullRow[0]); + Assert.Null(nullRow[1]); + Assert.Null(nullRow[2]); + Assert.Null(nullRow[3]); + Assert.Null(nullRow[4]); + Assert.Equal("", nullRow[5]); + Assert.Null(nullRow[6]); + } + + [Fact] + public void TestReadCsvWithPipeSeparator() + { + string data = @"vendor_id|rate_code|passenger_count|trip_time_in_secs|trip_distance|payment_type|fare_amount +CMT|1|1|1271|3.8|CRD|17.5 +CMT|1|1|474|1.5|CRD|8 +CMT|1|1|637|1.4|CRD|8.5 +|||||| +CMT|1|1|181|0.6|CSH|4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, separator: '|'); + + Assert.Equal(5, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + Assert.Equal("CMT", df.Columns["vendor_id"][4]); + VerifyColumnTypes(df); + + DataFrame reducedRows = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, separator: '|', numberOfRowsToRead: 3); + Assert.Equal(3, reducedRows.Rows.Count); + Assert.Equal(7, reducedRows.Columns.Count); + Assert.Equal("CMT", reducedRows.Columns["vendor_id"][2]); + VerifyColumnTypes(df); + + var nullRow = df.Rows[3]; + Assert.Equal("", nullRow[0]); + Assert.Null(nullRow[1]); + Assert.Null(nullRow[2]); + Assert.Null(nullRow[3]); + Assert.Null(nullRow[4]); + Assert.Equal("", nullRow[5]); + Assert.Null(nullRow[6]); + } + + [Fact] + public void TestReadCsvWithSemicolonSeparator() + { + string data = @"vendor_id;rate_code;passenger_count;trip_time_in_secs;trip_distance;payment_type;fare_amount +CMT;1;1;1271;3.8;CRD;17.5 +CMT;1;1;474;1.5;CRD;8 +CMT;1;1;637;1.4;CRD;8.5 +;;;;;; +CMT;1;1;181;0.6;CSH;4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, separator: ';'); + + Assert.Equal(5, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + Assert.Equal("CMT", df.Columns["vendor_id"][4]); + VerifyColumnTypes(df); + + DataFrame reducedRows = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, separator: ';', numberOfRowsToRead: 3); + Assert.Equal(3, reducedRows.Rows.Count); + Assert.Equal(7, reducedRows.Columns.Count); + Assert.Equal("CMT", reducedRows.Columns["vendor_id"][2]); + VerifyColumnTypes(df); + + var nullRow = df.Rows[3]; + Assert.Equal("", nullRow[0]); + Assert.Null(nullRow[1]); + Assert.Null(nullRow[2]); + Assert.Null(nullRow[3]); + Assert.Null(nullRow[4]); + Assert.Equal("", nullRow[5]); + Assert.Null(nullRow[6]); + } + + [Fact] + public void TestReadCsvWithExtraColumnInHeader() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount,extra +CMT,1,1,1271,3.8,CRD,17.5 +CMT,1,1,474,1.5,CRD,8 +CMT,1,1,637,1.4,CRD,8.5 +CMT,1,1,181,0.6,CSH,4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties); + + Assert.Equal(4, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + Assert.Equal("CMT", df.Columns["vendor_id"][3]); + VerifyColumnTypes(df); + + DataFrame reducedRows = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, numberOfRowsToRead: 3); + Assert.Equal(3, reducedRows.Rows.Count); + Assert.Equal(7, reducedRows.Columns.Count); + Assert.Equal("CMT", reducedRows.Columns["vendor_id"][2]); + VerifyColumnTypes(df); + } + + [Fact] + public void TestReadCsvWithExtraColumnInRow() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount +CMT,1,1,1271,3.8,CRD,17.5,0 +CMT,1,1,474,1.5,CRD,8,0 +CMT,1,1,637,1.4,CRD,8.5,0 +CMT,1,1,181,0.6,CSH,4.5,0"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + + Assert.Throws(() => DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties)); + } + + [Fact] + public void TestReadCsvWithLessColumnsInRow() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount +CMT,1,1,1271,3.8,CRD +CMT,1,1,474,1.5,CRD +CMT,1,1,637,1.4,CRD +CMT,1,1,181,0.6,CSH"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties); + Assert.Equal(4, df.Rows.Count); + Assert.Equal(6, df.Columns.Count); + Assert.Equal("CMT", df.Columns["vendor_id"][3]); + VerifyColumnTypes(df); + + DataFrame reducedRows = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, numberOfRowsToRead: 3); + Assert.Equal(3, reducedRows.Rows.Count); + Assert.Equal(6, reducedRows.Columns.Count); + Assert.Equal("CMT", reducedRows.Columns["vendor_id"][2]); + VerifyColumnTypes(df); + + } + + [Fact] + public void TestReadCsvWithAllNulls() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs +null,null,null,null +Null,Null,Null,Null +null,null,null,null +Null,Null,Null,Null +null,null,null,null +null,null,null,null"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties); + Assert.Equal(6, df.Rows.Count); + Assert.Equal(4, df.Columns.Count); + + Assert.True(typeof(string) == df.Columns[0].DataType); + Assert.True(typeof(string) == df.Columns[1].DataType); + Assert.True(typeof(string) == df.Columns[2].DataType); + Assert.True(typeof(string) == df.Columns[3].DataType); + + Assert.Equal("vendor_id", df.Columns[0].Name); + Assert.Equal("rate_code", df.Columns[1].Name); + Assert.Equal("passenger_count", df.Columns[2].Name); + Assert.Equal("trip_time_in_secs", df.Columns[3].Name); + VerifyColumnTypes(df); + + foreach (var column in df.Columns) + { + Assert.Equal(6, column.NullCount); + foreach (var value in column) + { + Assert.Null(value); + } + } + } + + [Fact] + public void TestReadCsvWithNullsAndDataTypes() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs +null,1,1,1271 +CMT,Null,1,474 +CMT,1,null,637 +Null,,, +,,, +CMT,1,1,null"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, dataTypes: new Type[] { typeof(string), typeof(short), typeof(int), typeof(long) }); + Assert.Equal(6, df.Rows.Count); + Assert.Equal(4, df.Columns.Count); + + Assert.True(typeof(string) == df.Columns[0].DataType); + Assert.True(typeof(short) == df.Columns[1].DataType); + Assert.True(typeof(int) == df.Columns[2].DataType); + Assert.True(typeof(long) == df.Columns[3].DataType); + + Assert.Equal("vendor_id", df.Columns[0].Name); + Assert.Equal("rate_code", df.Columns[1].Name); + Assert.Equal("passenger_count", df.Columns[2].Name); + Assert.Equal("trip_time_in_secs", df.Columns[3].Name); + VerifyColumnTypes(df); + + foreach (var column in df.Columns) + { + if (column.DataType != typeof(string)) + { + Assert.Equal(3, column.NullCount); + } + else + { + Assert.Equal(2, column.NullCount); + } + } + var nullRow = df.Rows[3]; + Assert.Null(nullRow[0]); + Assert.Null(nullRow[1]); + Assert.Null(nullRow[2]); + Assert.Null(nullRow[3]); + + nullRow = df.Rows[4]; + Assert.Equal("", nullRow[0]); + Assert.Null(nullRow[1]); + Assert.Null(nullRow[2]); + Assert.Null(nullRow[3]); + + Assert.Null(df[0, 0]); + Assert.Null(df[1, 1]); + Assert.Null(df[2, 2]); + Assert.Null(df[5, 3]); + } + + [Fact] + public void TestReadCsvWithNulls() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs +null,1,1,1271 +CMT,Null,1,474 +CMT,1,null,637 +Null,,, +,,, +CMT,1,1,null"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties); + Assert.Equal(6, df.Rows.Count); + Assert.Equal(4, df.Columns.Count); + + Assert.True(typeof(string) == df.Columns[0].DataType); + Assert.True(typeof(float) == df.Columns[1].DataType); + Assert.True(typeof(float) == df.Columns[2].DataType); + Assert.True(typeof(float) == df.Columns[3].DataType); + + Assert.Equal("vendor_id", df.Columns[0].Name); + Assert.Equal("rate_code", df.Columns[1].Name); + Assert.Equal("passenger_count", df.Columns[2].Name); + Assert.Equal("trip_time_in_secs", df.Columns[3].Name); + VerifyColumnTypes(df); + + foreach (var column in df.Columns) + { + if (column.DataType != typeof(string)) + { + Assert.Equal(3, column.NullCount); + } + else + { + Assert.Equal(2, column.NullCount); + } + } + var nullRow = df.Rows[3]; + Assert.Null(nullRow[0]); + Assert.Null(nullRow[1]); + Assert.Null(nullRow[2]); + Assert.Null(nullRow[3]); + + nullRow = df.Rows[4]; + Assert.Equal("", nullRow[0]); + Assert.Null(nullRow[1]); + Assert.Null(nullRow[2]); + Assert.Null(nullRow[3]); + + Assert.Null(df[0, 0]); + Assert.Null(df[1, 1]); + Assert.Null(df[2, 2]); + Assert.Null(df[5, 3]); + } + + [Fact] + public void TestReadCsvWithQuotes() + { + string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount" + Environment.NewLine + +"\"CMT, Comma\",1,1,1271,3.8,CRD,17.5" + Environment.NewLine + +@"CMT,1,1,474,1.5,CRD,8 +CMT,1,1,637,1.4,CRD,8.5 +CMT,1,1,181,0.6,CSH,4.5"; + + Stream GetStream(string streamData) + { + return new MemoryStream(Encoding.Default.GetBytes(streamData)); + } + DataFrame df = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties); + Assert.Equal(4, df.Rows.Count); + Assert.Equal(7, df.Columns.Count); + Assert.Equal("CMT, Comma", df.Columns["vendor_id"][0]); + VerifyColumnTypes(df); + + DataFrame reducedRows = DataFrameIO.LoadCsv(() => GetStream(data), SetTextFieldParserProperties, numberOfRowsToRead: 3); + Assert.Equal(3, reducedRows.Rows.Count); + Assert.Equal(7, reducedRows.Columns.Count); + Assert.Equal("CMT, Comma", df.Columns["vendor_id"][0]); + VerifyColumnTypes(df); + } + } +} diff --git a/tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests.csproj b/tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests.csproj new file mode 100644 index 00000000000..9b633fae5fa --- /dev/null +++ b/tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests/Microsoft.Data.Analysis.IO.Tests.csproj @@ -0,0 +1,17 @@ + + + + netcoreapp3.0;net461 + + false + + + + + + + + + + +