Added DecimalMarker in DoubleParser

src/Microsoft.ML.Core/Utilities/DoubleParser.cs

@@ -17,6 +17,13 @@ internal static class DoubleParser
         private const ulong TopThreeBits = 0xE000000000000000UL;
         private const char InfinitySymbol = '\u221E';
 
+        // The decimal marker that separates the integer part from the fractional part of a number
+        // written in decimal from can vary across different cultures as either '.' or ','. The
+        // default decimal marker in ML .NET is '.', however through this static char variable,
+        // we allow users to specify the decimal marker used in their datasets as ',' as well.
+        [BestFriend]
+        internal static char DecimalMarker = '.';
+
         // REVIEW: casting ulong to Double doesn't always do the right thing, for example
         // with 0x84595161401484A0UL. Hence the gymnastics several places in this code. Note that
         // long to Double does work. The work around is:
@@ -527,8 +534,6 @@ private static bool TryParseCore(ReadOnlySpan<char> span, ref int ich, ref bool
             Contracts.Assert(num == 0);
             Contracts.Assert(exp == 0);
 
-            const char decimalMarker = '.';
-
             if (ich >= span.Length)
                 return false;
 
@@ -556,7 +561,8 @@ private static bool TryParseCore(ReadOnlySpan<char> span, ref int ich, ref bool
                         return false;
                     break;
 
-                case decimalMarker:
+                case '.':
+                case ',':
                     goto LPoint;
 
                 // The common cases.
@@ -595,12 +601,12 @@ private static bool TryParseCore(ReadOnlySpan<char> span, ref int ich, ref bool
             }
             Contracts.Assert(i < span.Length);
 
-            if (span[i] != decimalMarker)
+            if (span[i] != DecimalMarker)
                 goto LAfterDigits;
 
             LPoint:
             Contracts.Assert(i < span.Length);
-            Contracts.Assert(span[i] == decimalMarker);
+            Contracts.Assert(span[i] == DecimalMarker);
 
             // Get the digits after the decimal marker, which may be '.' or ','
             for (; ; )

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs

@@ -709,11 +709,11 @@ public Bindings(TextLoader parent, Column[] cols, IMultiStreamSource headerFile,
                     ch.Assert(0 <= inputSize & inputSize < SrcLim);
                     List<ReadOnlyMemory<char>> lines = null;
                     if (headerFile != null)
-                        Cursor.GetSomeLines(headerFile, 1, parent.ReadMultilines, parent._separators, ref lines, parent._decimalMarker);
+                        Cursor.GetSomeLines(headerFile, 1, parent.ReadMultilines, parent._separators, ref lines);
                     if (needInputSize && inputSize == 0)
-                        Cursor.GetSomeLines(dataSample, 100, parent.ReadMultilines, parent._separators, ref lines, parent._decimalMarker);
+                        Cursor.GetSomeLines(dataSample, 100, parent.ReadMultilines, parent._separators, ref lines);
                     else if (headerFile == null && parent.HasHeader)
-                        Cursor.GetSomeLines(dataSample, 1, parent.ReadMultilines, parent._separators, ref lines, parent._decimalMarker);
+                        Cursor.GetSomeLines(dataSample, 1, parent.ReadMultilines, parent._separators, ref lines);
 
                     if (needInputSize && inputSize == 0)
                     {
@@ -1219,7 +1219,7 @@ internal TextLoader(IHostEnvironment env, Options options = null, IMultiStreamSo
                 }
             }
 
-            if (options.DecimalMarker == ',' && _separators.Contains<char>(','))
+            if (_separators.Contains(options.DecimalMarker))
                 throw _host.ExceptUserArg(nameof(Options.DecimalMarker), "Decimal marker and separator cannot be the same '{0}' character.", options.DecimalMarker);
             _decimalMarker = options.DecimalMarker;
             _bindings = new Bindings(this, cols, headerFile, dataSample);
@@ -1607,13 +1607,15 @@ public BoundLoader(TextLoader loader, IMultiStreamSource files)
             public DataViewRowCursor GetRowCursor(IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand = null)
             {
                 _host.CheckValueOrNull(rand);
+                DoubleParser.DecimalMarker = _loader._decimalMarker;
                 var active = Utils.BuildArray(_loader._bindings.OutputSchema.Count, columnsNeeded);
                 return Cursor.Create(_loader, _files, active);
             }
 
             public DataViewRowCursor[] GetRowCursorSet(IEnumerable<DataViewSchema.Column> columnsNeeded, int n, Random rand = null)
             {
                 _host.CheckValueOrNull(rand);
+                DoubleParser.DecimalMarker = _loader._decimalMarker;
                 var active = Utils.BuildArray(_loader._bindings.OutputSchema.Count, columnsNeeded);
                 return Cursor.CreateSet(_loader, _files, active, n);
             }

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderCursor.cs

@@ -146,7 +146,7 @@ public static DataViewRowCursor Create(TextLoader parent, IMultiStreamSource fil
                 SetupCursor(parent, active, 0, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, 1, parent._decimalMarker);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, 1);
                 var stats = new ParseStats(parent._host, 1);
                 return new Cursor(parent, stats, active, reader, srcNeeded, cthd);
             }
@@ -163,7 +163,7 @@ public static DataViewRowCursor[] CreateSet(TextLoader parent, IMultiStreamSourc
                 SetupCursor(parent, active, n, out srcNeeded, out cthd);
                 Contracts.Assert(cthd > 0);
 
-                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, cthd, parent._decimalMarker);
+                var reader = new LineReader(files, BatchSize, 100, parent.HasHeader, parent.ReadMultilines, parent._separators, parent._maxRows, cthd);
                 var stats = new ParseStats(parent._host, cthd);
                 if (cthd <= 1)
                     return new DataViewRowCursor[1] { new Cursor(parent, stats, active, reader, srcNeeded, 1) };
@@ -205,7 +205,7 @@ public override ValueGetter<DataViewRowId> GetIdGetter()
                     };
             }
 
-            public static void GetSomeLines(IMultiStreamSource source, int count, bool readMultilines, char[] separators, ref List<ReadOnlyMemory<char>> lines, char decimalMarker)
+            public static void GetSomeLines(IMultiStreamSource source, int count, bool readMultilines, char[] separators, ref List<ReadOnlyMemory<char>> lines)
             {
                 Contracts.AssertValue(source);
                 Contracts.Assert(count > 0);
@@ -215,7 +215,7 @@ public static void GetSomeLines(IMultiStreamSource source, int count, bool readM
                     count = 2;
 
                 LineBatch batch;
-                var reader = new LineReader(source, count, 1, false, readMultilines, separators, count, 1, decimalMarker);
+                var reader = new LineReader(source, count, 1, false, readMultilines, separators, count, 1);
                 try
                 {
                     batch = reader.GetBatch();
@@ -404,7 +404,6 @@ private sealed class LineReader
                 private readonly bool _hasHeader;
                 private readonly bool _readMultilines;
                 private readonly char[] _separators;
-                private readonly char _decimalMarker;
                 private readonly int _batchSize;
                 private readonly IMultiStreamSource _files;
 
@@ -414,7 +413,7 @@ private sealed class LineReader
                 private Task _thdRead;
                 private volatile bool _abort;
 
-                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, bool readMultilines, char[] separators, long limit, int cref, char decimalMarker)
+                public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool hasHeader, bool readMultilines, char[] separators, long limit, int cref)
                 {
                     // Note that files is allowed to be empty.
                     Contracts.AssertValue(files);
@@ -431,7 +430,6 @@ public LineReader(IMultiStreamSource files, int batchSize, int bufSize, bool has
                     _separators = separators;
                     _files = files;
                     _cref = cref;
-                    _decimalMarker = decimalMarker;
 
                     _queue = new BlockingQueue<LineBatch>(bufSize);
                     _thdRead = Utils.RunOnBackgroundThreadAsync(ThreadProc);

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs

@@ -633,7 +633,6 @@ public void Clear()
             }
 
             private readonly char[] _separators;
-            private readonly char _decimalMarker;
             private readonly OptionFlags _flags;
             private readonly int _inputSize;
             private readonly ColInfo[] _infos;
@@ -684,7 +683,6 @@ public Parser(TextLoader parent)
                 }
 
                 _separators = parent._separators;
-                _decimalMarker = parent._decimalMarker;
                 _flags = parent._flags;
                 _inputSize = parent._inputSize;
                 Contracts.Assert(_inputSize >= 0);