From f6ad50eadf00f47f298878fe1515f08e1bf585e0 Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Mon, 6 Jul 2020 12:55:10 -0700 Subject: [PATCH 1/9] initial checkin --- .../Database/DatabaseLoaderCursor.cs | 4 ++++ .../DataLoadSave/Database/DatabaseSource.cs | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs index 7d029c96a8..8dd8275f64 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs @@ -75,6 +75,10 @@ public DbCommand Command { _command = Connection.CreateCommand(); _command.CommandText = _source.CommandText; + if (_source.CommandTimeOut is {}) + { + _command.CommandTimeout = _source.CommandTimeOut; + } } return _command; } diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs index 34f8973c81..146338ed94 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs @@ -25,6 +25,26 @@ public DatabaseSource(DbProviderFactory providerFactory, string connectionString CommandText = commandText; } + /// Creates a new instance of the class. + /// The factory used to create the .. + /// The string used to open the connection. + /// The text command to run against the data source. + /// The text command to run against the data source. + public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText, int commandTimeOut) + { + Contracts.CheckValue(providerFactory, nameof(providerFactory)); + Contracts.CheckValue(connectionString, nameof(connectionString)); + Contracts.CheckValue(commandText, nameof(commandText)); + + ProviderFactory = providerFactory; + ConnectionString = connectionString; + CommandText = commandText; + CommandTimeOut = commandTimeOut; + } + + /// Gets the timeout for database command. + public int CommandTimeOut { get; } + /// Gets the text command to run against the data source. public string CommandText { get; } From d3c1f2532df75f767753929d95d10b7ec9b94f73 Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Mon, 6 Jul 2020 14:07:07 -0700 Subject: [PATCH 2/9] update tests --- .../Microsoft.ML.Tests/DatabaseLoaderTests.cs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index 6826eb9aba..5b72e671c0 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -27,6 +27,21 @@ public DatabaseLoaderTests(ITestOutputHelper output) [LightGBMFact] public void IrisLightGbm() + { + DatabaseSource dbs = GetIrisDatabaseSource("SELECT * FROM {0}"); + IrisLightGbmImpl(dbs); + } + + [LightGBMFact] + public void IrisLightGbmWithTimeOut() + { + DatabaseSource dbs = GetIrisDatabaseSourceWithTimeOut("WAITFOR DELAY '00:00:05'; SELECT * FROM {0}", 1); + var ex = Assert.Throws(() => IrisLightGbmImpl(dbs)); + Assert.Contains("Timeout expired", ex.InnerException.Message); + + } + + private void IrisLightGbmImpl(DatabaseSource dbs) { var mlContext = new MLContext(seed: 1); @@ -41,7 +56,7 @@ public void IrisLightGbm() var loader = mlContext.Data.CreateDatabaseLoader(loaderColumns); - var trainingData = loader.Load(GetIrisDatabaseSource("SELECT * FROM {0}")); + var trainingData = loader.Load(dbs); IEstimator pipeline = mlContext.Transforms.Conversion.MapValueToKey("Label") .Append(mlContext.Transforms.Concatenate("Features", "SepalLength", "SepalWidth", "PetalLength", "PetalWidth")) @@ -225,6 +240,22 @@ private DatabaseSource GetIrisDatabaseSource(string command) String.Format(command, TestDatasets.irisDbSQLite.trainFilename)); } + private DatabaseSource GetIrisDatabaseSourceWithTimeOut(string command, int commandTimeOut) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return new DatabaseSource( + SqlClientFactory.Instance, + GetMSSQLConnectionString(TestDatasets.irisDb.name), + String.Format(command, $@"""{TestDatasets.irisDb.trainFilename}"""), + commandTimeOut); + else + return new DatabaseSource( + SQLiteFactory.Instance, + GetSQLiteConnectionString(TestDatasets.irisDbSQLite.name), + String.Format(command, TestDatasets.irisDbSQLite.trainFilename), + commandTimeOut); + } + private string GetMSSQLConnectionString(string databaseName) { var databaseFile = Path.GetFullPath(Path.Combine("TestDatabases", $"{databaseName}.mdf")); From 2ffddfa66acba849ceef687f2e6ec82214e80623 Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Mon, 6 Jul 2020 14:14:17 -0700 Subject: [PATCH 3/9] fix comments --- src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs index 146338ed94..172955416f 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs @@ -29,7 +29,7 @@ public DatabaseSource(DbProviderFactory providerFactory, string connectionString /// The factory used to create the .. /// The string used to open the connection. /// The text command to run against the data source. - /// The text command to run against the data source. + /// The timeout(in seconds) for database command. public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText, int commandTimeOut) { Contracts.CheckValue(providerFactory, nameof(providerFactory)); From 2406db17649fd29c374088a9e6edb97aa1229828 Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Mon, 6 Jul 2020 14:46:27 -0700 Subject: [PATCH 4/9] remove empty lines --- test/Microsoft.ML.Tests/DatabaseLoaderTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index 5b72e671c0..00921e3adc 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -38,7 +38,6 @@ public void IrisLightGbmWithTimeOut() DatabaseSource dbs = GetIrisDatabaseSourceWithTimeOut("WAITFOR DELAY '00:00:05'; SELECT * FROM {0}", 1); var ex = Assert.Throws(() => IrisLightGbmImpl(dbs)); Assert.Contains("Timeout expired", ex.InnerException.Message); - } private void IrisLightGbmImpl(DatabaseSource dbs) From 6cfec0e3f8d607d4735b92eb499d12117f3f2362 Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Mon, 6 Jul 2020 17:08:28 -0700 Subject: [PATCH 5/9] resolve comments and attempt to fix test failure --- .../DataLoadSave/Database/DatabaseLoaderCursor.cs | 4 ++-- src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs | 2 +- test/Microsoft.ML.Tests/DatabaseLoaderTests.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs index 8dd8275f64..2b15328e6f 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs @@ -75,9 +75,9 @@ public DbCommand Command { _command = Connection.CreateCommand(); _command.CommandText = _source.CommandText; - if (_source.CommandTimeOut is {}) + if (_source.CommandTimeOut.HasValue) { - _command.CommandTimeout = _source.CommandTimeOut; + _command.CommandTimeout = _source.CommandTimeOut.Value; } } return _command; diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs index 172955416f..be0067a4c7 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs @@ -43,7 +43,7 @@ public DatabaseSource(DbProviderFactory providerFactory, string connectionString } /// Gets the timeout for database command. - public int CommandTimeOut { get; } + public int? CommandTimeOut { get; } /// Gets the text command to run against the data source. public string CommandText { get; } diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index 00921e3adc..f7384e27df 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -35,7 +35,7 @@ public void IrisLightGbm() [LightGBMFact] public void IrisLightGbmWithTimeOut() { - DatabaseSource dbs = GetIrisDatabaseSourceWithTimeOut("WAITFOR DELAY '00:00:05'; SELECT * FROM {0}", 1); + DatabaseSource dbs = GetIrisDatabaseSourceWithTimeOut("WAITFOR DELAY '00:00:01'; SELECT * FROM {0}", 1); var ex = Assert.Throws(() => IrisLightGbmImpl(dbs)); Assert.Contains("Timeout expired", ex.InnerException.Message); } From c8d3c4e12c7feb799c89c686817be2322c74b4b8 Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Tue, 7 Jul 2020 12:52:09 -0700 Subject: [PATCH 6/9] fix comments --- .../Database/DatabaseLoaderCursor.cs | 4 +-- .../DataLoadSave/Database/DatabaseSource.cs | 17 +++++------- .../Microsoft.ML.Tests/DatabaseLoaderTests.cs | 26 +++++-------------- 3 files changed, 15 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs index 2b15328e6f..e417782372 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs @@ -75,9 +75,9 @@ public DbCommand Command { _command = Connection.CreateCommand(); _command.CommandText = _source.CommandText; - if (_source.CommandTimeOut.HasValue) + if (_source.CommandTimeOutInSeconds.HasValue) { - _command.CommandTimeout = _source.CommandTimeOut.Value; + _command.CommandTimeout = _source.CommandTimeOutInSeconds.Value; } } return _command; diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs index be0067a4c7..9567d08ce2 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs @@ -29,21 +29,16 @@ public DatabaseSource(DbProviderFactory providerFactory, string connectionString /// The factory used to create the .. /// The string used to open the connection. /// The text command to run against the data source. - /// The timeout(in seconds) for database command. - public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText, int commandTimeOut) + /// The timeout(in seconds) for database command. + public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText, int commandTimeOutInSeconds) : + this(providerFactory, connectionString, commandText) { - Contracts.CheckValue(providerFactory, nameof(providerFactory)); - Contracts.CheckValue(connectionString, nameof(connectionString)); - Contracts.CheckValue(commandText, nameof(commandText)); - - ProviderFactory = providerFactory; - ConnectionString = connectionString; - CommandText = commandText; - CommandTimeOut = commandTimeOut; + Contracts.CheckUserArg(commandTimeOutInSeconds >= 0, nameof(commandTimeOutInSeconds)); + CommandTimeOutInSeconds = commandTimeOutInSeconds; } /// Gets the timeout for database command. - public int? CommandTimeOut { get; } + public int? CommandTimeOutInSeconds { get; } /// Gets the text command to run against the data source. public string CommandText { get; } diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index f7384e27df..3aab923393 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -35,9 +35,11 @@ public void IrisLightGbm() [LightGBMFact] public void IrisLightGbmWithTimeOut() { - DatabaseSource dbs = GetIrisDatabaseSourceWithTimeOut("WAITFOR DELAY '00:00:01'; SELECT * FROM {0}", 1); + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) //sqlite does not have built-in command for sleep + return; + DatabaseSource dbs = GetIrisDatabaseSource("WAITFOR DELAY '00:00:01'; SELECT * FROM {0}", 1); var ex = Assert.Throws(() => IrisLightGbmImpl(dbs)); - Assert.Contains("Timeout expired", ex.InnerException.Message); + Assert.Contains("Timeout", ex.InnerException.Message); } private void IrisLightGbmImpl(DatabaseSource dbs) @@ -225,34 +227,20 @@ public void IrisSdcaMaximumEntropy() /// SQLite database is used on Linux and MacOS builds. /// /// Return the appropiate Iris DatabaseSource according to build OS. - private DatabaseSource GetIrisDatabaseSource(string command) - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return new DatabaseSource( - SqlClientFactory.Instance, - GetMSSQLConnectionString(TestDatasets.irisDb.name), - String.Format(command, $@"""{TestDatasets.irisDb.trainFilename}""")); - else - return new DatabaseSource( - SQLiteFactory.Instance, - GetSQLiteConnectionString(TestDatasets.irisDbSQLite.name), - String.Format(command, TestDatasets.irisDbSQLite.trainFilename)); - } - - private DatabaseSource GetIrisDatabaseSourceWithTimeOut(string command, int commandTimeOut) + private DatabaseSource GetIrisDatabaseSource(string command, int commandTimeOutInSeconds = 30) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return new DatabaseSource( SqlClientFactory.Instance, GetMSSQLConnectionString(TestDatasets.irisDb.name), String.Format(command, $@"""{TestDatasets.irisDb.trainFilename}"""), - commandTimeOut); + commandTimeOutInSeconds); else return new DatabaseSource( SQLiteFactory.Instance, GetSQLiteConnectionString(TestDatasets.irisDbSQLite.name), String.Format(command, TestDatasets.irisDbSQLite.trainFilename), - commandTimeOut); + commandTimeOutInSeconds); } private string GetMSSQLConnectionString(string databaseName) From 364b4110bd42a80efbd964e1a469d9e667da899c Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Tue, 7 Jul 2020 15:01:50 -0700 Subject: [PATCH 7/9] update --- .../Database/DatabaseLoaderCursor.cs | 5 +--- .../DataLoadSave/Database/DatabaseSource.cs | 29 ++++++++++--------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs index e417782372..ed092be157 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseLoaderCursor.cs @@ -75,10 +75,7 @@ public DbCommand Command { _command = Connection.CreateCommand(); _command.CommandText = _source.CommandText; - if (_source.CommandTimeOutInSeconds.HasValue) - { - _command.CommandTimeout = _source.CommandTimeOutInSeconds.Value; - } + _command.CommandTimeout = _source.CommandTimeoutInSeconds; } return _command; } diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs index 9567d08ce2..f7c774bc48 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs @@ -10,35 +10,36 @@ namespace Microsoft.ML.Data /// Exposes the data required for opening a database for reading. public sealed class DatabaseSource { + public static readonly int DefaultCommandTimeoutInSeconds = 30; /// Creates a new instance of the class. /// The factory used to create the .. /// The string used to open the connection. /// The text command to run against the data source. - public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText) + public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText) : + this(providerFactory, connectionString, commandText, DefaultCommandTimeoutInSeconds) { - Contracts.CheckValue(providerFactory, nameof(providerFactory)); - Contracts.CheckValue(connectionString, nameof(connectionString)); - Contracts.CheckValue(commandText, nameof(commandText)); - - ProviderFactory = providerFactory; - ConnectionString = connectionString; - CommandText = commandText; } /// Creates a new instance of the class. /// The factory used to create the .. /// The string used to open the connection. /// The text command to run against the data source. - /// The timeout(in seconds) for database command. - public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText, int commandTimeOutInSeconds) : - this(providerFactory, connectionString, commandText) + /// The timeout(in seconds) for database command. + public DatabaseSource(DbProviderFactory providerFactory, string connectionString, string commandText, int commandTimeoutInSeconds) { - Contracts.CheckUserArg(commandTimeOutInSeconds >= 0, nameof(commandTimeOutInSeconds)); - CommandTimeOutInSeconds = commandTimeOutInSeconds; + Contracts.CheckValue(providerFactory, nameof(providerFactory)); + Contracts.CheckValue(connectionString, nameof(connectionString)); + Contracts.CheckValue(commandText, nameof(commandText)); + Contracts.CheckUserArg(commandTimeoutInSeconds >= 0, nameof(commandTimeoutInSeconds)); + + ProviderFactory = providerFactory; + ConnectionString = connectionString; + CommandText = commandText; + CommandTimeoutInSeconds = commandTimeoutInSeconds; } /// Gets the timeout for database command. - public int? CommandTimeOutInSeconds { get; } + public int CommandTimeoutInSeconds { get; } /// Gets the text command to run against the data source. public string CommandText { get; } From ae514d33c137d88058a4531ca385a9a765115299 Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Tue, 7 Jul 2020 15:03:51 -0700 Subject: [PATCH 8/9] update --- test/Microsoft.ML.Tests/DatabaseLoaderTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs index 3aab923393..b6ca011fed 100644 --- a/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs +++ b/test/Microsoft.ML.Tests/DatabaseLoaderTests.cs @@ -33,7 +33,7 @@ public void IrisLightGbm() } [LightGBMFact] - public void IrisLightGbmWithTimeOut() + public void IrisLightGbmWithTimeout() { if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) //sqlite does not have built-in command for sleep return; @@ -227,20 +227,20 @@ public void IrisSdcaMaximumEntropy() /// SQLite database is used on Linux and MacOS builds. /// /// Return the appropiate Iris DatabaseSource according to build OS. - private DatabaseSource GetIrisDatabaseSource(string command, int commandTimeOutInSeconds = 30) + private DatabaseSource GetIrisDatabaseSource(string command, int commandTimeoutInSeconds = 30) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return new DatabaseSource( SqlClientFactory.Instance, GetMSSQLConnectionString(TestDatasets.irisDb.name), String.Format(command, $@"""{TestDatasets.irisDb.trainFilename}"""), - commandTimeOutInSeconds); + commandTimeoutInSeconds); else return new DatabaseSource( SQLiteFactory.Instance, GetSQLiteConnectionString(TestDatasets.irisDbSQLite.name), String.Format(command, TestDatasets.irisDbSQLite.trainFilename), - commandTimeOutInSeconds); + commandTimeoutInSeconds); } private string GetMSSQLConnectionString(string databaseName) From 9ffea4a09f41b8570a5a4cbefcd02ff0a241599b Mon Sep 17 00:00:00 2001 From: Ye Wang <52801275+wangyems@users.noreply.github.com> Date: Tue, 7 Jul 2020 17:19:45 -0700 Subject: [PATCH 9/9] minor change --- src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs index f7c774bc48..77f971dba1 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Database/DatabaseSource.cs @@ -10,7 +10,8 @@ namespace Microsoft.ML.Data /// Exposes the data required for opening a database for reading. public sealed class DatabaseSource { - public static readonly int DefaultCommandTimeoutInSeconds = 30; + private const int DefaultCommandTimeoutInSeconds = 30; + /// Creates a new instance of the class. /// The factory used to create the .. /// The string used to open the connection.