Added unit test for checking canceled experiment

src/Microsoft.ML.AutoML/Experiment/Experiment.cs

@@ -26,6 +26,7 @@ internal class Experiment<TRunDetail, TMetrics> where TRunDetail : RunDetail
         private readonly IRunner<TRunDetail> _runner;
         private readonly IList<SuggestedPipelineRunDetail> _history;
         private readonly IChannel _logger;
+        private bool _endExperimentWhenAble = false;
 
         public Experiment(MLContext context,
             TaskKind task,
@@ -54,21 +55,36 @@ public Experiment(MLContext context,
 
         private void MaxExperimentTimeExpiredEvent(object sender, EventArgs e)
         {
-            _logger.Warning("Allocated time for Experiment of {0} seconds has elapsed with {1} models run. Ending experiment...",
-                _experimentSettings.MaxExperimentTimeInSeconds, _history.Count());
-            _context.CancelExecution();
+            // If at least one model was run, end experiment immediately.
+            // Else, wait for first model to run before experiment is concluded.
+            _endExperimentWhenAble = true;
+            if (_history.Count > 0)
+            {
+                _logger.Warning("Allocated time for Experiment of {0} seconds has elapsed with {1} models run. Ending experiment...",
+                    _experimentSettings.MaxExperimentTimeInSeconds, _history.Count());
+                _context.CancelExecution();
+            }
         }
 
         public IList<TRunDetail> Execute()
         {
             var iterationResults = new List<TRunDetail>();
             // Create a timer for the max duration of experiment. When given time has
             // elapsed, MaxExperimentTimeExpiredEvent is called to interrupt training
-            // of current model.
-            Timer timer = new Timer(_experimentSettings.MaxExperimentTimeInSeconds * 1000);
-            timer.Elapsed += MaxExperimentTimeExpiredEvent;
-            timer.AutoReset = false;
-            timer.Enabled = true;
+            // of current model. Timer is not used if no experiment time is given, or
+            // is not a positive number.
+            if (_experimentSettings.MaxExperimentTimeInSeconds > 0)
+            {
+                Timer timer = new Timer(_experimentSettings.MaxExperimentTimeInSeconds * 1000);
+                timer.Elapsed += MaxExperimentTimeExpiredEvent;
+                timer.AutoReset = false;
+                timer.Enabled = true;
+            }
+            // If given max duration of experiment is 0, only 1 model will be trained.
+            // _experimentSettings.MaxExperimentTimeInSeconds is of type uint, it is
+            // either 0 or >0.
+            else
+                _endExperimentWhenAble = true;
 
             do
             {
@@ -114,7 +130,8 @@ public IList<TRunDetail> Execute()
                 }
 
             } while (_history.Count < _experimentSettings.MaxModels &&
-                    !_experimentSettings.CancellationToken.IsCancellationRequested);
+                    !_experimentSettings.CancellationToken.IsCancellationRequested &&
+                    !_endExperimentWhenAble);
 
             return iterationResults;
         }

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -4,6 +4,7 @@
 
 using System.Linq;
 using Microsoft.ML.Data;
+using Microsoft.ML.Runtime;
 using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.TestFrameworkCommon;
@@ -320,6 +321,23 @@ public void AutoFitWithPresplittedData()
 
         }
 
+        [Fact]
+        public void AutoFitMaxExperimentTimeTest()
+		{
+            // 1 Binary classification experiment takes less than 5 seconds.
+            // System.OperationCanceledException is thrown when ongoing experiment
+            // is canceled and at least one model has been generated.
+            var context = new MLContext(1);
+            var dataPath = DatasetUtil.GetUciAdultDataset();
+            var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel);
+            var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions);
+            var trainData = textLoader.Load(dataPath);
+            var experiment = context.Auto()
+                .CreateBinaryClassificationExperiment(5)
+                .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel });
+            Assert.True((context.Model.GetEnvironment() as ICancelable).IsCanceled);
+        }
+
         private TextLoader.Options GetLoaderArgs(string labelColumnName, string userIdColumnName, string itemIdColumnName)
         {
             return new TextLoader.Options()