groupid addition option 2

src/Microsoft.ML.AutoML/API/RankingExperiment.cs

@@ -19,6 +19,12 @@ public sealed class RankingExperimentSettings : ExperimentSettings
         /// <value>The default value is <see cref="RankingMetric" />.</value>
         public RankingMetric OptimizingMetric { get; set; }
 
+        /// <summary>
+        /// Name for the GroupId column.
+        /// </summary>
+        /// <value>The default value is GroupId.</value>
+        public string GroupIdColumnName { get; set; }
+
         /// <summary>
         /// Collection of trainers the AutoML experiment can leverage.
         /// </summary>
@@ -28,6 +34,7 @@ public sealed class RankingExperimentSettings : ExperimentSettings
         public ICollection<RankingTrainer> Trainers { get; }
         public RankingExperimentSettings()
         {
+            GroupIdColumnName = "GroupId";
             OptimizingMetric = RankingMetric.Ndcg;
             Trainers = Enum.GetValues(typeof(RankingTrainer)).OfType<RankingTrainer>().ToList();
         }
@@ -68,10 +75,11 @@ public static class RankingExperimentResultExtensions
         /// </summary>
         /// <param name="results">Enumeration of AutoML experiment run results.</param>
         /// <param name="metric">Metric to consider when selecting the best run.</param>
+        /// <param name="groupIdColumnName">Name for the GroupId column.</param>
         /// <returns>The best experiment run.</returns>
-        public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg)
+        public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId")
         {
-            var metricsAgent = new RankingMetricsAgent(null, metric);
+            var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
             var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
             return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
         }
@@ -81,10 +89,11 @@ public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingM
         /// </summary>
         /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
         /// <param name="metric">Metric to consider when selecting the best run.</param>
+        /// <param name="groupIdColumnName">Name for the GroupId column.</param>
         /// <returns>The best experiment run.</returns>
-        public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg)
+        public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId")
         {
-            var metricsAgent = new RankingMetricsAgent(null, metric);
+            var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
             var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
             return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
         }
@@ -103,7 +112,7 @@ public sealed class RankingExperiment : ExperimentBase<RankingMetrics, RankingEx
     {
         internal RankingExperiment(MLContext context, RankingExperimentSettings settings)
             : base(context,
-                  new RankingMetricsAgent(context, settings.OptimizingMetric),
+                  new RankingMetricsAgent(context, settings.OptimizingMetric, settings.GroupIdColumnName),
                   new OptimizingMetricInfo(settings.OptimizingMetric),
                   settings,
                   TaskKind.Ranking,

src/Microsoft.ML.AutoML/Experiment/MetricsAgents/BinaryMetricsAgent.cs

@@ -78,7 +78,7 @@ public bool IsModelPerfect(double score)
             }
         }
 
-        public BinaryClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId)
+        public BinaryClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn)
         {
             return _mlContext.BinaryClassification.EvaluateNonCalibrated(data, labelColumn);
         }

src/Microsoft.ML.AutoML/Experiment/MetricsAgents/IMetricsAgent.cs

@@ -10,6 +10,6 @@ internal interface IMetricsAgent<T>
 
         bool IsModelPerfect(double score);
 
-        T EvaluateMetrics(IDataView data, string labelColumn, string groupId);
+        T EvaluateMetrics(IDataView data, string labelColumn);
     }
 }

src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs

@@ -66,7 +66,7 @@ public bool IsModelPerfect(double score)
             }
         }
 
-        public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId)
+        public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn)
         {
             return _mlContext.MulticlassClassification.Evaluate(data, labelColumn);
         }

src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs

@@ -10,11 +10,13 @@ internal class RankingMetricsAgent : IMetricsAgent<RankingMetrics>
     {
         private readonly MLContext _mlContext;
         private readonly RankingMetric _optimizingMetric;
+        private readonly string _groupIdColumnName;
 
-        public RankingMetricsAgent(MLContext mlContext, RankingMetric optimizingMetric)
+        public RankingMetricsAgent(MLContext mlContext, RankingMetric optimizingMetric, string groupIdColumnName)
         {
             _mlContext = mlContext;
             _optimizingMetric = optimizingMetric;
+            _groupIdColumnName = groupIdColumnName;
         }
 
         // Optimizing metric used: NDCG@10 and DCG@10
@@ -57,9 +59,9 @@ public bool IsModelPerfect(double score)
             }
         }
 
-        public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupIdColumn)
+        public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn)
         {
-            return _mlContext.Ranking.Evaluate(data, labelColumn, groupIdColumn);
+            return _mlContext.Ranking.Evaluate(data, labelColumn, _groupIdColumnName);
         }
     }
 }

src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs

@@ -61,7 +61,7 @@ public bool IsModelPerfect(double score)
             }
         }
 
-        public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId)
+        public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn)
         {
             return _mlContext.Regression.Evaluate(data, labelColumn);
         }

src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs

@@ -30,8 +30,8 @@ public CrossValRunner(MLContext context,
             IMetricsAgent<TMetrics> metricsAgent,
             IEstimator<ITransformer> preFeaturizer,
             ITransformer[] preprocessorTransforms,
-            string labelColumn,
             string groupIdColumn,
+            string labelColumn,
             IChannel logger)
         {
             _context = context;
@@ -55,7 +55,7 @@ public CrossValRunner(MLContext context,
             {
                 var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1);
                 var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i],
-                    _labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger);
+                    _labelColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger);
                 trainResults.Add(new SuggestedPipelineTrainResult<TMetrics>(trainResult.model, trainResult.metrics, trainResult.exception, trainResult.score));
             }
 

src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs

@@ -59,7 +59,7 @@ public CrossValSummaryRunner(MLContext context,
             {
                 var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1);
                 var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i],
-                    _labelColumn, _groupIdColumn,_metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema,
+                    _labelColumn, _metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema,
                     _logger);
                 trainResults.Add(trainResult);
             }

src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs

@@ -16,7 +16,6 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub
             IDataView trainData,
             IDataView validData,
             string labelColumn,
-            string groupId,
             IMetricsAgent<TMetrics> metricsAgent,
             ITransformer preprocessorTransform,
             FileInfo modelFileInfo,
@@ -29,7 +28,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub
                 var model = estimator.Fit(trainData);
 
                 var scoredData = model.Transform(validData);
-                var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn, groupId);
+                var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn);
                 var score = metricsAgent.GetScore(metrics);
 
                 if (preprocessorTransform != null)

src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs

@@ -48,7 +48,7 @@ public TrainValidateRunner(MLContext context,
         {
             var modelFileInfo = GetModelFileInfo(modelDirectory, iterationNum);
             var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainData, _validData,
-                _labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger);
+                _labelColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger);
             var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail<TMetrics>(pipeline,
                 trainResult.score,
                 trainResult.exception == null,

src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs

@@ -35,9 +35,9 @@ public static RunDetail<MulticlassClassificationMetrics> GetBestRun(IEnumerable<
         }
 
         public static RunDetail<RankingMetrics> GetBestRun(IEnumerable<RunDetail<RankingMetrics>> results,
-            RankingMetric metric)
+            RankingMetric metric, string groupIdColumnName)
         {
-            var metricsAgent = new RankingMetricsAgent(null, metric);
+            var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
             var metricInfo = new OptimizingMetricInfo(metric);
             return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing);
         }

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -136,7 +136,7 @@ public void AutoFitRankingTest()
             trainDataView = mlContext.Data.SkipRows(trainDataView, 500);
             // STEP 2: Run AutoML experiment
             ExperimentResult<RankingMetrics> experimentResult = mlContext.Auto()
-                .CreateRankingExperiment(5)
+                .CreateRankingExperiment(new RankingExperimentSettings() { GroupIdColumnName = "CustomGroupId", MaxExperimentTimeInSeconds = 5})
                 .Execute(trainDataView, testDataView,
                     new ColumnInformation()
                     {

test/Microsoft.ML.AutoML.Tests/MetricsAgentsTests.cs

@@ -127,14 +127,14 @@ public void RankingMetricsGetScoreTest()
             double[] ndcg = { 0.2, 0.3, 0.4 };
             double[] dcg = { 0.2, 0.3, 0.4 };
             var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg);
-            Assert.Equal(0.4, GetScore(metrics, RankingMetric.Dcg));
-            Assert.Equal(0.4, GetScore(metrics, RankingMetric.Ndcg));
+            Assert.Equal(0.4, GetScore(metrics, RankingMetric.Dcg, "GroupId"));
+            Assert.Equal(0.4, GetScore(metrics, RankingMetric.Ndcg, "GroupId"));
 
             double[] largeNdcg = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95 };
             double[] largeDcg = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95 };
             metrics = MetricsUtil.CreateRankingMetrics(largeDcg, largeNdcg);
-            Assert.Equal(0.9, GetScore(metrics, RankingMetric.Dcg));
-            Assert.Equal(0.9, GetScore(metrics, RankingMetric.Ndcg));
+            Assert.Equal(0.9, GetScore(metrics, RankingMetric.Dcg, "GroupId"));
+            Assert.Equal(0.9, GetScore(metrics, RankingMetric.Ndcg, "GroupId"));
         }
 
         [Fact]
@@ -143,8 +143,8 @@ public void RankingMetricsNonPerfectTest()
             double[] ndcg = { 0.2, 0.3, 0.4 };
             double[] dcg = { 0.2, 0.3, 0.4 };
             var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg);
-            Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg));
-            Assert.False(IsPerfectModel(metrics, RankingMetric.Ndcg));
+            Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg, "GroupId"));
+            Assert.False(IsPerfectModel(metrics, RankingMetric.Ndcg, "GroupId"));
         }
 
         [Fact]
@@ -153,8 +153,8 @@ public void RankingMetricsPerfectTest()
             double[] ndcg = { 0.2, 0.3, 1 };
             double[] dcg = { 0.2, 0.3, 1 };
             var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg);
-            Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg)); //REVIEW: No true Perfect model
-            Assert.True(IsPerfectModel(metrics, RankingMetric.Ndcg));
+            Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg, "GroupId")); //REVIEW: No true Perfect model
+            Assert.True(IsPerfectModel(metrics, RankingMetric.Ndcg, "GroupId"));
         }
 
         [Fact]
@@ -179,9 +179,9 @@ private static double GetScore(RegressionMetrics metrics, RegressionMetric metri
             return new RegressionMetricsAgent(null, metric).GetScore(metrics);
         }
 
-        private static double GetScore(RankingMetrics metrics, RankingMetric metric)
+        private static double GetScore(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName)
         {
-            return new RankingMetricsAgent(null, metric).GetScore(metrics);
+            return new RankingMetricsAgent(null, metric, groupIdColumnName).GetScore(metrics);
         }
 
         private static bool IsPerfectModel(BinaryClassificationMetrics metrics, BinaryClassificationMetric metric)
@@ -202,9 +202,9 @@ private static bool IsPerfectModel(RegressionMetrics metrics, RegressionMetric m
             return IsPerfectModel(metricsAgent, metrics);
         }
 
-        private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric)
+        private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName)
         {
-            var metricsAgent = new RankingMetricsAgent(null, metric);
+            var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
             return IsPerfectModel(metricsAgent, metrics);
         }