add RecommendationExperiment and other functions

docs/samples/Microsoft.ML.AutoML.Samples/Program.cs

@@ -8,6 +8,9 @@ public static void Main(string[] args)
         {
             try
             {
+                RecommendationExperiment.Run();
+                Console.Clear();
+
                 RegressionExperiment.Run();
                 Console.Clear();
 

docs/samples/Microsoft.ML.AutoML.Samples/RecommendationExperiment.cs

@@ -0,0 +1,77 @@
+using System;
+using System.IO;
+using System.Linq;
+using Microsoft.ML.AutoML;
+using Microsoft.ML.Data;
+
+namespace Microsoft.ML.AutoML.Samples
+{
+    public static class RecommendationExperiment
+    {
+        private static string TrainDataPath = @"C:\Users\xiaoyuz\Desktop\machinelearning-samples\datasets\recommendation-ratings-train.csv";
+        private static string TestDataPath = @"C:\Users\xiaoyuz\Desktop\machinelearning-samples\datasets\recommendation-ratings-test.csv";
+        private static string ModelPath = @"C:\Users\xiaoyuz\source\test\recommendation.zip";
+        private static string LabelColumnName = "rating";
+        private static uint ExperimentTime = 60;
+
+        public static void Run()
+        {
+            MLContext mlContext = new MLContext();
+
+            // STEP 1: Load data
+            IDataView trainDataView = mlContext.Data.LoadFromTextFile<TaxiTrip>(TrainDataPath, hasHeader: true, separatorChar: ',');
+            IDataView testDataView = mlContext.Data.LoadFromTextFile<TaxiTrip>(TestDataPath, hasHeader: true, separatorChar: ',');
+
+            var settings = new RecommendationExperimentSettings(RecommendationExperimentScenario.MF, "userId", "movieId");
+            // STEP 2: Run AutoML experiment
+            Console.WriteLine($"Running AutoML regression experiment for {ExperimentTime} seconds...");
+            ExperimentResult<RegressionMetrics> experimentResult = mlContext.Auto()
+                .CreateRecommendationExperiment(settings)
+                .Execute(trainDataView, LabelColumnName);
+
+            // STEP 3: Print metric from best model
+            RunDetail<RegressionMetrics> bestRun = experimentResult.BestRun;
+            Console.WriteLine($"Total models produced: {experimentResult.RunDetails.Count()}");
+            Console.WriteLine($"Best model's trainer: {bestRun.TrainerName}");
+            Console.WriteLine($"Metrics of best model from validation data --");
+            PrintMetrics(bestRun.ValidationMetrics);
+
+            // STEP 5: Evaluate test data
+            IDataView testDataViewWithBestScore = bestRun.Model.Transform(testDataView);
+            RegressionMetrics testMetrics = mlContext.Regression.Evaluate(testDataViewWithBestScore, labelColumnName: LabelColumnName);
+            Console.WriteLine($"Metrics of best model on test data --");
+            PrintMetrics(testMetrics);
+
+            // STEP 6: Save the best model for later deployment and inferencing
+            using (FileStream fs = File.Create(ModelPath))
+                mlContext.Model.Save(bestRun.Model, trainDataView.Schema, fs);
+
+            // STEP 7: Create prediction engine from the best trained model
+            var predictionEngine = mlContext.Model.CreatePredictionEngine<TaxiTrip, TaxiTripFarePrediction>(bestRun.Model);
+
+            // STEP 8: Initialize a new test taxi trip, and get the predicted fare
+            var testTaxiTrip = new TaxiTrip
+            {
+                VendorId = "VTS",
+                RateCode = 1,
+                PassengerCount = 1,
+                TripTimeInSeconds = 1140,
+                TripDistance = 3.75f,
+                PaymentType = "CRD"
+            };
+            var prediction = predictionEngine.Predict(testTaxiTrip);
+            Console.WriteLine($"Predicted fare for test taxi trip: {prediction.FareAmount}");
+
+            Console.WriteLine("Press any key to continue...");
+            Console.ReadKey();
+        }
+
+        private static void PrintMetrics(RegressionMetrics metrics)
+        {
+            Console.WriteLine($"MeanAbsoluteError: {metrics.MeanAbsoluteError}");
+            Console.WriteLine($"MeanSquaredError: {metrics.MeanSquaredError}");
+            Console.WriteLine($"RootMeanSquaredError: {metrics.RootMeanSquaredError}");
+            Console.WriteLine($"RSquared: {metrics.RSquared}");
+        }
+    }
+}

src/Microsoft.ML.AutoML/API/AutoCatalog.cs

@@ -17,7 +17,7 @@ public sealed class AutoCatalog
         /// <summary>
         /// save some intermediate value
         /// </summary>
-        public static Dictionary<string, object> ValuePairs { get; set; };
+        public static Dictionary<string, object> ValuePairs { get; set; }
 
         internal AutoCatalog(MLContext context)
         {
@@ -129,6 +129,11 @@ public MulticlassClassificationExperiment CreateMulticlassClassificationExperime
             return new MulticlassClassificationExperiment(_context, experimentSettings);
         }
 
+        public RecommendationExperiment CreateRecommendationExperiment(RecommendationExperimentSettings experimentSettings)
+        {
+            return new RecommendationExperiment(_context, experimentSettings);
+        }
+
         /// <summary>
         /// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
         /// </summary>

src/Microsoft.ML.AutoML/API/RecommendationExperiment.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+
+namespace Microsoft.ML.AutoML
+{
+    public enum RecommendationExperimentScenario
+    {
+        MF,
+    }
+
+    public sealed class RecommendationExperimentSettings : ExperimentSettings
+    {
+        public RecommendationExperimentScenario Scenerio { get; set; }
+
+        public string MatrixColumnIndexColumnName { get; set; }
+
+        public string MatrixRowIndexColumnName { get; set; }
+
+        // We can use RegressionMetric as evaluation Metric
+        public RegressionMetric OptimizingMetric { get; set; }
+
+        public ICollection<RecommendationTrainer> Trainers { get; }
+
+        public RecommendationExperimentSettings(RecommendationExperimentScenario scenario, string columnIndexName, string rowIndexName)
+            : this()
+        {
+            if(scenario == RecommendationExperimentScenario.MF)
+            {
+                AutoCatalog.ValuePairs[nameof(MatrixFactorizationTrainer.Options.MatrixColumnIndexColumnName)] = columnIndexName;
+                AutoCatalog.ValuePairs[nameof(MatrixFactorizationTrainer.Options.MatrixRowIndexColumnName)] = rowIndexName;
+                return;
+            }
+            throw new NotImplementedException();
+        }
+
+        private RecommendationExperimentSettings()
+        {
+            OptimizingMetric = RegressionMetric.RSquared;
+            Trainers = Enum.GetValues(typeof(RecommendationTrainer)).OfType<RecommendationTrainer>().ToList();
+        }
+    }
+
+    public enum RecommendationTrainer
+    {
+        MatrixFactorization,
+    }
+
+    public sealed class RecommendationExperiment : ExperimentBase<RegressionMetrics, RecommendationExperimentSettings>
+    {
+        internal RecommendationExperiment(MLContext context, RecommendationExperimentSettings settings)
+            : base(context,
+                  new RegressionMetricsAgent(context, settings.OptimizingMetric),
+                  new OptimizingMetricInfo(settings.OptimizingMetric),
+                  settings,
+                  TaskKind.Recommendation,
+                  TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
+        {
+        }
+        private protected override CrossValidationRunDetail<RegressionMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<RegressionMetrics>> results)
+        {
+            return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
+        }
+
+        private protected override RunDetail<RegressionMetrics> GetBestRun(IEnumerable<RunDetail<RegressionMetrics>> results)
+        {
+            return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
+        }
+    }
+}

src/Microsoft.ML.AutoML/TrainerExtensions/RecommendationTrainerExtensions.cs

@@ -1,7 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Microsoft.ML.Data;
 using Microsoft.ML.Trainers;
+using Microsoft.ML.Trainers.Recommender;
 
 namespace Microsoft.ML.AutoML
 {
@@ -12,9 +14,8 @@ internal class MatrixFactorizationExtension : ITrainerExtension
         public ITrainerEsitmator CreateInstance(MLContext mlContext, IEnumerable<SweepableParam> sweepParams, ColumnInformation columnInfo)
         {
             // TODO
-            // MatrixFactorizationTrainer.Options should inheriat from ABC TrainerInputBaseWithGroupId 
-            var options = TrainerExtensionUtil.CreateOptions<MatrixFactorizationTrainer.Options>(sweepParams);
-            options.LabelColumnName = columnInfo.LabelColumnName;
+            // MatrixFactorizationTrainer.Options should inheriat from ABC TrainerInputBaseWithGroupId
+            var options = TrainerExtensionUtil.CreateOptions<MatrixFactorizationTrainer.Options>(sweepParams, columnInfo.LabelColumnName);
             options.MatrixColumnIndexColumnName = (string)AutoCatalog.ValuePairs[nameof(options.MatrixColumnIndexColumnName)];
             options.MatrixRowIndexColumnName = (string)AutoCatalog.ValuePairs[nameof(options.MatrixRowIndexColumnName)];
             return mlContext.Recommendation().Trainers.MatrixFactorization(options);

src/Microsoft.ML.AutoML/TrainerExtensions/SweepableParams.cs

@@ -126,7 +126,6 @@ public static IEnumerable<SweepableParam> BuildMatrixFactorizationParmas()
                 new SweepableDiscreteParam(nameof(MatrixFactorizationTrainer.Options.ApproximationRank), new object[] { 10, 20, 50, 100, 150, 200 }),
             };
         }
-
         public static IEnumerable<SweepableParam> BuildLinearSvmParams()
         {
             return new SweepableParam[] {

src/Microsoft.ML.AutoML/TrainerExtensions/TrainerExtensionUtil.cs

@@ -41,7 +41,8 @@ internal enum TrainerName
         SgdCalibratedBinary,
         SgdCalibratedOva,
         SymbolicSgdLogisticRegressionBinary,
-        SymbolicSgdLogisticRegressionOva
+        SymbolicSgdLogisticRegressionOva,
+        MatrixFactorization,
     }
 
     internal static class TrainerExtensionUtil
@@ -60,16 +61,6 @@ public static T CreateOptions<T>(IEnumerable<SweepableParam> sweepParams, string
             return options;
         }
 
-        public static T CreateOptions<T>(IEnumerable<SweepableParam> sweepParams) where T : class
-        {
-            var options = Activator.CreateInstance<T>();
-            if (sweepParams != null)
-            {
-                UpdateFields(options, sweepParams);
-            }
-            return options;
-        }
-
         private static string[] _lightGbmBoosterParamNames = new[] { "L2Regularization", "L1Regularization" };
         private const string LightGbmBoosterPropName = "Booster";
 
@@ -373,6 +364,18 @@ public static TrainerName GetTrainerName(RegressionTrainer regressionTrainer)
             throw new NotSupportedException($"{regressionTrainer} not supported");
         }
 
+        public static TrainerName GetTrainerName(RecommendationTrainer recommendationTrainer)
+        {
+            switch (recommendationTrainer)
+            {
+                case RecommendationTrainer.MatrixFactorization:
+                    return TrainerName.MatrixFactorization;
+            }
+
+            // never expected to reach here
+            throw new NotSupportedException($"{recommendationTrainer} not supported");
+        }
+
         public static IEnumerable<TrainerName> GetTrainerNames(IEnumerable<BinaryClassificationTrainer> binaryTrainers)
         {
             return binaryTrainers?.Select(t => GetTrainerName(t));
@@ -387,5 +390,10 @@ public static IEnumerable<TrainerName> GetTrainerNames(IEnumerable<RegressionTra
         {
             return regressionTrainers?.Select(t => GetTrainerName(t));
         }
+
+        public static IEnumerable<TrainerName> GetTrainerNames(IEnumerable<RecommendationTrainer> recommendationTrainers)
+        {
+            return recommendationTrainers?.Select(t => GetTrainerName(t));
+        }
     }
 }