From 4c9c786eecc37f24d2a756b0227c7f0d3e10f69c Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 12 Jun 2019 17:32:05 -0700 Subject: [PATCH] Change default EvaluationMetric for LightGbm trainers to conform to default metric in standalone LightGbm --- src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs | 2 +- src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs | 2 +- src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs | 2 +- src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs | 2 +- test/BaselineOutput/Common/EntryPoints/core_manifest.json | 8 ++++---- .../LightGBMReg-CV-generatedRegressionDataset-out.txt | 4 ++-- ...ghtGBMReg-TrainTest-generatedRegressionDataset-out.txt | 2 +- ...htGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt | 4 ++-- ...gRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs index 22de1c03f7..568e5a0ddb 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs @@ -162,7 +162,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Logloss; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { diff --git a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs index 29f19f6b2c..f82b937a14 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs @@ -95,7 +95,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Error; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { diff --git a/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs index 12cc0b148e..85dd19f65b 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs @@ -143,7 +143,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.NormalizedDiscountedCumulativeGain; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { diff --git a/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs index 90ad8cfa39..618b04274e 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs @@ -133,7 +133,7 @@ public enum EvaluateMetricType [Argument(ArgumentType.AtMostOnce, HelpText = "Evaluation metrics.", ShortName = "em")] - public EvaluateMetricType EvaluationMetric = EvaluateMetricType.RootMeanSquaredError; + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default; static Options() { diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 99b0d3a8ab..abc8891856 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -11297,7 +11297,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "Logloss" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", @@ -11782,7 +11782,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "Error" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", @@ -12279,7 +12279,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "NormalizedDiscountedCumulativeGain" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", @@ -12737,7 +12737,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "RootMeanSquaredError" + "Default": "Default" }, { "Name": "MaximumBinCountPerFeature", diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt index c81802feb6..29dfd1294a 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt @@ -35,10 +35,10 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-rmse: 6.09160118577349 +[2] (%Time%) Iteration: 50 Training-: 37.107605006517 [2] 'Training with LightGBM' finished in %Time%. [3] 'Loading data for LightGBM #2' started. [3] 'Loading data for LightGBM #2' finished in %Time%. [4] 'Training with LightGBM #2' started. -[4] (%Time%) Iteration: 50 Training-rmse: 5.26343689176522 +[4] (%Time%) Iteration: 50 Training-: 27.7037679135951 [4] 'Training with LightGBM #2' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt index c88cad62b7..668caf5901 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt @@ -26,7 +26,7 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-rmse: 5.10533343749577 +[2] (%Time%) Iteration: 50 Training-: 26.0644295080124 [2] 'Training with LightGBM' finished in %Time%. [3] 'Saving model' started. [3] 'Saving model' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt index d855ef6c79..ba3cb14b0d 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -26.59978 1393.326 37.32081 1393.326 0.923402 50 0.2 20 10 + 1 LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /em /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +26.59978 1393.326 37.32081 1393.326 0.923402 RootMeanSquaredError 50 0.2 20 10 + 1 LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /em:RootMeanSquaredError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt index bf64ad5ed2..cdff8dbb38 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -3.428896 25.23601 5.023546 25.23601 0.998616 50 0.2 20 10 + 1 LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /em /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +3.428896 25.23601 5.023546 25.23601 0.998616 RootMeanSquaredError 50 0.2 20 10 + 1 LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /em:RootMeanSquaredError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1