Add FixZero for LogMeanVariance normalizer (dotnet#3916)

Author: artidoro

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVariance.cs

@@ -12,15 +12,15 @@ public class NormalizeLogMeanVariance
     {
         public static void Example()
         {
-            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging, 
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
             // as well as the source of randomness.
             var mlContext = new MLContext();
             var samples = new List<DataPoint>()
             {
-                new DataPoint(){ Features = new float[4] { 1, 1, 3, 0} },
-                new DataPoint(){ Features = new float[4] { 2, 2, 2, 0} },
-                new DataPoint(){ Features = new float[4] { 0, 0, 1, 0} },
-                new DataPoint(){ Features = new float[4] {-1,-1,-1, 1} }
+                new DataPoint(){ Features = new float[5] { 1, 1, 3, 0, float.MaxValue } },
+                new DataPoint(){ Features = new float[5] { 2, 2, 2, 0, float.MinValue } },
+                new DataPoint(){ Features = new float[5] { 0, 0, 1, 0, 0} },
+                new DataPoint(){ Features = new float[5] {-1,-1,-1, 1, 1} }
             };
             // Convert training data to IDataView, the general data type used in ML.NET.
             var data = mlContext.Data.LoadFromEnumerable(samples);
@@ -41,19 +41,19 @@ public static void Example()
             foreach (var row in column)
                 Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4"))));
             // Expected output:
-            //  0.1587, 0.1587, 0.8654, 0.0000
-            //  0.8413, 0.8413, 0.5837, 0.0000
-            //  0.0000, 0.0000, 0.0940, 0.0000
-            //  0.0000, 0.0000, 0.0000, 0.0000
+            //  0.1587, 0.1587, 0.8654, 0.0000, 0.8413
+            //  0.8413, 0.8413, 0.5837, 0.0000, 0.0000
+            //  0.0000, 0.0000, 0.0940, 0.0000, 0.0000
+            //  0.0000, 0.0000, 0.0000, 0.0000, 0.1587
 
             var columnFixZero = noCdfData.GetColumn<float[]>("Features").ToArray();
             foreach (var row in columnFixZero)
                 Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4"))));
             // Expected output:
-            //  1.8854, 1.8854, 5.2970, 0.0000
-            //  4.7708, 4.7708, 3.0925, 0.0000
-            // -1.0000,-1.0000, 0.8879, 0.0000
-            //  3.8854,-3.8854,-3.5213, 0.0000
+            //  1.8854, 1.8854, 5.2970, 0.0000, 7670682000000000000000000000000000000.0000
+            //  4.7708, 4.7708, 3.0925, 0.0000, -7670682000000000000000000000000000000.0000
+            // -1.0000,-1.0000, 0.8879, 0.0000, -1.0000
+            // -3.8854,-3.8854,-3.5213, 0.0000, -0.9775
 
             // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters.
             // If we have multiple columns transformations we need to pass index of InputOutputColumnPair.
@@ -64,18 +64,18 @@ public static void Example()
             // ERF is https://en.wikipedia.org/wiki/Error_function.
             // Expected output:
             //  The 1-index value in resulting array would be produce by:
-            //  y = 0.5* (1 + ERF((Math.Log(x)- 0.3465736) / (0.3465736 * sqrt(2)))
+            //  y = 0.5 * (1 + ERF((Math.Log(x) - 0.3465736) / (0.3465736 * sqrt(2)))
             var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters<ImmutableArray<float>>;
             var offset = noCdfParams.Offset.Length == 0 ? 0 : noCdfParams.Offset[1];
             var scale = noCdfParams.Scale[1];
             Console.WriteLine($"The 1-index value in resulting array would be produce by: y = (x - ({offset})) * {scale}");
             // Expected output:
-            //  The 1-index value in resulting array would be produce by: y = (x - (2.88539)) * 0.3465736
+            // The 1-index value in resulting array would be produce by: y = (x - (0.3465736)) * 2.88539
         }
 
         private class DataPoint
         {
-            [VectorType(4)]
+            [VectorType(5)]
             public float[] Features { get; set; }
         }
     }

docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVarianceFixZero.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using static Microsoft.ML.Transforms.NormalizingTransformer;
+
+namespace Samples.Dynamic
+{
+    public class NormalizeLogMeanVarianceFixZero
+    {
+        public static void Example()
+        {
+            // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
+            // as well as the source of randomness.
+            var mlContext = new MLContext();
+            var samples = new List<DataPoint>()
+            {
+                new DataPoint(){ Features = new float[5] { 1, 1, 3, 0, float.MaxValue } },
+                new DataPoint(){ Features = new float[5] { 2, 2, 2, 0, float.MinValue } },
+                new DataPoint(){ Features = new float[5] { 0, 0, 1, 0, 0} },
+                new DataPoint(){ Features = new float[5] {-1,-1,-1, 1, 1} }
+            };
+            // Convert training data to IDataView, the general data type used in ML.NET.
+            var data = mlContext.Data.LoadFromEnumerable(samples);
+            // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data.
+            // Uses Cumulative distribution function as output.
+            var normalize = mlContext.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: true);
+
+            // NormalizeLogMeanVariance normalizes the data based on the computed mean and variance of the logarithm of the data.
+            var normalizeNoCdf = mlContext.Transforms.NormalizeLogMeanVariance("Features", true, useCdf: false);
+
+            // Now we can transform the data and look at the output to confirm the behavior of the estimator.
+            // This operation doesn't actually evaluate data until we read the data below.
+            var normalizeTransform = normalize.Fit(data);
+            var transformedData = normalizeTransform.Transform(data);
+            var normalizeNoCdfTransform = normalizeNoCdf.Fit(data);
+            var noCdfData = normalizeNoCdfTransform.Transform(data);
+            var column = transformedData.GetColumn<float[]>("Features").ToArray();
+            foreach (var row in column)
+                Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4"))));
+            // Expected output:
+            //  0.1587, 0.1587, 0.8654, 0.0000, 0.8413
+            //  0.8413, 0.8413, 0.5837, 0.0000, 0.0000
+            //  0.0000, 0.0000, 0.0940, 0.0000, 0.0000
+            //  0.0000, 0.0000, 0.0000, 0.0000, 0.1587
+
+            var columnFixZero = noCdfData.GetColumn<float[]>("Features").ToArray();
+            foreach (var row in columnFixZero)
+                Console.WriteLine(string.Join(", ", row.Select(x => x.ToString("f4"))));
+            // Expected output:
+            //  2.0403, 2.0403, 4.0001, 0.0000, 5423991000000000000000000000000000000.0000
+            //  4.0806, 4.0806, 2.6667, 0.0000,-5423991000000000000000000000000000000.0000
+            //  0.0000, 0.0000, 1.3334, 0.0000, 0.0000
+            // -2.0403,-2.0403,-1.3334, 0.0000, 0.0159
+
+            // Let's get transformation parameters. Since we work with only one column we need to pass 0 as parameter for GetNormalizerModelParameters.
+            // If we have multiple columns transformations we need to pass index of InputOutputColumnPair.
+            var transformParams = normalizeTransform.GetNormalizerModelParameters(0) as CdfNormalizerModelParameters<ImmutableArray<float>>;
+            Console.WriteLine("The values in the column with index 1 in the resulting array would be produced by:");
+            Console.WriteLine($"y = 0.5* (1 + ERF((Math.Log(x)- {transformParams.Mean[1]}) / ({transformParams.StandardDeviation[1]} * sqrt(2)))");
+
+            // ERF is https://en.wikipedia.org/wiki/Error_function.
+            // Expected output:
+            // The values in the column with index 1 in the resulting array would be produced by:
+            // y = 0.5 * (1 + ERF((Math.Log(x) - 0.3465736) / (0.3465736 * sqrt(2)))
+            var noCdfParams = normalizeNoCdfTransform.GetNormalizerModelParameters(0) as AffineNormalizerModelParameters<ImmutableArray<float>>;
+            var offset = noCdfParams.Offset.Length == 0 ? 0 : noCdfParams.Offset[1];
+            var scale = noCdfParams.Scale[1];
+            Console.WriteLine($"The values in the column with index 1 in the resulting array would be produced by: y = (x - ({offset})) * {scale}");
+            // Expected output:
+            // The values in the column with index 1 in the resulting array would be produced by: y = (x - (0)) * 2.040279
+        }
+
+        private class DataPoint
+        {
+            [VectorType(5)]
+            public float[] Features { get; set; }
+        }
+    }
+}

src/Microsoft.ML.Data/Transforms/NormalizeColumnSng.cs

@@ -1557,7 +1557,7 @@ public static IColumnFunctionBuilder Create(NormalizingEstimator.LogMeanVariance
                 {
                     var lim = column.MaximumExampleCount;
                     host.CheckUserArg(lim > 1, nameof(column.MaximumExampleCount), "Must be greater than 1");
-                    return new MeanVarOneColumnFunctionBuilder(host, lim, false, getter, true, column.UseCdf);
+                    return new MeanVarOneColumnFunctionBuilder(host, lim, column.EnsureZeroUntouched, getter, true, column.UseCdf);
                 }
 
                 protected override bool ProcessValue(in TFloat origVal)
@@ -1633,7 +1633,7 @@ public static IColumnFunctionBuilder Create(NormalizingEstimator.LogMeanVariance
                     var lim = column.MaximumExampleCount;
                     host.CheckUserArg(lim > 1, nameof(column.MaximumExampleCount), "Must be greater than 1");
                     var cv = srcType.Size;
-                    return new MeanVarVecColumnFunctionBuilder(host, cv, lim, false, getter, true, column.UseCdf);
+                    return new MeanVarVecColumnFunctionBuilder(host, cv, lim, column.EnsureZeroUntouched, getter, true, column.UseCdf);
                 }
 
                 protected override bool ProcessValue(in VBuffer<TFloat> buffer)

src/Microsoft.ML.Data/Transforms/Normalizer.cs

@@ -42,7 +42,7 @@ namespace Microsoft.ML.Transforms
     /// The resulting NormalizingEstimator will normalize the data in one of the following ways based upon how it was created:
     /// * Min Max - A linear rescale that is based upon the minimum and maximum values for each row.
     /// * Mean Variance - Rescale each row to unit variance and, optionally, zero mean.
-    /// * Log Mean Variance - Rescale each row to unit variance based on a log scale.
+    /// * Log Mean Variance - Rescale each row to unit variance, optionally, zero mean based on computations in log scale.
     /// * Binning - Bucketizes the data in each row and performs a linear rescale based on the calculated bins.
     /// * Supervised Binning - Bucketize the data in each row and performas a linear rescale based on the calculated bins. The bin calculation is based on correlation of the Label column.
     ///
@@ -54,6 +54,13 @@ namespace Microsoft.ML.Transforms
     /// With Min Max, the distribution depends on how far away the number is from 0, resulting in the number with the largest distance being mapped to 1 if its a positive number
     /// or -1 if its a negative number. The distance from 0 will affect the distribution with a majority of numbers that are closer together normalizing towards 0.
     ///
+    /// The equation for the output $y$ of applying both Mean Variance and Log Mean Variance on input $x$ without
+    /// using the CDF option is: $y = (x - \text{offset}) \text{scale}$. Where offset and scale are computed during training.
+    ///
+    /// Using the CDF option it is: $y = 0.5 * (1 + \text{ERF}((x - \text{mean}) / (\text{standard deviation} * sqrt(2)))$.
+    /// Where ERF is the [Error Function](https://en.wikipedia.org/wiki/Error_function) used to approximate the CDF of a random variable assumed to
+    /// normally distributed. The mean and standard deviation are computing during training.
+    ///
     /// To create this estimator use one of the following:
     /// * [NormalizeMinMax](xref:Microsoft.ML.NormalizationCatalog.NormalizeMinMax(Microsoft.ML.TransformsCatalog, System.String, System.String, System.Int64, System.Boolean))
     /// * [NormalizeMeanVariance](xref:Microsoft.ML.NormalizationCatalog.NormalizeMeanVariance(Microsoft.ML.TransformsCatalog,System.String,System.String,System.Int64,System.Boolean,System.Boolean))
@@ -183,13 +190,13 @@ internal override IColumnFunctionBuilder MakeBuilder(IHost host, int srcIndex, D
         }
 
         [BestFriend]
-        internal sealed class LogMeanVarianceColumnOptions : ColumnOptionsBase
+        internal sealed class LogMeanVarianceColumnOptions : ControlZeroColumnOptionsBase
         {
             public readonly bool UseCdf;
 
             public LogMeanVarianceColumnOptions(string outputColumnName, string inputColumnName = null,
-                long maximumExampleCount = Defaults.MaximumExampleCount, bool useCdf = Defaults.LogMeanVarCdf)
-                : base(outputColumnName, inputColumnName ?? outputColumnName, maximumExampleCount)
+                long maximumExampleCount = Defaults.MaximumExampleCount, bool useCdf = Defaults.LogMeanVarCdf, bool fixZero = Defaults.EnsureZeroUntouched)
+                : base(outputColumnName, inputColumnName ?? outputColumnName, maximumExampleCount, fixZero)
             {
                 UseCdf = useCdf;
             }

src/Microsoft.ML.Transforms/NormalizerCatalog.cs

@@ -141,7 +141,7 @@ public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatal
             long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
             bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf)
         {
-            var columnOptions = new NormalizingEstimator.LogMeanVarianceColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, useCdf);
+            var columnOptions = new NormalizingEstimator.LogMeanVarianceColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, useCdf, false);
             return new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions);
         }
 
@@ -159,7 +159,54 @@ public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatal
             bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf) =>
             new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog),
                 columns.Select(column =>
-                    new NormalizingEstimator.LogMeanVarianceColumnOptions(column.OutputColumnName, column.InputColumnName, maximumExampleCount, useCdf)).ToArray());
+                    new NormalizingEstimator.LogMeanVarianceColumnOptions(column.OutputColumnName, column.InputColumnName, maximumExampleCount, useCdf, false)).ToArray());
+
+        /// <summary>
+        /// Create a <see cref="NormalizingEstimator"/>, which normalizes based on the computed mean and variance of the logarithm of the data.
+        /// </summary>
+        /// <param name="catalog">The transform catalog</param>
+        /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
+        ///                                The data type on this column is the same as the input column.</param>
+        /// <param name="fixZero">Whether to map zero to zero, preserving sparsity.</param>
+        /// <param name="inputColumnName">Name of the column to transform. If set to <see langword="null"/>, the value of the <paramref name="outputColumnName"/> will be used as source.
+        ///                               The data type on this column should be <see cref="System.Single"/>, <see cref="System.Double"/> or a known-sized vector of those types.</param>
+        /// <param name="maximumExampleCount">Maximum number of examples used to train the normalizer.</param>
+        /// <param name="useCdf">Whether to use CDF as the output.</param>
+        /// <example>
+        /// <format type="text/markdown">
+        /// <![CDATA[
+        /// [!code-csharp[NormalizeLogMeanVariance](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/NormalizeLogMeanVarianceFixZero.cs)]
+        /// ]]>
+        /// </format>
+        /// </example>
+        public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatalog catalog,
+            string outputColumnName,
+            bool fixZero,
+            string inputColumnName = null,
+            long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
+            bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf)
+        {
+            var columnOptions = new NormalizingEstimator.LogMeanVarianceColumnOptions(outputColumnName, inputColumnName, maximumExampleCount, useCdf, fixZero);
+            return new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog), columnOptions);
+        }
+
+        /// <summary>
+        /// Create a <see cref="NormalizingEstimator"/>, which normalizes based on the computed mean and variance of the logarithm of the data.
+        /// </summary>
+        /// <param name="catalog">The transform catalog</param>
+        /// <param name="columns">The pairs of input and output columns.
+        ///             The input columns must be of data type <see cref="System.Single"/>, <see cref="System.Double"/> or a known-sized vector of those types.
+        ///             The data type for the output column will be the same as the associated input column.</param>
+        /// <param name="fixZero">Whether to map zero to zero, preserving sparsity.</param>
+        /// <param name="maximumExampleCount">Maximum number of examples used to train the normalizer.</param>
+        /// <param name="useCdf">Whether to use CDF as the output.</param>
+        public static NormalizingEstimator NormalizeLogMeanVariance(this TransformsCatalog catalog, InputOutputColumnPair[] columns,
+            bool fixZero,
+            long maximumExampleCount = NormalizingEstimator.Defaults.MaximumExampleCount,
+            bool useCdf = NormalizingEstimator.Defaults.LogMeanVarCdf) =>
+            new NormalizingEstimator(CatalogUtils.GetEnvironment(catalog),
+                columns.Select(column =>
+                    new NormalizingEstimator.LogMeanVarianceColumnOptions(column.OutputColumnName, column.InputColumnName, maximumExampleCount, useCdf, fixZero)).ToArray());
 
         /// <summary>
         /// Create a <see cref="NormalizingEstimator"/>, which normalizes by assigning the data into bins with equal density.