Added functionality to load images as VBuffer<byte> in ImageLoader. If no DataViewType options are provided it defaults to loading images as ImageDataViewType. Made LoadImages a part of the sample in ResnetV2101TransferLearningTrainTestSplit.cs. Addressed some of the comments from Zeeshan and Yael. Added a unit test for testing the API. Added TargetFrameworks to get cross platform functionality for System.IO.Stream.Read(Span<Byte>) which doesn't work for netstandard2.0.

Author: harshithapv

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs

@@ -56,18 +56,22 @@ public static void Example()
 
                 IDataView trainDataset = trainTestData.TrainSet;
                 IDataView testDataset = trainTestData.TestSet;
-
-                var pipeline = mlContext.Model.ImageClassification(
-                    "ImageVBuf", "Label",
-                    // Just by changing/selecting InceptionV3 here instead of 
-                    // ResnetV2101 you can try a different architecture/pre-trained 
-                    // model. 
-                    arch: ImageClassificationEstimator.Architecture.ResnetV2101, 
-                    epoch: 50,
-                    batchSize: 10,
-                    learningRate: 0.01f,
-                    metricsCallback: (metrics) => Console.WriteLine(metrics),
-                    validationSet: testDataset);
+                var validationSet = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, new VectorDataViewType(NumberDataViewType.Byte), "ImagePath")
+                    .Fit(testDataset)
+                    .Transform(testDataset);
+
+                var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, new VectorDataViewType(NumberDataViewType.Byte), "ImagePath")
+                    .Append(mlContext.Model.ImageClassification(
+                        "Image", "Label",
+                        // Just by changing/selecting InceptionV3 here instead of 
+                        // ResnetV2101 you can try a different architecture/pre-trained 
+                        // model. 
+                        arch: ImageClassificationEstimator.Architecture.ResnetV2101,
+                        epoch: 50,
+                        batchSize: 10,
+                        learningRate: 0.01f,
+                        metricsCallback: (metrics) => Console.WriteLine(metrics),
+                        validationSet: validationSet));
 
 
                 Console.WriteLine("*** Training the image classification model with " +
@@ -82,7 +86,7 @@ public static void Example()
                 watch.Stop();
                 long elapsedMs = watch.ElapsedMilliseconds;
 
-                Console.WriteLine("Training with transfer learning took: " + 
+                Console.WriteLine("Training with transfer learning took: " +
                     (elapsedMs / 1000).ToString() + " seconds");
 
                 mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema,
@@ -99,7 +103,7 @@ public static void Example()
                 loadedModel.GetOutputSchema(schema)["Label"].GetKeyValues(ref keys);
 
                 watch = System.Diagnostics.Stopwatch.StartNew();
-                TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel, 
+                TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel,
                     keys.DenseValues().ToArray());
 
                 watch.Stop();
@@ -133,8 +137,7 @@ private static void TrySinglePrediction(string imagesForPredictions,
 
             ImageData imageToPredict = new ImageData
             {
-                ImagePath = testImages.First().ImagePath,
-                ImageVBuf = imgData
+                ImagePath = testImages.First().ImagePath
             };
 
             var prediction = predictionEngine.Predict(imageToPredict);
@@ -168,45 +171,15 @@ private static void EvaluateModel(MLContext mlContext,
             Console.WriteLine("Predicting and Evaluation took: " +
                 (elapsed2Ms / 1000).ToString() + " seconds");
         }
-
-        public static int LoadDataIntoBuffer(string path, ref VBuffer<Byte> imgData)
-        {
-            int count = -1;
-            // bufferSize == 1 used to avoid unnecessary buffer in FileStream
-            using (FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1))
-            {
-                long fileLength = fs.Length;
-                if (fileLength > int.MaxValue)
-                    throw new IOException($"File {path} too big to open.");
-                else if (fileLength == 0)
-                {
-                    byte[] _imageBuffer;
-
-                    // Some file systems (e.g. procfs on Linux) return 0 for length even when there's content.
-                    // Thus we need to assume 0 doesn't mean empty.
-                    _imageBuffer = File.ReadAllBytes(path);
-                    count = _imageBuffer.Length;
-                    Console.WriteLine("File length is zero");
-                }
-
-                count = (int)fileLength;
-                var editor = VBufferEditor.Create(ref imgData, count);
-                fs.Read(editor.Values);                
-                imgData = editor.Commit();
-
-                return count;
-            }
-        }
-
+        
         public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder,
             bool useFolderNameasLabel = true)
         {
             var files = Directory.GetFiles(folder, "*",
                 searchOption: SearchOption.AllDirectories);
-            VBuffer<Byte> imgData = new VBuffer<byte>();
             foreach (var file in files)
             {
-                if (Path.GetExtension(file) != ".JPEG" &&  Path.GetExtension(file) != ".jpg")
+                if (Path.GetExtension(file) != ".JPEG" && Path.GetExtension(file) != ".jpg")
                     continue;
 
                 var label = Path.GetFileName(file);
@@ -223,15 +196,11 @@ public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder,
                         }
                     }
                 }
-
-                // Get the buffer of bytes
-                int imgSize = LoadDataIntoBuffer(file, ref imgData);
-
+                
                 yield return new ImageData()
                 {
                     ImagePath = file,
-                    Label = label,
-                    ImageVBuf = imgData
+                    Label = label
                 };
 
             }
@@ -328,8 +297,6 @@ public class ImageData
 
             [LoadColumn(1)]
             public string Label;
-
-            public VBuffer<byte> ImageVBuf;
         }
 
         public class ImagePrediction
@@ -341,5 +308,4 @@ public class ImagePrediction
             public UInt32 PredictedLabel;
         }
     }
-}
-
+}

docs/samples/Microsoft.ML.Samples/Program.cs

@@ -10,7 +10,6 @@ public static class Program
 
         internal static void RunAll()
         {
-            
             int samples = 0;
             foreach (var type in Assembly.GetExecutingAssembly().GetTypes())
             {
@@ -24,8 +23,7 @@ internal static void RunAll()
                 }
             }
 
-            Console.WriteLine("Number of samples that ran without any exception: " + samples);           
-
+            Console.WriteLine("Number of samples that ran without any exception: " + samples);
         }
     }
 }

src/Microsoft.ML.Core/Data/IEstimator.cs

@@ -64,7 +64,7 @@ internal Column(string name, VectorKind vecKind, DataViewType itemType, bool isK
                 Contracts.CheckNonEmpty(name, nameof(name));
                 Contracts.CheckValueOrNull(annotations);
                 Contracts.CheckParam(!(itemType is KeyDataViewType), nameof(itemType), "Item type cannot be a key");
-                Contracts.CheckParam(!(itemType is VectorDataViewType), nameof(itemType), "Item type cannot be a vector");
+                //Contracts.CheckParam(!(itemType is VectorDataViewType), nameof(itemType), "Item type cannot be a vector");
                 Contracts.CheckParam(!isKey || KeyDataViewType.IsValidDataType(itemType.RawType), nameof(itemType), "The item type must be valid for a key");
 
                 Name = name;

src/Microsoft.ML.Dnn/ImageClassificationTransform.cs

@@ -189,7 +189,7 @@ private void CheckTrainingParameters(ImageClassificationEstimator.Options option
             return (jpegData, resizedImage);
         }
 
-        private static Tensor Encode(VBuffer<byte> buffer)
+        private static Tensor EncodeByteAsString(VBuffer<byte> buffer)
         {
             int length = buffer.Length;
             var size = c_api.TF_StringEncodedSize((UIntPtr)length);
@@ -220,9 +220,9 @@ public ImageProcessor(ImageClassificationTransformer transformer)
                 _imagePreprocessingRunner.AddOutputs(transformer._resizedImageTensorName);
             }
 
-            public Tensor ProcessImage(VBuffer<byte> imgBuf)
+            public Tensor ProcessImage(VBuffer<byte> imageBuffer)
             {
-                var imageTensor = Encode(imgBuf);
+                var imageTensor = EncodeByteAsString(imageBuffer);
                 var processedTensor = _imagePreprocessingRunner.AddInput(imageTensor, 0).Run()[0];
                 imageTensor.Dispose();
                 return processedTensor;
@@ -240,26 +240,26 @@ private void CacheFeaturizedImagesToDisk(IDataView input, string labelColumnName
                     labelColumnName, typeof(uint).ToString(),
                     labelColumn.Type.RawType.ToString());
 
-            var imageBufColumn = input.Schema[imageColumnName];
+            var imageColumn = input.Schema[imageColumnName];
             Runner runner = new Runner(_session);
             runner.AddOutputs(outputTensorName);
 
             using (TextWriter writer = File.CreateText(cacheFilePath))
-            using (var cursor = input.GetRowCursor(input.Schema.Where(c => c.Index == labelColumn.Index || c.Index == imageBufColumn.Index)))
+            using (var cursor = input.GetRowCursor(input.Schema.Where(c => c.Index == labelColumn.Index || c.Index == imageColumn.Index)))
             {
                 var labelGetter = cursor.GetGetter<uint>(labelColumn);
-                var imageBufGetter = cursor.GetGetter<VBuffer<byte>>(imageBufColumn);
+                var imageGetter = cursor.GetGetter<VBuffer<byte>>(imageColumn);
                 UInt32 label = UInt32.MaxValue;
-                VBuffer<byte> imageBuf = default;
+                VBuffer<byte> image = default;
                 runner.AddInput(inputTensorName);
                 ImageClassificationMetrics metrics = new ImageClassificationMetrics();
                 metrics.Bottleneck = new BottleneckMetrics();
                 metrics.Bottleneck.DatasetUsed = dataset;
                 while (cursor.MoveNext())
                 {
                     labelGetter(ref label);
-                    imageBufGetter(ref imageBuf);
-                    var imageTensor = imageProcessor.ProcessImage(imageBuf);
+                    imageGetter(ref image);
+                    var imageTensor = imageProcessor.ProcessImage(image);
                     runner.AddInput(imageTensor, 0);
                     var featurizedImage = runner.Run()[0]; // Reuse memory?
                     writer.WriteLine(label - 1 + "," + string.Join(",", featurizedImage.ToArray<float>()));

src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs

@@ -97,6 +97,53 @@ internal static ImageLoadingEstimator LoadImages(this TransformsCatalog catalog,
             env.CheckValue(columns, nameof(columns));
             return new ImageLoadingEstimator(env, imageFolder, InputOutputColumnPair.ConvertToValueTuples(columns));
         }
+        /// <summary>
+        /// Create a <see cref="ImageLoadingEstimator"/>, which loads the data from the column specified in <paramref name="inputColumnName"/>
+        /// as an image to a new column: <paramref name="outputColumnName"/>.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="outputColumnName">Name of the column resulting from the transformation of <paramref name="inputColumnName"/>.
+        /// This column's data type will be <see cref="VectorDataViewType"/>.</param>
+        /// <param name="inputColumnName">Name of the column with paths to the images to load.
+        /// This estimator operates over text data.</param>
+        /// <param name="imageFolder">Folder where to look for images.</param>
+        /// <param name="type">Image type - VectorDataView type or ImageDataViewType. Defaults to ImageDataViewType if not specified or null.</param>
+        /// <example>
+        /// <format type="text/markdown">
+        /// <![CDATA[
+        ///  [!code-csharp[LoadImages](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/LoadImages.cs)]
+        /// ]]></format>
+        /// </example>
+        public static ImageLoadingEstimator LoadImages(this TransformsCatalog catalog, string outputColumnName, string imageFolder, DataViewType type, string inputColumnName = null)
+           => new ImageLoadingEstimator(CatalogUtils.GetEnvironment(catalog), imageFolder, type, new[] { (outputColumnName, inputColumnName ?? outputColumnName) });
+
+        /// <summary>
+        /// Loads the images from the <see cref="ImageLoadingTransformer.ImageFolder" /> into memory.
+        /// </summary>
+        /// <remarks>
+        /// The image get loaded in memory as a <see cref="VectorDataViewType" /> of bytes.
+        /// Loading is the first step of almost every pipeline that does image processing, and further analysis on images.
+        /// The images to load need to be in the formats supported by <see cref = "VectorDataViewType" /> of bytes.
+        /// For end-to-end image processing pipelines, and scenarios in your applications, see the
+        /// <a href="https://github.com/dotnet/machinelearning-samples/tree/master/samples/csharp/getting-started"> examples in the machinelearning-samples github repository.</a>
+        /// </remarks>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="imageFolder">Folder where to look for images.</param>
+        /// <param name="type">Image type - VectorDataView type or ImageDataViewType. Defaults to ImageDataViewType if not specified or null.</param>
+        /// <param name="columns">Specifies the names of the input columns for the transformation, and their respective output column names.</param>
+        /// <example>
+        /// <format type="text/markdown">
+        /// <![CDATA[
+        ///  [!code-csharp[LoadImagesAsBytes](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/LoadImages.cs)]
+        /// ]]></format>
+        /// </example>
+        [BestFriend]
+        internal static ImageLoadingEstimator LoadImages(this TransformsCatalog catalog, string imageFolder, DataViewType type, params InputOutputColumnPair[] columns)
+        {
+            var env = CatalogUtils.GetEnvironment(catalog);
+            env.CheckValue(columns, nameof(columns));
+            return new ImageLoadingEstimator(env, imageFolder, type, InputOutputColumnPair.ConvertToValueTuples(columns));
+        }
 
         /// <summary>
         /// Create a <see cref="ImagePixelExtractingEstimator"/>, which extracts pixels values from the data specified in column: <paramref name="inputColumnName"/>