Add aka.ms aliases for files downloaded from Azure blobs (dotnet#4523)

yaeldMS · codemzs · commit 78c406cf1f4a · 2019-12-26T10:33:53.000-08:00
* update tlcresources links

* Update more links

* update more links

* Undo changes in Samples project
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault.cs
@@ -242,10 +242,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)
 
             //SINGLE SMALL FLOWERS IMAGESET (200 files)
             string fileName = "flower_photos_small_set.zip";
-            string url = $"https://mlnetfilestorage.file.core.windows.net/" +
-                $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
-                $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
-                $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
+            string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/LearningRateSchedulingCifarResnetTransferLearning.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/LearningRateSchedulingCifarResnetTransferLearning.cs
@@ -271,7 +271,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)
             // get a set of images to teach the network about the new classes
             // CIFAR dataset ( 50000 train images and 10000 test images )
             string fileName = "cifar10.zip";
-            string url = $"https://tlcresources.blob.core.windows.net/" +
+            string url = $"https://aka.ms/mlnet-resources/" +
                 "datasets/cifar10.zip";
 
             Download(url, imagesDownloadFolder, fileName);
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs
@@ -230,10 +230,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)
 
             //SINGLE SMALL FLOWERS IMAGESET (200 files)
             string fileName = "flower_photos_small_set.zip";
-            string url = $"https://mlnetfilestorage.file.core.windows.net/" +
-                $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
-                $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
-                $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
+            string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs
@@ -251,10 +251,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)
 
             //SINGLE SMALL FLOWERS IMAGESET (200 files)
             string fileName = "flower_photos_small_set.zip";
-            string url = $"https://mlnetfilestorage.file.core.windows.net/" +
-                $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
-                $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
-                $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
+            string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
diff --git a/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs b/test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs
@@ -26,7 +26,7 @@ internal static class DatasetUtil
 
         public static IDataView GetUciAdultDataView()
         {
-            if(_uciAdultDataView == null)
+            if (_uciAdultDataView == null)
             {
                 var context = new MLContext();
                 var uciAdultDataFile = DownloadUciAdultDataset();
@@ -69,7 +69,7 @@ private static string DownloadIfNotExists(string baseGitPath, string dataFile)
                             }
                         }
                     }
-                    catch(Exception)
+                    catch (Exception)
                     {
                     }
                 }
@@ -146,10 +146,7 @@ public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder)
         public static string DownloadImageSet(string imagesDownloadFolder)
         {
             string fileName = "flower_photos_tiny_set_for_unit_tests.zip";
-            string url = $"https://mlnetfilestorage.file.core.windows.net/imagesets" +
-                $"/flower_images/flower_photos_tiny_set_for_unit_tests.zip?st=2019" +
-                $"-08-29T00%3A07%3A21Z&se=2030-08-30T00%3A07%3A00Z&sp=rl&sv=2018" +
-                $"-03-28&sr=f&sig=N8HbLziTcT61kstprNLmn%2BDC0JoMrNwo6yRWb3hLLag%3D";
+            string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_tiny_set_for_unit_test.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
diff --git a/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs b/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs
@@ -139,10 +139,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)
 
             //SINGLE SMALL FLOWERS IMAGESET (200 files)
             string fileName = "flower_photos_small_set.zip";
-            string url = $"https://mlnetfilestorage.file.core.windows.net/" +
-                $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
-                $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
-                $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
+            string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip/";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs
@@ -1316,12 +1316,12 @@ internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedV
             }
             return isReuse;
         }
-        
+
         internal (string, string, string, bool) getInitialParameters(ImageClassificationTrainer.Architecture arch, string finalImagesFolderName)
         {
-            string trainSetBottleneckCachedValuesFileName = "TrainsetCached_" + finalImagesFolderName + "_" + (int) arch;
-            string validationSetBottleneckCachedValuesFileName = "validationsetCached_" + finalImagesFolderName + "_" + (int) arch;
-            string workspacePath = Path.Combine(TensorFlowScenariosTestsFixture.parentWorkspacePath, finalImagesFolderName + "_" + (int) arch);
+            string trainSetBottleneckCachedValuesFileName = "TrainsetCached_" + finalImagesFolderName + "_" + (int)arch;
+            string validationSetBottleneckCachedValuesFileName = "validationsetCached_" + finalImagesFolderName + "_" + (int)arch;
+            string workspacePath = Path.Combine(TensorFlowScenariosTestsFixture.parentWorkspacePath, finalImagesFolderName + "_" + (int)arch);
             bool isReuse = ShouldReuse(workspacePath, trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName);
             return (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName, workspacePath, isReuse);
         }
@@ -1368,9 +1368,9 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur
                     .Transform(testDataset);
 
             // Check if the bottleneck cached values already exist
-            var (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName, 
-                workspacePath, isReuse)  = getInitialParameters(arch, finalImagesFolderName);
-            
+            var (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName,
+                workspacePath, isReuse) = getInitialParameters(arch, finalImagesFolderName);
+
             var options = new ImageClassificationTrainer.Options()
             {
                 FeatureColumnName = "Image",
@@ -1424,7 +1424,7 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur
 
             string[] directories = Directory.GetDirectories(fullImagesetFolderPath);
             string[] labels = new string[directories.Length];
-            for(int j = 0; j < labels.Length; j++)
+            for (int j = 0; j < labels.Length; j++)
             {
                 var dir = new DirectoryInfo(directories[j]);
                 labels[j] = dir.Name;
@@ -1813,21 +1813,21 @@ public static string DownloadImageSet(string imagesDownloadFolder)
         {
             string fileName = "flower_photos_tiny_set_for_unit_tests.zip";
             string filenameAlias = "FPTSUT"; // FPTSUT = flower photos tiny set for unit tests
-            string url = "https://aka.ms/mlnet-resources/flower_photos_tiny_set_for_unit_test.zip";
+            string url = "https://aka.ms/mlnet-resources/datasets/flower_photos_tiny_set_for_unit_test.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
             // Sometimes tests fail because the path is too long. So rename the dataset folder to a shorter directory.
-            if(!Directory.Exists(Path.Combine(imagesDownloadFolder, filenameAlias)))
-                Directory.Move(Path.Combine(imagesDownloadFolder, Path.GetFileNameWithoutExtension(fileName)), Path.Combine(imagesDownloadFolder,"FPTSUT"));
+            if (!Directory.Exists(Path.Combine(imagesDownloadFolder, filenameAlias)))
+                Directory.Move(Path.Combine(imagesDownloadFolder, Path.GetFileNameWithoutExtension(fileName)), Path.Combine(imagesDownloadFolder, "FPTSUT"));
             return filenameAlias;
         }
 
         public static string DownloadBadImageSet(string imagesDownloadFolder)
         {
             string fileName = "CatsVsDogs_tiny_for_unit_tests.zip";
-            string url = $"https://tlcresources.blob.core.windows.net/datasets/" +
-                $"CatsVsDogs_tiny_for_unit_tests.zip"; 
+            string url = $"https://aka.ms/mlnet-resources/datasets/" +
+                $"CatsVsDogs_tiny_for_unit_tests.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);

Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ internal static class DatasetUtil`
`26`	`26`
`27`	`27`	`public static IDataView GetUciAdultDataView()`
`28`	`28`	`{`
`29`		`- if(_uciAdultDataView == null)`
	`29`	`+ if (_uciAdultDataView == null)`
`30`	`30`	`{`
`31`	`31`	`var context = new MLContext();`
`32`	`32`	`var uciAdultDataFile = DownloadUciAdultDataset();`
`@@ -69,7 +69,7 @@ private static string DownloadIfNotExists(string baseGitPath, string dataFile)`
`69`	`69`	`}`
`70`	`70`	`}`
`71`	`71`	`}`
`72`		`- catch(Exception)`
	`72`	`+ catch (Exception)`
`73`	`73`	`{`
`74`	`74`	`}`
`75`	`75`	`}`
`@@ -146,10 +146,7 @@ public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder)`
`146`	`146`	`public static string DownloadImageSet(string imagesDownloadFolder)`
`147`	`147`	`{`
`148`	`148`	`string fileName = "flower_photos_tiny_set_for_unit_tests.zip";`
`149`		`- string url = $"https://mlnetfilestorage.file.core.windows.net/imagesets" +`
`150`		`- $"/flower_images/flower_photos_tiny_set_for_unit_tests.zip?st=2019" +`
`151`		`- $"-08-29T00%3A07%3A21Z&se=2030-08-30T00%3A07%3A00Z&sp=rl&sv=2018" +`
`152`		`- $"-03-28&sr=f&sig=N8HbLziTcT61kstprNLmn%2BDC0JoMrNwo6yRWb3hLLag%3D";`
	`149`	`+ string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_tiny_set_for_unit_test.zip";`
`153`	`150`
`154`	`151`	`Download(url, imagesDownloadFolder, fileName);`
`155`	`152`	`UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);`