Publishing nuget packages to myget feed. (#155)

* Publishing nuget packages to myget feed.

Also - set the symbols expiration days default based on feedback from the .NET core-eng team.

Fixes #11

* Shorten nuget push timeout to match corefx and coreclr.

Author: eerhardt

README.md

@@ -34,6 +34,10 @@ Install-Package Microsoft.ML
 
 Or alternatively you can add the Microsoft.ML package from within Visual Studio's NuGet package manager or via [Paket](https://github.com/fsprojects/Paket).
 
+Daily NuGet builds of the project are also available in our MyGet feed:
+
+> [https://dotnet.myget.org/F/dotnet-core/api/v3/index.json](https://dotnet.myget.org/F/dotnet-core/api/v3/index.json)
+
 ## Building
 
 To build ML.NET from source please visit our [developers guide](Documentation/project-docs/developer-guide.md).

build/publish.proj

@@ -1,14 +1,50 @@
-<Project DefaultTargets="Publish" >
+<Project>
   <Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props" />
 
+  <UsingTask TaskName="ExecWithRetriesForNuGetPush" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll" />
+
   <PropertyGroup>
     <PublishSymbolsPackage>Microsoft.SymbolUploader.Build.Task</PublishSymbolsPackage>
     <EnablePublishSymbols Condition="'$(EnablePublishSymbols)'==''" >true</EnablePublishSymbols>
+    <NuGetPushTimeoutSeconds Condition="'$(NuGetPushTimeoutSeconds)' == ''">600</NuGetPushTimeoutSeconds>
   </PropertyGroup>
 
   <Import Project="$(PackagesDir)\$(PublishSymbolsPackage.ToLower())\$(PublishSymbolsPackageVersion)\build\PublishSymbols.targets" />
 
-  <Target Name="Publish" Condition="'$(EnablePublishSymbols)'=='true'" DependsOnTargets="SetupPublishSymbols">
+  <Target Name="PublishPackages">
+    <Error Condition="'$(NuGetFeedUrl)' == ''" Text="Missing required property NuGetFeedUrl" />
+    <Error Condition="'$(NuGetApiKey)' == ''" Text="Missing required property NuGetApiKey" />
+
+    <ItemGroup>
+      <NuGetPackages Include="$(PackageOutputPath)**\*.nupkg"
+                     Exclude="$(PackageOutputPath)**\*.symbols.*nupkg" />
+
+      <!--
+      IgnorableErrorMessages applies to the "ExectWithRetriesForNuGetPush" task.
+      There's a very special failure scenario that we want to ignore.  That scenario is
+      when NuGet hits a timeout on one "push" attempt, and then gets a "Forbidden" response
+      because the package "already exists" on the next response.  This indicates that the
+      timeout occurred, but the push was actually successful.
+      -->
+      <IgnorableErrorMessages Include="Overwriting existing packages is forbidden according to the package retention settings for this feed.">
+        <ConditionalErrorMessage>Pushing took too long</ConditionalErrorMessage>
+      </IgnorableErrorMessages>
+    </ItemGroup>
+    
+    <Message Text="Pushing ML.NET packages to $(NuGetFeedUrl)" />
+    
+    <PropertyGroup>
+      <DotnetToolCommand>$(ToolsDir)dotnetcli/dotnet</DotnetToolCommand>
+      <NuGetPushCommand>$(DotnetToolCommand) nuget push --source $(NuGetFeedUrl) --api-key $(NuGetApiKey) --timeout $(NuGetPushTimeoutSeconds)</NuGetPushCommand>
+    </PropertyGroup>
+
+    <ExecWithRetriesForNuGetPush Command="$(NuGetPushCommand) %(NuGetPackages.Identity)"
+                                 IgnoredErrorMessagesWithConditional="@(IgnorableErrorMessages)" />
+  </Target>
+  
+  <Target Name="PublishSymbolPackages"
+          Condition="'$(EnablePublishSymbols)'=='true'"
+          DependsOnTargets="SetupPublishSymbols">
     <Message Text="Attempting to Publish Symbols" />
     <Error Condition="!Exists('$(PackageOutputPath)')" Text="'PackageOutputPath' folder '$(PackageOutputPath)' does not exist."/>
     <Error Condition="'$(SymbolServerPath)'==''" Text="Missing property SymbolServerPath" />
@@ -18,7 +54,7 @@
 
   <Target Name="SetupPublishSymbols">
     <PropertyGroup>
-      <SymbolExpirationInDays Condition="'$(SymbolExpirationInDays)'=='' and '$(SymbolExpirationDate)'==''">1</SymbolExpirationInDays>
+      <SymbolExpirationInDays Condition="'$(SymbolExpirationInDays)'=='' and '$(SymbolExpirationDate)'==''">180</SymbolExpirationInDays>
       <ConvertPortablePdbsToWindowsPdbs>true</ConvertPortablePdbsToWindowsPdbs>
     </PropertyGroup>
 

build/vsts-ci.yml

@@ -6,6 +6,9 @@ phases:
     _PREVIEW_VSTS_DOCKER_IMAGE: microsoft/dotnet-buildtools-prereqs:centos-7-d485f41-20173404063424
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
   queue:
     name: DotNetCore-Test
     demands:
@@ -27,6 +30,9 @@ phases:
   variables:
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
   queue:
     name: DotNetCore-Build
     demands:
@@ -48,6 +54,9 @@ phases:
   variables:
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
     _SignType: real
     _UseEsrpSigning: true
     _TeamName: DotNetCore
@@ -96,12 +105,15 @@ phases:
   variables:
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
     _SignType: real
     _UseEsrpSigning: true
     _TeamName: DotNetCore
+    _NuGetFeedUrl: https://dotnet.myget.org/F/dotnet-core/api/v2/package
     _SymwebSymbolServerPath: https://microsoft.artifacts.visualstudio.com/DefaultCollection
     _MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
-    _SymbolExpirationInDays: 30
   queue:
     name: DotNetCore-Build
     demands: 
@@ -145,26 +157,25 @@ phases:
       msbuildVersion: 15.0
     continueOnError: false
 
-  - task: NuGetCommand@2
-    displayName: Publish Packages to VSTS Feed
+  - task: MSBuild@1
+    displayName: Publish Packages to MyGet Feed
     inputs:
-      command: push
-      packagesToPush: $(Build.SourcesDirectory)/bin/packages/**/*.nupkg;!$(Build.SourcesDirectory)/bin/packages/**/*.symbols.nupkg
-      nuGetFeedType: internal
-      feedPublish: MachineLearning
+      solution: build/publish.proj
+      msbuildArguments: /t:PublishPackages /p:NuGetFeedUrl=$(_NuGetFeedUrl) /p:NuGetApiKey=$(dotnet-myget-org-api-key)
+      msbuildVersion: 15.0
 
   - task: MSBuild@1
     displayName: Publish Symbols to SymWeb Symbol Server
     inputs:
       solution: build/publish.proj
-      msbuildArguments: /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT) /p:SymbolExpirationInDays=$(_SymbolExpirationInDays)
+      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT)
       msbuildVersion: 15.0
     continueOnError: true
 
   - task: MSBuild@1
     displayName: Publish Symbols to Msdl Symbol Server
     inputs:
       solution: build/publish.proj
-      msbuildArguments: /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT) /p:SymbolExpirationInDays=$(_SymbolExpirationInDays)
+      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT)
       msbuildVersion: 15.0
     continueOnError: true