Skip to content

Commit 11df22b

Browse files
jchen351jchen351
authored
Reenabling Nuget Cuda Packaging Pipeline (microsoft#20688)
### Description <!-- Describe your changes. --> ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
1 parent fefae0c commit 11df22b

File tree

5 files changed

+543
-343
lines changed

5 files changed

+543
-343
lines changed

‎tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml‎

Lines changed: 27 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -116,47 +116,11 @@ variables:
116116
value: $(Agent.TempDirectory)\v12.2
117117

118118
stages:
119-
- stage: Setup
120-
jobs:
121-
- job: Set_Variables
122-
pool:
123-
vmImage: ubuntu-latest
124-
steps:
125-
- checkout: none
126-
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
127-
displayName: 'Clean Agent Directories'
128-
condition: always()
129-
- bash: |
130-
# Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
131-
set +x
132-
if [[ "${{ parameters.IsReleaseBuild }}" = True && "${{ parameters.PreReleaseVersionSuffixString }}" != "none" ]]; then
133-
if [[ "${{ parameters.PreReleaseVersionSuffixNumber }}" -eq 0 ]]; then
134-
echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]-${{ parameters.PreReleaseVersionSuffixString }}"
135-
else
136-
echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]-${{ parameters.PreReleaseVersionSuffixString }}.${{ parameters.PreReleaseVersionSuffixNumber }}"
137-
fi
138-
else
139-
echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]"
140-
fi
141-
name: Set_Release_Version_Suffix
142-
- script: |
143-
# Extracting hours and minutes
144-
date=$(date +'%Y%m%d')
145-
# Set the hhmm value as a pipeline variable
146-
echo "##vso[task.setvariable variable=BuildDate;isOutput=true]$date"
147-
displayName: 'Set Start Date as Variable'
148-
name: Set_Build_Date
149-
150-
- script: |
151-
# Extracting hours and minutes
152-
hhmm=$(date +'%H%M')
153-
# Set the hhmm value as a pipeline variable
154-
echo "##vso[task.setvariable variable=BuildTime;isOutput=true]$hhmm"
155-
displayName: 'Set Start Time as Variable'
156-
name: Set_Build_Time
157-
- template: templates/component-governance-component-detection-steps.yml
158-
parameters :
159-
condition : 'succeeded'
119+
- template: stages/set_packaging_variables_stage.yml
120+
parameters:
121+
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
122+
PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
123+
PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
160124

161125
- stage: Debug
162126
dependsOn: Setup
@@ -237,13 +201,28 @@ stages:
237201
AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos'
238202
BuildVariant: 'default'
239203

240-
- template: stages/nuget-linux-cuda-packaging-stage.yml
204+
- template: stages/java-cuda-packaging-stage.yml
241205
parameters:
242206
CudaVersion: ${{ parameters.CudaVersion }}
243-
docker_base_image: ${{ variables.docker_base_image }}
244-
linux_trt_version: ${{ variables.linux_trt_version }}
245-
buildJava: true
246-
buildNodejs: true
207+
SpecificArtifact: ${{ parameters.SpecificArtifact }}
208+
BuildId: ${{ parameters.BuildId }}
209+
210+
- template: stages/nuget-combine-cuda-stage.yml
211+
parameters:
212+
DoCompliance: ${{ parameters.DoCompliance }}
213+
CudaVersion: ${{ parameters.CudaVersion }}
214+
docker_base_image: ${{ variables.docker_base_image }}
215+
linux_trt_version: ${{ variables.linux_trt_version }}
216+
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
217+
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
218+
win_trt_home: ${{ variables.win_trt_home }}
219+
win_cuda_home: ${{ variables.win_cuda_home }}
220+
DoEsrp: ${{ parameters.DoEsrp }}
221+
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
222+
buildJava: true
223+
buildNodejs: true
224+
SpecificArtifact: ${{ parameters.SpecificArtifact }}
225+
BuildId: ${{ parameters.BuildId }}
247226

248227
# ROCm
249228
- stage: Linux_C_API_Packaging_ROCm_x64
@@ -311,29 +290,6 @@ stages:
311290
condition: 'succeeded'
312291
- template: templates/clean-agent-build-directory-step.yml
313292

314-
- template: stages/java-cuda-packaging-stage.yml
315-
parameters:
316-
CudaVersion: ${{ parameters.CudaVersion }}
317-
SpecificArtifact: ${{ parameters.SpecificArtifact }}
318-
BuildId: ${{ parameters.BuildId }}
319-
320-
- template: stages/nuget-win-cuda-packaging-stage.yml
321-
parameters:
322-
CudaVersion: ${{ parameters.CudaVersion }}
323-
SpecificArtifact: ${{ parameters.SpecificArtifact }}
324-
BuildId: ${{ parameters.BuildId }}
325-
DoEsrp: ${{ parameters.DoEsrp }}
326-
DoCompliance: ${{ parameters.DoCompliance }}
327-
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
328-
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
329-
win_trt_home: ${{ variables.win_trt_home }}
330-
win_cuda_home: ${{ variables.win_cuda_home }}
331-
332-
- template: stages/nuget-combine-cuda-stage.yml
333-
parameters:
334-
DoEsrp: ${{ parameters.DoEsrp }}
335-
DoCompliance: ${{ parameters.DoCompliance }}
336-
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
337293

338294
- stage: NuGet_Packaging_ROCm
339295
dependsOn:
@@ -556,50 +512,6 @@ stages:
556512
displayName: 'Clean Agent Directories'
557513
condition: always()
558514

559-
- template: nuget/templates/test_win.yml
560-
parameters:
561-
AgentPool: 'onnxruntime-Win2022-GPU-A10'
562-
NugetPackageName: 'Microsoft.ML.OnnxRuntime.Gpu'
563-
ArtifactSuffix: 'GPU'
564-
StageSuffix: 'GPU'
565-
Skipx86Tests: 'true'
566-
CudaVersion: ${{ parameters.CudaVersion }}
567-
SpecificArtifact: ${{ parameters.SpecificArtifact }}
568-
BuildId: ${{ parameters.BuildId }}
569-
570-
- template: nuget/templates/test_win.yml
571-
parameters:
572-
AgentPool: 'onnxruntime-Win2022-GPU-A10'
573-
NugetPackageName: 'Microsoft.ML.OnnxRuntime.Gpu.Windows'
574-
ArtifactSuffix: 'GPU'
575-
StageSuffix: 'GPU'
576-
MoreSuffix: '_Windows'
577-
Skipx86Tests: 'true'
578-
CudaVersion: ${{ parameters.CudaVersion }}
579-
SpecificArtifact: ${{ parameters.SpecificArtifact }}
580-
BuildId: ${{ parameters.BuildId }}
581-
582-
- template: nuget/templates/test_linux.yml
583-
parameters:
584-
AgentPool: Onnxruntime-Linux-GPU-A10
585-
ArtifactSuffix: 'GPU'
586-
StageSuffix: 'GPU'
587-
NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'
588-
CudaVersion: ${{ parameters.CudaVersion }}
589-
SpecificArtifact: ${{ parameters.specificArtifact }}
590-
BuildId: ${{ parameters.BuildId }}
591-
592-
- template: nuget/templates/test_linux.yml
593-
parameters:
594-
AgentPool: Onnxruntime-Linux-GPU-A10
595-
ArtifactSuffix: 'GPU'
596-
StageSuffix: 'GPU'
597-
MoreSuffix: '_Linux'
598-
NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu.Linux'
599-
CudaVersion: ${{ parameters.CudaVersion }}
600-
SpecificArtifact: ${{ parameters.specificArtifact }}
601-
BuildId: ${{ parameters.BuildId }}
602-
603515
- template: nuget/templates/test_linux.yml
604516
parameters:
605517
AgentPool: AMD-GPU
@@ -813,13 +725,13 @@ stages:
813725
inputs:
814726
artifactName: 'drop-nuget-qnn-x64'
815727
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64'
816-
728+
817729
- task: DownloadPipelineArtifact@0
818730
displayName: 'Download Pipeline Artifact - QNN NuGet arm64'
819731
inputs:
820732
artifactName: 'drop-nuget-qnn-arm64'
821733
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64'
822-
734+
823735
- task: PowerShell@2
824736
displayName: 'Bundle NuGet'
825737
inputs:

‎tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
parameters:
2+
- name: RunOnnxRuntimeTests
3+
displayName: Run Tests?
4+
type: boolean
5+
default: true
6+
7+
- name: UseIncreasedTimeoutForTests
8+
displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
9+
type: boolean
10+
default: false
11+
12+
- name: DoCompliance
13+
displayName: Run Compliance Tasks?
14+
type: boolean
15+
default: true
16+
17+
- name: DoEsrp
18+
displayName: Run code sign tasks? Must be true if you are doing an ONNX Runtime release
19+
type: boolean
20+
default: true
21+
22+
- name: IsReleaseBuild
23+
displayName: Is a release build? Set it to true if you are doing an ONNX Runtime release.
24+
type: boolean
25+
default: false
26+
27+
- name: PreReleaseVersionSuffixString
28+
displayName: Suffix added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the type of pre-release package.
29+
type: string
30+
values:
31+
- alpha
32+
- beta
33+
- rc
34+
- none
35+
default: none
36+
37+
- name: PreReleaseVersionSuffixNumber
38+
displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
39+
type: number
40+
default: 0
41+
42+
# these 2 parameters are used for debugging.
43+
- name: SpecificArtifact
44+
displayName: Use Specific Artifact (Debugging only)
45+
type: boolean
46+
default: false
47+
48+
- name: BuildId
49+
displayName: Pipeline BuildId, you could find it in the URL
50+
type: string
51+
default: '0'
52+
53+
- name: CudaVersion
54+
displayName: CUDA version
55+
type: string
56+
default: '12.2'
57+
values:
58+
- 11.8
59+
- 12.2
60+
61+
variables:
62+
- name: ReleaseVersionSuffix
63+
value: ''
64+
- name: docker_base_image
65+
${{ if eq(parameters.CudaVersion, '11.8') }}:
66+
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
67+
${{ if eq(parameters.CudaVersion, '12.2') }}:
68+
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
69+
- name: linux_trt_version
70+
${{ if eq(parameters.CudaVersion, '11.8') }}:
71+
value: 10.0.1.6-1.cuda11.8
72+
${{ if eq(parameters.CudaVersion, '12.2') }}:
73+
value: 10.0.1.6-1.cuda12.4
74+
- name: win_trt_home
75+
${{ if eq(parameters.CudaVersion, '11.8') }}:
76+
value: $(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8
77+
${{ if eq(parameters.CudaVersion, '12.2') }}:
78+
value: $(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-12.4
79+
- name: win_cuda_home
80+
${{ if eq(parameters.CudaVersion, '11.8') }}:
81+
value: $(Agent.TempDirectory)\v11.8
82+
${{ if eq(parameters.CudaVersion, '12.2') }}:
83+
value: $(Agent.TempDirectory)\v12.2
84+
resources:
85+
repositories:
86+
- repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
87+
type: github
88+
endpoint: ort-examples
89+
name: microsoft/onnxruntime-inference-examples
90+
- repository: manylinux
91+
type: Github
92+
endpoint: Microsoft
93+
name: pypa/manylinux
94+
ref: 5eda9aded5462201e6310105728d33016e637ea7
95+
96+
stages:
97+
# Set ReleaseVersionSuffix
98+
- template: stages/set_packaging_variables_stage.yml
99+
parameters:
100+
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
101+
PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
102+
PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
103+
104+
# this is needed for certain artifacts to be published
105+
- stage: Linux_C_API_Packaging_CPU_x64
106+
dependsOn: [ ]
107+
jobs:
108+
- template: templates/c-api-linux-cpu.yml
109+
parameters:
110+
BaseImage: 'registry.access.redhat.com/ubi8/ubi'
111+
OnnxruntimeArch: 'x64'
112+
OnnxruntimeNodejsBindingArch: 'x64'
113+
PoolName: 'onnxruntime-Ubuntu2204-AMD-CPU'
114+
PackageJava: false
115+
PackageNodeJS: false
116+
117+
# Nuget Packaging
118+
- template: stages/nuget-combine-cuda-stage.yml
119+
parameters:
120+
DoCompliance: ${{ parameters.DoCompliance }}
121+
CudaVersion: ${{ parameters.CudaVersion }}
122+
docker_base_image: ${{ variables.docker_base_image }}
123+
linux_trt_version: ${{ variables.linux_trt_version }}
124+
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
125+
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
126+
win_trt_home: ${{ variables.win_trt_home }}
127+
win_cuda_home: ${{ variables.win_cuda_home }}
128+
DoEsrp: ${{ parameters.DoEsrp }}
129+
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
130+
buildJava: false
131+
buildNodejs: false
132+
SpecificArtifact: ${{ parameters.SpecificArtifact }}
133+
BuildId: ${{ parameters.BuildId }}

0 commit comments

Comments
 (0)