Skip to content

Commit 81e3ec5

Browse files
SteveL-MSFTsdwheeler
SteveL-MSFT
authored and
sdwheeler
committed
Enable parallel runs of platyPS per version (#4171)
* make building versions of docs in parallel * fix typo in .gitignore update build.ps1 to build the different versions in parallel * install ThreadJob if not available
1 parent 9c5818b commit 81e3ec5

File tree

2 files changed

+126
-87
lines changed

2 files changed

+126
-87
lines changed

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ _site/
66
Tools/NuGet/
77
_site/
88
maml/
9-
updateablehelp/
9+
updatablehelp/
1010

1111

1212
.openpublishing.build.mdproj

‎build.ps1

Lines changed: 125 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -5,37 +5,7 @@ param(
55

66
# Turning off the progress display, by default
77
$global:ProgressPreference = 'SilentlyContinue'
8-
if($ShowProgress){$ProgressPreference = 'Continue'}
9-
10-
function Get-ContentWithoutHeader {
11-
param(
12-
$path
13-
)
14-
15-
$doc = Get-Content $path -Encoding UTF8
16-
$start = $end = -1
17-
18-
# search the first 30 lines for the Yaml header
19-
# no yaml header in our docset will ever be that long
20-
21-
for ($x = 0; $x -lt 30; $x++) {
22-
if ($doc[$x] -eq '---') {
23-
if ($start -eq -1) {
24-
$start = $x
25-
} else {
26-
if ($end -eq -1) {
27-
$end = $x+1
28-
break
29-
}
30-
}
31-
}
32-
}
33-
if ($end -gt $start) {
34-
Write-Output ($doc[$end..$($doc.count)] -join "`r`n")
35-
} else {
36-
Write-Output ($doc -join "`r`n")
37-
}
38-
}
8+
if ($ShowProgress) { $ProgressPreference = 'Continue' }
399

4010
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
4111

@@ -50,86 +20,155 @@ Invoke-WebRequest -Uri $pandocSourceURL -OutFile $pandocZipPath
5020
Expand-Archive -Path (Join-Path $pandocDestinationPath "pandoc-$panDocVersion-windows.zip") -DestinationPath $pandocDestinationPath -Force
5121
$pandocExePath = Join-Path (Join-Path $pandocDestinationPath "pandoc-$panDocVersion") "pandoc.exe"
5222

23+
# Install ThreadJob if not available
24+
$threadJob = Get-Module ThreadJob -ListAvailable
25+
if ($null -eq $threadjob) {
26+
Install-Module ThreadJob -RequiredVersion 1.1.2 -Scope CurrentUser -Force
27+
}
28+
5329
# Find the reference folder path w.r.t the script
5430
$ReferenceDocset = Join-Path $PSScriptRoot 'reference'
5531

56-
# Variable to collect any errors in during processing
57-
$allErrors = @()
58-
5932
# Go through all the directories in the reference folder
33+
$jobs = [System.Collections.Generic.List[object]]::new()
6034
Get-ChildItem $ReferenceDocset -Directory -Exclude 'docs-conceptual', 'mapping', 'bread' | ForEach-Object -Process {
61-
$Version = $_.Name
62-
Write-Verbose -Verbose "Version = $Version"
35+
$job = Start-ThreadJob -Name $_.Name -ArgumentList @($SkipCabs,$pandocExePath,$PSScriptRoot,$_) -ScriptBlock {
36+
param($SkipCabs, $pandocExePath, $WorkingDirectory, $DocSet)
37+
38+
$tempDir = [System.IO.Path]::GetTempPath()
39+
$workingDir = Join-Path $tempDir $DocSet.Name
40+
$workingDir = New-Item -ItemType Directory -Path $workingDir -Force
41+
Set-Location $WorkingDir
42+
43+
function Get-ContentWithoutHeader {
44+
param(
45+
$path
46+
)
47+
48+
$doc = Get-Content $path -Encoding UTF8
49+
$start = $end = -1
50+
51+
# search the first 30 lines for the Yaml header
52+
# no yaml header in our docset will ever be that long
53+
54+
for ($x = 0; $x -lt 30; $x++) {
55+
if ($doc[$x] -eq '---') {
56+
if ($start -eq -1) {
57+
$start = $x
58+
}
59+
else {
60+
if ($end -eq -1) {
61+
$end = $x + 1
62+
break
63+
}
64+
}
65+
}
66+
}
67+
if ($end -gt $start) {
68+
Write-Output ($doc[$end..$($doc.count)] -join "`r`n")
69+
}
70+
else {
71+
Write-Output ($doc -join "`r`n")
72+
}
73+
}
6374

64-
$VersionFolder = $_.FullName
65-
Write-Verbose -Verbose "VersionFolder = $VersionFolder"
75+
$Version = $DocSet.Name
76+
Write-Verbose -Verbose "Version = $Version"
6677

67-
# For each of the directories, go through each module folder
68-
Get-ChildItem $VersionFolder -Directory | ForEach-Object -Process {
69-
$ModuleName = $_.Name
70-
Write-Verbose -Verbose "ModuleName = $ModuleName"
78+
$VersionFolder = $DocSet.FullName
79+
Write-Verbose -Verbose "VersionFolder = $VersionFolder"
7180

72-
$ModulePath = Join-Path $VersionFolder $ModuleName
73-
Write-Verbose -Verbose "ModulePath = $ModulePath"
81+
# For each of the directories, go through each module folder
82+
Get-ChildItem $VersionFolder -Directory | ForEach-Object -Process {
83+
$ModuleName = $_.Name
84+
Write-Verbose -Verbose "ModuleName = $ModuleName"
7485

75-
$LandingPage = Join-Path $ModulePath "$ModuleName.md"
76-
Write-Verbose -Verbose "LandingPage = $LandingPage"
86+
$ModulePath = Join-Path $VersionFolder $ModuleName
87+
Write-Verbose -Verbose "ModulePath = $ModulePath"
7788

78-
$MamlOutputFolder = Join-Path "$PSScriptRoot\maml" "$Version\$ModuleName"
79-
Write-Verbose -Verbose "MamlOutputFolder = $MamlOutputFolder"
89+
$LandingPage = Join-Path $ModulePath "$ModuleName.md"
90+
Write-Verbose -Verbose "LandingPage = $LandingPage"
8091

81-
$CabOutputFolder = Join-Path "$PSScriptRoot\updatablehelp" "$Version\$ModuleName"
82-
Write-Verbose -Verbose "CabOutputFolder = $CabOutputFolder"
92+
$MamlOutputFolder = Join-Path "$WorkingDirectory\maml" "$Version\$ModuleName"
93+
Write-Verbose -Verbose "MamlOutputFolder = $MamlOutputFolder"
8394

84-
if (-not (Test-Path $MamlOutputFolder)) {
85-
New-Item $MamlOutputFolder -ItemType Directory -Force > $null
86-
}
95+
$CabOutputFolder = Join-Path "$WorkingDirectory\updatablehelp" "$Version\$ModuleName"
96+
Write-Verbose -Verbose "CabOutputFolder = $CabOutputFolder"
8797

88-
# Process the about topics if any
89-
$AboutFolder = Join-Path $ModulePath "About"
90-
91-
if (Test-Path $AboutFolder) {
92-
Write-Verbose -Verbose "AboutFolder = $AboutFolder"
93-
Get-ChildItem "$aboutfolder/about_*.md" | ForEach-Object {
94-
$aboutFileFullName = $_.FullName
95-
$aboutFileOutputName = "$($_.BaseName).help.txt"
96-
$aboutFileOutputFullName = Join-Path $MamlOutputFolder $aboutFileOutputName
97-
98-
$pandocArgs = @(
99-
"--from=gfm",
100-
"--to=plain+multiline_tables+inline_code_attributes",
101-
"--columns=75",
102-
"--output=$aboutFileOutputFullName",
103-
"--quiet"
104-
)
105-
106-
Get-ContentWithoutHeader $aboutFileFullName | & $pandocExePath $pandocArgs
98+
if (-not (Test-Path $MamlOutputFolder)) {
99+
New-Item $MamlOutputFolder -ItemType Directory -Force > $null
107100
}
108-
}
109101

110-
try {
111-
# For each module, create a single maml help file
112-
# Adding warningaction=stop to throw errors for all warnings, erroraction=stop to make them terminating errors
113-
New-ExternalHelp -Path $ModulePath -OutputPath $MamlOutputFolder -Force -WarningAction Stop -ErrorAction Stop
102+
# Process the about topics if any
103+
$AboutFolder = Join-Path $ModulePath "About"
104+
105+
if (Test-Path $AboutFolder) {
106+
Write-Verbose -Verbose "AboutFolder = $AboutFolder"
107+
Get-ChildItem "$aboutfolder/about_*.md" | ForEach-Object {
108+
$aboutFileFullName = $_.FullName
109+
$aboutFileOutputName = "$($_.BaseName).help.txt"
110+
$aboutFileOutputFullName = Join-Path $MamlOutputFolder $aboutFileOutputName
111+
112+
$pandocArgs = @(
113+
"--from=gfm",
114+
"--to=plain+multiline_tables+inline_code_attributes",
115+
"--columns=75",
116+
"--output=$aboutFileOutputFullName",
117+
"--quiet"
118+
)
119+
120+
Get-ContentWithoutHeader $aboutFileFullName | & $pandocExePath $pandocArgs
121+
}
122+
}
123+
124+
try {
125+
# For each module, create a single maml help file
126+
# Adding warningaction=stop to throw errors for all warnings, erroraction=stop to make them terminating errors
127+
New-ExternalHelp -Path $ModulePath -OutputPath $MamlOutputFolder -Force -WarningAction Stop -ErrorAction Stop
114128

115-
# For each module, create update-help help files (cab and helpinfo.xml files)
116-
if (-not $SkipCabs) {
117-
$cabInfo = New-ExternalHelpCab -CabFilesFolder $MamlOutputFolder -LandingPagePath $LandingPage -OutputFolder $CabOutputFolder
129+
# For each module, create update-help help files (cab and helpinfo.xml files)
130+
if (-not $SkipCabs) {
131+
$cabInfo = New-ExternalHelpCab -CabFilesFolder $MamlOutputFolder -LandingPagePath $LandingPage -OutputFolder $CabOutputFolder
118132

119-
# Only output the cab fileinfo object
120-
if ($cabInfo.Count -eq 8) {$cabInfo[-1].FullName}
133+
# Only output the cab fileinfo object
134+
if ($cabInfo.Count -eq 8) { $cabInfo[-1].FullName }
135+
}
136+
}
137+
catch {
138+
Write-Error -Message "PlatyPS failure: $ModuleName -- $Version" -Exception $_
121139
}
122140
}
123-
catch {
124-
$allErrors += $_
125-
Write-Error -Message "PlatyPS failure: $ModuleName -- $Version" -Exception $_
141+
142+
Remove-Item $workingDir -Force -ErrorAction SilentlyContinue
143+
}
144+
Write-Verbose -Verbose "Started job for $($_.Name)"
145+
$jobs += $job
146+
}
147+
148+
$null = $jobs | Wait-Job
149+
150+
# Variable to collect any errors in during processing
151+
$allErrors = [System.Collections.Generic.List[string]]::new()
152+
foreach ($job in $jobs) {
153+
Write-Verbose -Verbose "$($job.Name) output:"
154+
if ($job.Verbose.Count -gt 0) {
155+
foreach ($verboseMessage in $job.Verbose) {
156+
Write-Verbose -Verbose $verboseMessage
126157
}
127158
}
128159

160+
if ($job.State -eq "Failed") {
161+
$allErrors += "$($job.Name) failed due to unhandled exception"
162+
}
163+
164+
if ($job.Error.Count -gt 0) {
165+
$allErrors += "$($job.Name) failed with errors:"
166+
$allErrors += $job.Error.ReadAll()
167+
}
129168
}
130169

131170
# If the above block, produced any errors, throw and fail the job
132-
if ($allErrors) {
171+
if ($allErrors.Count -gt 0) {
133172
$allErrors
134173
throw "There are errors during platyPS run!`nPlease fix your markdown to comply with the schema: https://github.com/PowerShell/platyPS/blob/master/platyPS.schema.md"
135174
}

0 commit comments

Comments
 (0)