Skip to content

Refactor local module find process #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 22, 2023
Merged

Refactor local module find process #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f5f91d5
Initial Rewrite of Local Module Find Process
JustinGrote Dec 22, 2023
c452fea
More fixups
JustinGrote Dec 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 100 additions & 89 deletions ModuleFast.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,9 @@ function Install-ModuleFast {
$planAlreadySatisfiedMessage = "`u{2705} $($ModulesToInstall.count) Module Specifications have all been satisfied by installed modules. If you would like to check for newer versions remotely, specify -Update"
if ($WhatIfPreference) {
Write-Host -fore DarkGreen $planAlreadySatisfiedMessage
} else {
Write-Verbose $planAlreadySatisfiedMessage
}
#TODO: Deduplicate this with the end into its own function
Write-Verbose $planAlreadySatisfiedMessage
return
}

Expand Down Expand Up @@ -335,11 +335,15 @@ function Get-ModuleFastPlan {
Write-Verbose "${moduleSpec}: Evaluating Module Specification"
[ModuleFastInfo]$localMatch = Find-LocalModule $moduleSpec -Update:$Update
if ($localMatch) {
Write-Debug "FOUND local module $($localMatch.Name) $($localMatch.ModuleVersion) at $($localMatch.Location) that satisfies $moduleSpec. Skipping..."
Write-Debug "${localMatch}: 🎯 FOUND satisfing version $($localMatch.ModuleVersion) at $($localMatch.Location). Skipping remote search."
#TODO: Capture this somewhere that we can use it to report in the deploy plan
continue
}

#If we get this far, we didn't find a manifest in this module path
Write-Debug "${moduleSpec}: 🔍 No installed versions matched the spec. Will check remotely."


$task = Get-ModuleInfoAsync @httpContext -Endpoint $Source -Name $moduleSpec.Name
$taskSpecMap[$task] = $moduleSpec
$currentTasks.Add($task)
Expand Down Expand Up @@ -532,7 +536,7 @@ function Get-ModuleFastPlan {

[ModuleFastSpec]::new($PSItem.id, $range)
}
Write-Debug "$currentModuleSpec`: has $($dependencies.count) additional dependencies."
Write-Debug "$currentModuleSpec`: has $($dependencies.count) additional dependencies: $($dependencies -join ', ')"

# TODO: Where loop filter maybe
[ModuleFastSpec[]]$dependenciesToResolve = $dependencies | Where-Object {
Expand Down Expand Up @@ -1032,6 +1036,9 @@ class ModuleFastSpec {
[string] ToString() {
$guid = $this._Guid -ne [Guid]::Empty ? " [$($this._Guid)]" : ''
$versionRange = $this._VersionRange.ToString() -eq '(, )' ? '' : " $($this._VersionRange)"
if ($this._VersionRange.MaxVersion -eq $this._VersionRange.MinVersion) {
$versionRange = "=$($this._VersionRange.MinVersion)"
}
return "$($this._Name)$guid$versionRange"
}
[int] GetHashCode() {
Expand Down Expand Up @@ -1246,20 +1253,21 @@ function Find-LocalModule {
)
$ErrorActionPreference = 'Stop'

# Search all psmodulepaths for the module
$modulePaths = $env:PSModulePath.Split([Path]::PathSeparator, [StringSplitOptions]::RemoveEmptyEntries)
if (-Not $modulePaths) {
if (-not $modulePaths) {
Write-Warning 'No PSModulePaths found in $env:PSModulePath. If you are doing isolated testing you can disregard this.'
return
}

#First property is the manifest path, second property is the actual version (may be different from the folder version as prerelease versions go in the same location)
#We want to minimize reading the manifest files, so we will do a fast file-based search first and then do a more detailed inspection on high confidence candidate(s). Any module in any folder path that satisfies the spec will be sufficient, we don't care about finding the "latest" version, so we will return the first module that satisfies the spec.

#We will store potential candidates in this list, with their evaluated "guessed" version based on the folder name and the path. The first items added to the list should be the highest likelihood candidates in Path priority order, so no sorting should be necessary.


[List[[Tuple[Version, string]]]]$candidateModules = foreach ($modulePath in $modulePaths) {
foreach ($modulePath in $modulePaths) {
[List[[Tuple[Version, string]]]]$candidatePaths = @()
if (-not [Directory]::Exists($modulePath)) {
Write-Debug "$($ModuleSpec.Name): PSModulePath $modulePath is configured but does not exist, skipping..."
Write-Debug "${ModuleSpec}: Skipping PSModulePath $modulePath - Configured but does not exist."
$modulePaths = $modulePaths | Where-Object { $_ -ne $modulePath }
continue
}
Expand All @@ -1268,12 +1276,10 @@ function Find-LocalModule {
$moduleBaseDir = [Directory]::GetDirectories($modulePath, $moduleSpec.Name, [EnumerationOptions]@{MatchCasing = 'CaseInsensitive' })
if ($moduleBaseDir.count -gt 1) { throw "$($moduleSpec.Name) folder is ambiguous, please delete one of these folders: $moduleBaseDir" }
if (-not $moduleBaseDir) {
Write-Debug "$($moduleSpec.Name): PSModulePath $modulePath does not have this module. Skipping..."
Write-Debug "${ModuleSpec}: Skipping PSModulePath $modulePath - Does not have this module."
continue
}

$manifestName = "$($ModuleSpec.Name).psd1"

#We can attempt a fast-search for modules if the ModuleSpec is for a specific version
$required = $ModuleSpec.Required
if ($required) {
Expand All @@ -1285,111 +1291,92 @@ function Find-LocalModule {
$manifestPath = Join-Path $moduleFolder $manifestName

if (Test-Path $ModuleFolder) {
#Linux/Mac support requires a case insensitive search on a user supplied argument.
$manifestPath = [Directory]::GetFiles($moduleFolder, "$($ModuleSpec.Name).psd1", [EnumerationOptions]@{MatchCasing = 'CaseInsensitive' })

if ($manifestPath.count -gt 1) { throw "$moduleFolder manifest is ambiguous, please delete one of these: $manifestPath" }

#Early return if we found a manifest, we don't need to do further checking
if ($manifestPath.count -eq 1) {
[Tuple]::Create([version]$moduleVersion, $manifestPath[0])
continue
}
}
}

#Check for versioned module folders next
foreach ($folder in [Directory]::GetDirectories($moduleBaseDir)) {
#Sanity check
$versionCandidate = Split-Path -Leaf $folder
[Version]$version = $null
if (-not [Version]::TryParse($versionCandidate, [ref]$version)) {
Write-Debug "Could not parse $folder in $moduleBaseDir as a valid version. This is either a bad version directory or this folder is a classic module."
continue
$candidatePaths.Add([Tuple]::Create($moduleVersion, $manifestPath))
}
} else {
#Check for versioned module folders next and sort by the folder versions to process them in descending order.
[Directory]::GetDirectories($moduleBaseDir)
| ForEach-Object {
$folder = $PSItem
$version = $null
$isVersion = [Version]::TryParse((Split-Path -Leaf $PSItem), [ref]$version)
if (-not $isVersion) {
Write-Debug "Could not parse $folder in $moduleBaseDir as a valid version. This is either a bad version directory or this folder is a classic module."
return
}

#Try to retrieve the manifest
#TODO: Create a "Assert-CaseSensitiveFileExists" function for this pattern used multiple times
$versionedManifestPath = [Directory]::GetFiles($folder, $manifestName, [EnumerationOptions]@{MatchCasing = 'CaseInsensitive' })

if ($versionedManifestPath.count -gt 1) { throw "$folder manifest is ambiguous, this happens on Linux if you have two manifests with different case sensitivity. Please delete one of these: $versionedManifestPath" }
#Fast filter items that are above the upper bound, we dont need to read these manifests
if ($ModuleSpec.Max -and $version -gt $ModuleSpec.Max.Version) {
Write-Debug "${ModuleSpec}: Skipping $folder - above the upper bound"
return
}

if (-not $versionedManifestPath) {
Write-Warning "Found a candidate versioned module folder $folder but no $manifestName manifest was found in the folder. This is an indication of a corrupt module and you should clean this folder up"
continue
}
#We can fast filter items that are below the lower bound, we dont need to read these manifests
if ($ModuleSpec.Min) {
#HACK: Nuget does not correctly convert major.minor.build versions.
[version]$originalBaseVersion = ($modulespec.Min.OriginalVersion -split '-')[0]
[Version]$minVersion = $originalBaseVersion.Revision -eq -1 ? $originalBaseVersion : $ModuleSpec.Min.Version
if ($minVersion -lt $ModuleSpec.Min.OriginalVersion) {
Write-Debug "${ModuleSpec}: Skipping $folder - below the lower bound"
return
}
}

if ($versionedManifestPath.count -eq 1) {
[Tuple]::Create([version]$version, $versionedManifestPath[0])
$candidatePaths.Add([Tuple]::Create($version, $PSItem))
}
}

#Check for a "classic" module if no versioned folders were found
if ($candidateModules.count -eq 0) {
if ($candidatePaths.count -eq 0) {
[string[]]$classicManifestPaths = [Directory]::GetFiles($moduleBaseDir, $manifestName, [EnumerationOptions]@{MatchCasing = 'CaseInsensitive' })
if ($classicManifestPaths.count -gt 1) { throw "$moduleBaseDir manifest is ambiguous, please delete one of these: $classicManifestPath" }
[string]$classicManifestPath = $classicManifestPaths[0]
if ($classicManifestPath) {
#TODO: Optimize this so that import-powershelldatafile is not called twice. This should be a rare occurance so it's not a big deal.
#NOTE: This does result in Import-PowerShellData getting called twice which isn't ideal for performance, but classic modules should be fairly rare and not worth optimizing.
[version]$classicVersion = (Import-PowerShellDataFile $classicManifestPath).ModuleVersion
[Tuple]::Create($classicVersion, $classicManifestPath)
continue
Write-Debug "${ModuleSpec}: Found classic module $classicVersion at $moduleBaseDir"
$candidatePaths.Add([Tuple]::Create($classicVersion, $moduleBaseDir))
}
}

#If we get this far, we didn't find a manifest in this module path
Write-Debug "$moduleSpec`: module folder exists at $moduleBaseDir but no modules found that match the version spec."
}

if ($candidateModules.count -eq 0) { return $null }

# We have to read the manifests to verify if the specified installed module is a prerelease module, which can affect whether it is selected by this function.
# TODO: Filter to likely candidates first
#NOTE: We use the sort rather than FindBestMatch because we want the highest compatible version, due to auto assembly redirect in PSCore
foreach ($moduleInfo in ($candidateModules | Sort-Object Item1 -Descending)) {
[NugetVersion]$version = [NugetVersion]::new($moduleInfo.Item1)
[string]$manifestPath = $moduleInfo.Item2

#The ModuleSpec.Max.Version check is to support an edge case where the module prerelease version is actually less than the prerelease constraint but we haven't read the manifest yet to determine that.
if (-not $ModuleSpec.SatisfiedBy($version) -and $ModuleSpec.Max.Version -ne $version) {
Write-Debug "$($ModuleSpec.Name): Found a module $($moduleInfo.Item2) that matches the name but does not satisfy the version spec $($ModuleSpec). Skipping..."
if ($candidatePaths.count -eq 0) {
Write-Debug "${ModuleSpec}: Skipping PSModulePath $modulePath - No installed versions matched the spec."
continue
}

$manifestData = Import-PowerShellDataFile -Path $manifestPath -ErrorAction stop
foreach ($candidateItem in $candidatePaths) {
[version]$version = $candidateItem.Item1
[string]$folder = $candidateItem.Item2

[Version]$manifestVersionData = $null
if (-not [Version]::TryParse($manifestData.ModuleVersion, [ref]$manifestVersionData)) {
Write-Warning "Found a manifest at $manifestPath but the version $($manifestData.ModuleVersion) in the manifest information is not a valid version. This is probably an invalid or corrupt manifest"
continue
}

[NuGetVersion]$manifestVersion = [NuGetVersion]::new(
$manifestVersionData,
$manifestData.PrivateData.PSData.Prerelease
)
#Read the module manifest to check for prerelease versions.
$manifestName = "$($ModuleSpec.Name).psd1"
$versionedManifestPath = [Directory]::GetFiles($folder, $manifestName, [EnumerationOptions]@{MatchCasing = 'CaseInsensitive' })

#Re-Test against the manifest loaded version to be sure
if (-not $ModuleSpec.SatisfiedBy($manifestVersion)) {
Write-Debug "$($ModuleSpec.Name): Found a module $($moduleInfo.Item2) that initially matched the name and version folder but after reading the manifest, the version label not satisfy the version spec $($ModuleSpec). This is an edge case and should only occur if you specified a prerelease upper bound that is less than the PreRelease label in the manifest. Skipping..."
continue
}
if ($versionedManifestPath.count -gt 1) { throw "$folder manifest is ambiguous, this happens on Linux if you have two manifests with different case sensitivity. Please delete one of these: $versionedManifestPath" }

#If Update is specified, we will be more strict and only report a matching module if it exactly matches the upper bound of the version spec (otherwise there may be a newer module available remotely)
if ($Update) {
if ($ModuleSpec.Max -ne $manifestVersion) {
Write-Debug "$($ModuleSpec.Name): Found a module $($moduleInfo.Item2) that matches the name and version folder but does not exactly match the upper bound of the version spec $($ModuleSpec). Skipping..."
if (-not $versionedManifestPath) {
Write-Warning "${ModuleSpec}: Found a candidate versioned module folder $folder but no $manifestName manifest was found in the folder. This is an indication of a corrupt module and you should clean this folder up"
continue
} else {
Write-Debug "$($ModuleSpec.Name): Found a module $($moduleInfo.Item2) that matches the name and version folder and exactly matches the upper bound of the version spec $($ModuleSpec) because -Update was specified, so it will not be evaluated for install"
}
}

#If we pass all sanity checks, we can return this module as meeting the criteria and skip checking all lower modules.
return [ModuleFastInfo]::new($ModuleSpec.Name, $manifestVersion, $manifestPath)
#Read the manifest so we can compare prerelease info. If this matches, we have a valid candidate and don't need to check anything further.
$manifestCandidate = ConvertFrom-ModuleManifest $versionedManifestPath[0]
$candidateVersion = $manifestCandidate.ModuleVersion

if ($ModuleSpec.SatisfiedBy($candidateVersion)) {
if ($Update -and ($ModuleSpec.Max -ne $candidateVersion)) {
Write-Debug "${ModuleSpec}: Skipping $candidateVersion - The -Update was specified and the version does not exactly meet the upper bound of the spec, meaning there is a possible newer version remotely."
continue
}

#TODO: Collect InstalledButSatisfied Modules into an array so they can later be referenced in the lockfile and/or plan, right now the lockfile only includes modules that changed.
return $manifestCandidate
}
}
}
}


function ConvertTo-AuthenticationHeaderValue ([PSCredential]$Credential) {
$basicCredential = [Convert]::ToBase64String(
[Encoding]::UTF8.GetBytes(
Expand Down Expand Up @@ -1550,6 +1537,30 @@ filter Resolve-FolderVersion([NuGetVersion]$version) {
[Version]::new($version.Major, $version.Minor, $version.Patch)
}

filter ConvertFrom-ModuleManifest {
[CmdletBinding()]
[OutputType([ModuleFastInfo])]
param(
[Parameter(Mandatory)][string]$ManifestPath
)
$ErrorActionPreference = 'Stop'

$ManifestName = Split-Path -Path $ManifestPath -LeafBase
$manifestData = Import-PowerShellDataFile -Path $ManifestPath -ErrorAction stop

[Version]$manifestVersionData = $null
if (-not [Version]::TryParse($manifestData.ModuleVersion, [ref]$manifestVersionData)) {
throw [InvalidDataException]"The manifest at $ManifestPath has an invalid ModuleVersion $($manifestData.ModuleVersion). This is probably an invalid or corrupt manifest"
}

[NuGetVersion]$manifestVersion = [NuGetVersion]::new(
$manifestVersionData,
$manifestData.PrivateData.PSData.Prerelease
)

return [ModuleFastInfo]::new($ManifestName, $manifestVersion, $ManifestPath)
}

#endregion Helpers

### ISSUES
Expand Down
13 changes: 9 additions & 4 deletions ModuleFast.tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,30 +459,30 @@ Describe 'Install-ModuleFast' -Tag 'E2E' {
Get-Module Az.Accounts -ListAvailable
| Limit-ModulePath $installTempPath
| Select-Object -ExpandProperty Version
| Sort-Object Version -Descending
| Sort-Object -Descending
| Select-Object -First 1
| Should -Be '2.10.2'

Install-ModuleFast @imfParams 'Az.Compute', 'Az.Accounts' #Should not update
Get-Module Az.Accounts -ListAvailable
| Limit-ModulePath $installTempPath
| Select-Object -ExpandProperty Version
| Sort-Object Version -Descending
| Sort-Object -Descending
| Select-Object -First 1
| Should -Be '2.10.2'

Install-ModuleFast @imfParams 'Az.Compute' -Update #Should disregard local install and update latest Az.Accounts
Get-Module Az.Accounts -ListAvailable
| Limit-ModulePath $installTempPath
| Select-Object -ExpandProperty Version
| Sort-Object Version -Descending
| Sort-Object -Descending
| Select-Object -First 1
| Should -BeGreaterThan ([version]'2.10.2')

Get-Module Az.Compute -ListAvailable
| Limit-ModulePath $installTempPath
| Select-Object -ExpandProperty Version
| Sort-Object Version -Descending
| Sort-Object -Descending
| Select-Object -First 1
| Should -BeGreaterThan ([version]'5.0.0')
}
Expand All @@ -506,6 +506,11 @@ Describe 'Install-ModuleFast' -Tag 'E2E' {
Install-ModuleFast @imfParams 'PrereleaseTest=0.0.1-bprerelease' -WarningVariable actual *>&1 | Out-Null
$actual | Should -BeLike '*is newer than existing prerelease version*'
}
It 'Doesnt install prerelease if same-version Prerelease already installed' {
Install-ModuleFast @imfParams 'PrereleaseTest=0.0.1-prerelease'
$plan = Install-ModuleFast @imfParams 'PrereleaseTest=0.0.1-prerelease' -WhatIf
$plan | Should -BeNullOrEmpty
}

It 'Installs from <Name> SpecFile' {
$SCRIPT:Mocks = Resolve-Path "$PSScriptRoot/Test/Mocks"
Expand Down