cache the find_all_in_cache_path call during parallel precompilation (#56369)

Before (in an environment with DifferentialEquations.jl):

```julia
julia> @time Pkg.precompile()
  0.733576 seconds (3.44 M allocations: 283.676 MiB, 6.24% gc time)

julia> isfile_calls[1:10]
10-element Vector{Pair{String, Int64}}:
        "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/Printf/3FQLY_zHycD.ji" => 178
        "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/Printf/3FQLY_xxrt3.ji" => 178
         "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/Dates/p8See_xxrt3.ji" => 158
         "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/Dates/p8See_zHycD.ji" => 158
          "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/TOML/mjrwE_zHycD.ji" => 155
          "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/TOML/mjrwE_xxrt3.ji" => 155
                                     "/home/kc/.julia/compiled/v1.12/Preferences/pWSk8_4Qv86.ji" => 152
                                     "/home/kc/.julia/compiled/v1.12/Preferences/pWSk8_juhqb.ji" => 152
 "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/StyledStrings/UcVoM_zHycD.ji" => 144
 "/home/kc/.julia/juliaup/julia-nightly/share/julia/compiled/v1.12/StyledStrings/UcVoM_xxrt3.ji" => 144
 ```

After:

```julia
julia> @time Pkg.precompile()
  0.460077 seconds (877.59 k allocations: 108.075 MiB, 4.77% gc time)

julia> isfile_calls[1:10]
  10-element Vector{Pair{String, Int64}}:
"/tmp/jl_a5xFWK/Project.toml" => 15
"/tmp/jl_a5xFWK/Manifest.toml" => 7
"/home/kc/.julia/registries/General.toml" => 6

"/home/kc/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.12/Markdown/src/Markdown.jl"
=> 3

"/home/kc/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.12/Serialization/src/Serialization.jl"
=> 3

"/home/kc/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.12/Distributed/src/Distributed.jl"
=> 3

"/home/kc/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.12/UUIDs/src/UUIDs.jl"
=> 3

"/home/kc/.julia/juliaup/julia-nightly/share/julia/stdlib/v1.12/LibCURL/src/LibCURL.jl"
=> 3
```

Performance is improved and we are not calling `isfile` on a bunch of the same ji files hundreds times.

Benchmark is made on a linux machine so performance diff should be a lot better on Windows where these `isfile_casesensitive` call is much more expensive.

Fixes #56366

---------

Co-authored-by: KristofferC <[email protected]>
Co-authored-by: Ian Butterworth <[email protected]>
(cherry picked from commit 9850a38)

Author: 3 people

base/loading.jl

@@ -1770,7 +1770,8 @@ const StaleCacheKey = Tuple{Base.PkgId, UInt128, String, String}
 function compilecache_path(pkg::PkgId;
         ignore_loaded::Bool=false,
         stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(),
-        cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg),
+        cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(),
+        cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(pkg), cachepath_cache, pkg),
         sourcepath::Union{String,Nothing}=Base.locate_package(pkg),
         flags::CacheFlags=CacheFlags())
     path = nothing
@@ -1785,7 +1786,7 @@ function compilecache_path(pkg::PkgId;
         for dep in staledeps
             dep isa Module && continue
             modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
-            modpaths = find_all_in_cache_path(modkey)
+            modpaths = get!(() -> find_all_in_cache_path(modkey), cachepath_cache, modkey)
             for modpath_to_try in modpaths::Vector{String}
                 stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey
                 if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded, requested_flags=flags) === true,
@@ -1827,10 +1828,11 @@ fresh julia session specify `ignore_loaded=true`.
 function isprecompiled(pkg::PkgId;
         ignore_loaded::Bool=false,
         stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(),
-        cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg),
+        cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(),
+        cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(pkg), cachepath_cache, pkg),
         sourcepath::Union{String,Nothing}=Base.locate_package(pkg),
         flags::CacheFlags=CacheFlags())
-    path = compilecache_path(pkg; ignore_loaded, stale_cache, cachepaths, sourcepath, flags)
+    path = compilecache_path(pkg; ignore_loaded, stale_cache, cachepath_cache, cachepaths, sourcepath, flags)
     return !isnothing(path)
 end
 

base/precompilation.jl

@@ -400,6 +400,8 @@ function _precompilepkgs(pkgs::Vector{String},
     color_string(cstr::String, col::Union{Int64, Symbol}) = _color_string(cstr, col, hascolor)
 
     stale_cache = Dict{StaleCacheKey, Bool}()
+    cachepath_cache = Dict{PkgId, Vector{String}}()
+
     exts = Dict{PkgId, String}() # ext -> parent
     # make a flat map of each dep and its direct deps
     depsmap = Dict{PkgId, Vector{PkgId}}()
@@ -773,7 +775,7 @@ function _precompilepkgs(pkgs::Vector{String},
     ## precompilation loop
 
     for (pkg, deps) in depsmap
-        cachepaths = Base.find_all_in_cache_path(pkg)
+        cachepaths = get!(() -> Base.find_all_in_cache_path(pkg), cachepath_cache, pkg)
         sourcepath = Base.locate_package(pkg)
         single_requested_pkg = length(pkgs) == 1 && only(pkgs) == pkg.name
         for config in configs
@@ -796,7 +798,7 @@ function _precompilepkgs(pkgs::Vector{String},
                         wait(was_processed[(dep,config)])
                     end
                     circular = pkg in circular_deps
-                    is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags)
+                    is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepath_cache, cachepaths, sourcepath, flags=cacheflags)
                     if !circular && is_stale
                         Base.acquire(parallel_limiter)
                         is_direct_dep = pkg in direct_deps