allow building a "fat sysimage" with all the stdlibs in it

base/sysimg_stdlibs.jl

@@ -0,0 +1,59 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Second-stage sysimage builder that loads all stdlibs from precompiled cache files
+# This is used when building a "fat" sysimage with all stdlibs included
+# This file follows the same structure as sysimg.jl but loads all available stdlibs
+
+Base.reinit_stdio()
+@eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String
+@eval Sys STDLIB = abspath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "stdlib", string('v', VERSION.major, '.', VERSION.minor))
+@eval Base _atexit_hooks_finished = false
+
+# Set up depot & load paths to be able to find stdlib packages
+Base.init_depot_path()
+Base.init_load_path()
+
+if Base.is_primary_base_module
+# Load all stdlib packages (similar to sysimg.jl but loading all stdlibs)
+let
+    # Collect all stdlibs from the stdlib directory
+    stdlibs = Symbol[]
+    for entry in readdir(Sys.STDLIB)
+        stdlib_path = joinpath(Sys.STDLIB, entry)
+        # Check if it's a directory with a Project.toml (indicates a stdlib)
+        if isdir(stdlib_path) && isfile(joinpath(stdlib_path, "Project.toml"))
+            push!(stdlibs, Symbol(entry))
+        end
+    end
+
+
+    maxlen = maximum(textwidth.(string.(stdlibs)); init=0)
+
+    m = Core.Module()
+    GC.@preserve m begin
+        print_time = @eval m (mod, t) -> (print(rpad(string(mod) * "  ", $maxlen + 3, "─"));
+                                          Base.time_print(stdout, t * 10^9); println())
+
+        tot_time_stdlib = @elapsed for stdlib in stdlibs
+            tt = @elapsed Base.require(Base, stdlib)
+            print_time(stdlib, tt)
+        end
+
+        print_time("Stdlibs total", tot_time_stdlib)
+    end
+
+    # Clear global state
+    empty!(Core.ARGS)
+    empty!(Base.ARGS)
+    empty!(LOAD_PATH)
+    empty!(DEPOT_PATH)
+end
+
+empty!(Base.TOML_CACHE.d)
+Base.TOML.reinit!(Base.TOML_CACHE.p, "")
+@eval Base BUILDROOT = ""
+@eval Sys begin
+    BINDIR = ""
+    STDLIB = ""
+end
+end

pkgimage.mk

@@ -4,6 +4,9 @@ JULIAHOME := $(SRCDIR)
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/stdlib/stdlib.mk
 
+# Import the fat sysimage setting from sysimage.mk
+JULIA_BUILD_FAT_SYSIMAGE ?= 0
+
 DEPOTDIR := $(build_prefix)/share/julia
 
 # set some influential environment variables
@@ -25,13 +28,27 @@ $(DEPOTDIR)/compiled:
 print-depot-path:
 	@$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH')
 
+ifeq ($(JULIA_BUILD_FAT_SYSIMAGE),1)
+# In fat mode, use sysbase and add stdlib to LOAD_PATH
+$(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sysbase.$(SHLIB_EXT) $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled
+	@$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --sysimage=$(build_private_libdir)/sysbase.$(SHLIB_EXT) --startup-file=no -e \
+		'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])')
+	touch $@
+
+$(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT) $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled
+	@$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --sysimage=$(build_private_libdir)/sysbase-debug.$(SHLIB_EXT) --startup-file=no -e \
+		'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])')
+	touch $@
+else
+# In normal mode, use default sysimage
 $(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled
 	@$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \
 		'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])')
 	touch $@
 
 $(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sys.$(SHLIB_EXT)
 $(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
+endif
 
 clean:
 	rm -rf $(DEPOTDIR)/compiled

sysimage.mk

@@ -4,6 +4,17 @@ JULIAHOME := $(SRCDIR)
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/stdlib/stdlib.mk
 
+# Allow building a "fat" sysimage with all stdlibs included
+# When enabled, this creates a two-stage build:
+#   1. Build the normal sysimage (with default stdlibs)
+#   2. Precompile all remaining stdlibs using the normal sysimage
+#   3. Build the final fat sysimage by loading all precompiled stdlibs on top
+# This allows stdlibs to run their __init__() during precompilation,
+# which is necessary for stdlibs with complex precompile workloads.
+# Enable by setting JULIA_BUILD_FAT_SYSIMAGE=1 in Make.user or environment
+# TODO: Default to false
+JULIA_BUILD_FAT_SYSIMAGE ?= 1
+
 default: sysimg-$(JULIA_BUILD_MODE) # contains either "debug" or "release"
 all: sysimg-release sysimg-debug
 basecompiler-ji: $(build_private_libdir)/basecompiler.ji
@@ -147,7 +158,54 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil
 .SECONDARY: $$(build_private_libdir)/sys$1-o.a $(build_private_libdir)/sys$1-bc.a # request Make to keep these files around
 .SECONDARY: $$(build_private_libdir)/sysbase$1-o.a $(build_private_libdir)/sysbase$1-bc.a # request Make to keep these files around
 endef
+
+# Fat sysimage builder - builds on top of normal sysimage with all stdlibs
+# Parameters: $1=suffix (e.g., -debug), $2=image name (release/debug), $3=opt flags (e.g., -O3), $4=executable
+define sysimg_builder_fat
+build_sysbase_$1 := $$(or $$(CROSS_BOOTSTRAP_SYSBASE),$$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT))
+$$(build_private_libdir)/sys$1-o.a : $$(build_sysbase_$1) $$(BUILDROOT)/stdlib/$2.image $$(JULIAHOME)/base/sysimg_stdlibs.jl
+	@$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \
+	if ! JULIA_BINDIR=$$(call cygpath_w,$$(build_bindir)) \
+		 WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \
+		 JULIA_LOAD_PATH='@stdlib' \
+		 JULIA_PROJECT= \
+		 JULIA_DEPOT_PATH=$$(call cygpath_w,$$(build_prefix)/share/julia) \
+		 JULIA_NUM_THREADS=1 \
+			$$(call spawn, $4) $3 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-o $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
+			--startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) sysimg_stdlibs.jl; then \
+		echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \
+		false; \
+	fi )
+	@mv [email protected] $$@
+.SECONDARY: $$(build_private_libdir)/sys$1-o.a # request Make to keep these files around
+endef
+
+ifeq ($(JULIA_BUILD_FAT_SYSIMAGE),1)
+# For fat sysimage, build normal sysbase first, then precompile all stdlibs, then build fat sys
+# Stage 1: Build normal sysbase (with default stdlibs)
+# Stage 2: Precompile all stdlibs (handled by pkgimage.mk)
+# Stage 3: Build final fat sys with all precompiled stdlibs
+
+# Delegate to pkgimage.mk for building the precompilation stamps
+$(BUILDROOT)/stdlib/release.image: $(build_private_libdir)/sysbase.$(SHLIB_EXT)
+	@$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/pkgimage.mk release JULIA_BUILD_FAT_SYSIMAGE=$(JULIA_BUILD_FAT_SYSIMAGE)
+
+$(BUILDROOT)/stdlib/debug.image: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT)
+	@$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/pkgimage.mk debug JULIA_BUILD_FAT_SYSIMAGE=$(JULIA_BUILD_FAT_SYSIMAGE)
+
+$(eval $(call base_builder,,-O1,$(JULIA_EXECUTABLE_release)))
+$(eval $(call base_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
+$(eval $(call sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release)))
+$(eval $(call sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
+$(eval $(call sysimg_builder_fat,,release,-O3,$(JULIA_EXECUTABLE_release)))
+$(eval $(call sysimg_builder_fat,-debug,debug,-O0,$(JULIA_EXECUTABLE_debug)))
+
+# In fat mode, the normal sysimg targets build the fat sysimage
+# The dependency on stdlib/.image is implicit through sys-o.a dependencies
+
+else
 $(eval $(call base_builder,,-O1,$(JULIA_EXECUTABLE_release)))
 $(eval $(call base_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
 $(eval $(call sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release)))
 $(eval $(call sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
+endif