Optimize the size for tsbuildinfo (#43155)

* Baseline readable buildinfo

* Use file names as index in file name list
This is extension of the idea given by @sokra to optimize size of tsbuildinfo

* Deduplicate reference map lists and use file name index to sort them
Different implementation of #43079 based on idea suggested by @sokra

* Minimal json.stringify for the tsbuildinfo
Again implementaion of suggestion by @sokra

* Update src/testRunner/unittests/tsbuild/helpers.ts

Co-authored-by: Tobias Koppers <[email protected]>

* Readable version of buildinfo all the time

* Some renames for readability as per feedback

Co-authored-by: Tobias Koppers <[email protected]>

Author: sheetalkamat

src/compiler/builder.ts

@@ -690,13 +690,16 @@ namespace ts {
         return filterSemanticDiagnotics(diagnostics, state.compilerOptions);
     }
 
-    export type ProgramBuildInfoDiagnostic = string | [string, readonly ReusableDiagnostic[]];
-    export type ProgramBuilderInfoFilePendingEmit = [string, BuilderFileEmit];
+    export type ProgramBuildInfoDiagnostic = number | [fileId: number, diagnostics: readonly ReusableDiagnostic[]];
+    export type ProgramBuilderInfoFilePendingEmit = [fileId: number, emitKind: BuilderFileEmit];
+    export type ProgramBuildInfoReferencedMap = [fileId: number, fileIdListId: number][];
     export interface ProgramBuildInfo {
-        fileInfos: MapLike<BuilderState.FileInfo>;
+        fileNames: readonly string[];
+        fileInfos: readonly BuilderState.FileInfo[];
         options: CompilerOptions;
-        referencedMap?: MapLike<string[]>;
-        exportedModulesMap?: MapLike<string[]>;
+        fileIdsList?: readonly (readonly number[])[];
+        referencedMap?: ProgramBuildInfoReferencedMap;
+        exportedModulesMap?: ProgramBuildInfoReferencedMap;
         semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
         affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
     }
@@ -708,66 +711,74 @@ namespace ts {
         if (outFile(state.compilerOptions)) return undefined;
         const currentDirectory = Debug.checkDefined(state.program).getCurrentDirectory();
         const buildInfoDirectory = getDirectoryPath(getNormalizedAbsolutePath(getTsBuildInfoEmitOutputFilePath(state.compilerOptions)!, currentDirectory));
-        const fileInfos: MapLike<BuilderState.FileInfo> = {};
-        state.fileInfos.forEach((value, key) => {
+        const fileNames: string[] = [];
+        const fileNameToFileId = new Map<string, number>();
+        let fileIdsList: (readonly number[])[] | undefined;
+        let fileNamesToFileIdListId: ESMap<string, number> | undefined;
+        const fileInfos = arrayFrom(state.fileInfos.entries(), ([key, value]) => {
+            // Ensure fileId
+            const fileId = toFileId(key);
+            Debug.assert(fileNames[fileId] === relativeToBuildInfo(key));
             const signature = state.currentAffectedFilesSignatures && state.currentAffectedFilesSignatures.get(key);
-            fileInfos[relativeToBuildInfo(key)] = signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
+            return signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
         });
 
-        const result: ProgramBuildInfo = {
-            fileInfos,
-            options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath)
-        };
+        let referencedMap: ProgramBuildInfoReferencedMap | undefined;
         if (state.referencedMap) {
-            const referencedMap: MapLike<string[]> = {};
-            for (const key of arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive)) {
-                referencedMap[relativeToBuildInfo(key)] = arrayFrom(state.referencedMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
-            }
-            result.referencedMap = referencedMap;
+            referencedMap = arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive).map(key => [
+                toFileId(key),
+                toFileIdListId(state.referencedMap!.get(key)!)
+            ]);
         }
 
+        let exportedModulesMap: ProgramBuildInfoReferencedMap | undefined;
         if (state.exportedModulesMap) {
-            const exportedModulesMap: MapLike<string[]> = {};
-            for (const key of arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive)) {
+            exportedModulesMap = mapDefined(arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive), key => {
                 const newValue = state.currentAffectedFilesExportedModulesMap && state.currentAffectedFilesExportedModulesMap.get(key);
                 // Not in temporary cache, use existing value
-                if (newValue === undefined) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(state.exportedModulesMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
+                if (newValue === undefined) return [toFileId(key), toFileIdListId(state.exportedModulesMap!.get(key)!)];
                 // Value in cache and has updated value map, use that
-                else if (newValue) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(newValue.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
-            }
-            result.exportedModulesMap = exportedModulesMap;
+                else if (newValue) return [toFileId(key), toFileIdListId(newValue)];
+            });
         }
 
+        let semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] | undefined;
         if (state.semanticDiagnosticsPerFile) {
-            const semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] = [];
             for (const key of arrayFrom(state.semanticDiagnosticsPerFile.keys()).sort(compareStringsCaseSensitive)) {
                 const value = state.semanticDiagnosticsPerFile.get(key)!;
-                semanticDiagnosticsPerFile.push(
+                (semanticDiagnosticsPerFile ||= []).push(
                     value.length ?
                         [
-                            relativeToBuildInfo(key),
+                            toFileId(key),
                             state.hasReusableDiagnostic ?
                                 value as readonly ReusableDiagnostic[] :
                                 convertToReusableDiagnostics(value as readonly Diagnostic[], relativeToBuildInfo)
                         ] :
-                        relativeToBuildInfo(key)
+                        toFileId(key)
                 );
             }
-            result.semanticDiagnosticsPerFile = semanticDiagnosticsPerFile;
         }
 
+        let affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] | undefined;
         if (state.affectedFilesPendingEmit) {
-            const affectedFilesPendingEmit: ProgramBuilderInfoFilePendingEmit[] = [];
             const seenFiles = new Set<Path>();
             for (const path of state.affectedFilesPendingEmit.slice(state.affectedFilesPendingEmitIndex).sort(compareStringsCaseSensitive)) {
                 if (tryAddToSet(seenFiles, path)) {
-                    affectedFilesPendingEmit.push([relativeToBuildInfo(path), state.affectedFilesPendingEmitKind!.get(path)!]);
+                    (affectedFilesPendingEmit ||= []).push([toFileId(path), state.affectedFilesPendingEmitKind!.get(path)!]);
                 }
             }
-            result.affectedFilesPendingEmit = affectedFilesPendingEmit;
         }
 
-        return result;
+        return {
+            fileNames,
+            fileInfos,
+            options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath),
+            fileIdsList,
+            referencedMap,
+            exportedModulesMap,
+            semanticDiagnosticsPerFile,
+            affectedFilesPendingEmit,
+        };
 
         function relativeToBuildInfoEnsuringAbsolutePath(path: string) {
             return relativeToBuildInfo(getNormalizedAbsolutePath(path, currentDirectory));
@@ -776,6 +787,22 @@ namespace ts {
         function relativeToBuildInfo(path: string) {
             return ensurePathIsNonModuleName(getRelativePathFromDirectory(buildInfoDirectory, path, getCanonicalFileName));
         }
+
+        function toFileId(path: Path): number {
+            const existing = fileNameToFileId.get(path);
+            if (existing !== undefined) return existing;
+            fileNameToFileId.set(path, fileNames.length);
+            return fileNames.push(relativeToBuildInfo(path)) - 1;
+        }
+
+        function toFileIdListId(set: ReadonlySet<Path>): number {
+            const fileIds = arrayFrom(set.keys(), toFileId).sort(compareValues);
+            const key = fileIds.join();
+            const existing = fileNamesToFileIdListId?.get(key);
+            if (existing !== undefined) return existing;
+            (fileNamesToFileIdListId ||= new Map()).set(key, fileIdsList?.length || 0);
+            return (fileIdsList ||= []).push(fileIds) - 1;
+        }
     }
 
     function convertToReusableCompilerOptions(options: CompilerOptions, relativeToBuildInfo: (path: string) => string) {
@@ -1167,39 +1194,23 @@ namespace ts {
         }
     }
 
-    function getMapOfReferencedSet(mapLike: MapLike<readonly string[]> | undefined, toPath: (path: string) => Path): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
-        if (!mapLike) return undefined;
-        const map = new Map<Path, BuilderState.ReferencedSet>();
-        // Copies keys/values from template. Note that for..in will not throw if
-        // template is undefined, and instead will just exit the loop.
-        for (const key in mapLike) {
-            if (hasProperty(mapLike, key)) {
-                map.set(toPath(key), new Set(mapLike[key].map(toPath)));
-            }
-        }
-        return map;
-    }
-
     export function createBuildProgramUsingProgramBuildInfo(program: ProgramBuildInfo, buildInfoPath: string, host: ReadBuildProgramHost): EmitAndSemanticDiagnosticsBuilderProgram {
         const buildInfoDirectory = getDirectoryPath(getNormalizedAbsolutePath(buildInfoPath, host.getCurrentDirectory()));
         const getCanonicalFileName = createGetCanonicalFileName(host.useCaseSensitiveFileNames());
 
+        const filePaths = program.fileNames.map(toPath);
+        const filePathsSetList = program.fileIdsList?.map(fileIds => new Set(fileIds.map(toFilePath)));
         const fileInfos = new Map<Path, BuilderState.FileInfo>();
-        for (const key in program.fileInfos) {
-            if (hasProperty(program.fileInfos, key)) {
-                fileInfos.set(toPath(key), program.fileInfos[key]);
-            }
-        }
-
+        program.fileInfos.forEach((fileInfo, fileId) => fileInfos.set(toFilePath(fileId), fileInfo));
         const state: ReusableBuilderProgramState = {
             fileInfos,
             compilerOptions: convertToOptionsWithAbsolutePaths(program.options, toAbsolutePath),
-            referencedMap: getMapOfReferencedSet(program.referencedMap, toPath),
-            exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, toPath),
-            semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toPath(isString(value) ? value : value[0]), value => isString(value) ? emptyArray : value[1]),
+            referencedMap: toMapOfReferencedSet(program.referencedMap),
+            exportedModulesMap: toMapOfReferencedSet(program.exportedModulesMap),
+            semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toFilePath(isNumber(value) ? value : value[0]), value => isNumber(value) ? emptyArray : value[1]),
             hasReusableDiagnostic: true,
-            affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toPath(value[0])),
-            affectedFilesPendingEmitKind: program.affectedFilesPendingEmit && arrayToMap(program.affectedFilesPendingEmit, value => toPath(value[0]), value => value[1]),
+            affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toFilePath(value[0])),
+            affectedFilesPendingEmitKind: program.affectedFilesPendingEmit && arrayToMap(program.affectedFilesPendingEmit, value => toFilePath(value[0]), value => value[1]),
             affectedFilesPendingEmitIndex: program.affectedFilesPendingEmit && 0,
         };
         return {
@@ -1234,6 +1245,18 @@ namespace ts {
         function toAbsolutePath(path: string) {
             return getNormalizedAbsolutePath(path, buildInfoDirectory);
         }
+
+        function toFilePath(fileId: number) {
+            return filePaths[fileId];
+        }
+
+        function toFilePathsSet(fileIdsListId: number) {
+            return filePathsSetList![fileIdsListId];
+        }
+
+        function toMapOfReferencedSet(referenceMap: ProgramBuildInfoReferencedMap | undefined): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
+            return referenceMap && arrayToMap(referenceMap, value => toFilePath(value[0]), value => toFilePathsSet(value[1]));
+        }
     }
 
     export function createRedirectedBuilderProgram(state: { program: Program | undefined; compilerOptions: CompilerOptions; }, configFileParsingDiagnostics: readonly Diagnostic[]): BuilderProgram {

src/compiler/emitter.ts

@@ -663,7 +663,7 @@ namespace ts {
 
     /*@internal*/
     export function getBuildInfoText(buildInfo: BuildInfo) {
-        return JSON.stringify(buildInfo, undefined, 2);
+        return JSON.stringify(buildInfo);
     }
 
     /*@internal*/

src/testRunner/unittests/tsbuild/helpers.ts

@@ -236,18 +236,78 @@ interface Symbol {
         }
     }
 
+    function generateBuildInfoProgramBaseline(sys: System, originalWriteFile: System["writeFile"], buildInfoPath: string, buildInfo: BuildInfo) {
+        type ProgramBuildInfoDiagnostic = string | [string, readonly ReusableDiagnostic[]];
+        type ProgramBuilderInfoFilePendingEmit = [string, BuilderFileEmit];
+        interface ProgramBuildInfo {
+            fileInfos: MapLike<BuilderState.FileInfo>;
+            options: CompilerOptions;
+            referencedMap?: MapLike<string[]>;
+            exportedModulesMap?: MapLike<string[]>;
+            semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
+            affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
+        }
+        const fileInfos: ProgramBuildInfo["fileInfos"] = {};
+        buildInfo.program?.fileInfos.forEach((fileInfo, fileId) => {
+            fileInfos[toFileName(fileId)] = fileInfo;
+        });
+        const fileNamesList = buildInfo.program?.fileIdsList?.map(fileIdsListId => fileIdsListId.map(toFileName));
+        const program: ProgramBuildInfo | undefined = buildInfo.program && {
+            fileInfos,
+            options: buildInfo.program.options,
+            referencedMap: toMapOfReferencedSet(buildInfo.program.referencedMap),
+            exportedModulesMap: toMapOfReferencedSet(buildInfo.program.exportedModulesMap),
+            semanticDiagnosticsPerFile: buildInfo.program.semanticDiagnosticsPerFile?.map(d =>
+                isNumber(d) ?
+                    toFileName(d) :
+                    [toFileName(d[0]), d[1]]
+            ),
+            affectedFilesPendingEmit: buildInfo.program.affectedFilesPendingEmit?.map(([fileId, emitKind]) => [
+                toFileName(fileId),
+                emitKind
+            ]),
+        };
+        const result: Omit<BuildInfo, "program"> & { program: ProgramBuildInfo | undefined; } = {
+            bundle: buildInfo.bundle,
+            program,
+            version: buildInfo.version === version ? fakes.version : buildInfo.version,
+        };
+        // For now its just JSON.stringify
+        originalWriteFile.call(sys, `${buildInfoPath}.readable.baseline.txt`, JSON.stringify(result, /*replacer*/ undefined, 2));
+
+        function toFileName(fileId: number) {
+            return buildInfo.program!.fileNames[fileId];
+        }
+
+        function toFileNames(fileIdsListId: number) {
+            return fileNamesList![fileIdsListId];
+        }
+
+        function toMapOfReferencedSet(referenceMap: ProgramBuildInfoReferencedMap | undefined): MapLike<string[]> | undefined {
+            if (!referenceMap) return undefined;
+            const result: MapLike<string[]> = {};
+            for (const [fileNamesKey, fileNamesListKey] of referenceMap) {
+                result[toFileName(fileNamesKey)] = toFileNames(fileNamesListKey);
+            }
+            return result;
+        }
+    }
+
     export function baselineBuildInfo(
         options: CompilerOptions,
         sys: System & { writtenFiles: ReadonlyCollection<string>; },
-        originalReadCall?: System["readFile"]
+        originalReadCall?: System["readFile"],
+        originalWriteFile?: System["writeFile"],
     ) {
-        const out = outFile(options);
-        if (!out) return;
-        const { buildInfoPath, jsFilePath, declarationFilePath } = getOutputPathsForBundle(options, /*forceDts*/ false);
+        const buildInfoPath = getTsBuildInfoEmitOutputFilePath(options);
         if (!buildInfoPath || !sys.writtenFiles.has(buildInfoPath)) return;
         if (!sys.fileExists(buildInfoPath)) return;
 
         const buildInfo = getBuildInfo((originalReadCall || sys.readFile).call(sys, buildInfoPath, "utf8")!);
+        generateBuildInfoProgramBaseline(sys, originalWriteFile || sys.writeFile, buildInfoPath, buildInfo);
+
+        if (!outFile(options)) return;
+        const { jsFilePath, declarationFilePath } = getOutputPathsForBundle(options, /*forceDts*/ false);
         const bundle = buildInfo.bundle;
         if (!bundle || (!length(bundle.js && bundle.js.sections) && !length(bundle.dts && bundle.dts.sections))) return;
 
@@ -256,9 +316,8 @@ interface Symbol {
         generateBundleFileSectionInfo(sys, originalReadCall || sys.readFile, baselineRecorder, bundle.js, jsFilePath);
         generateBundleFileSectionInfo(sys, originalReadCall || sys.readFile, baselineRecorder, bundle.dts, declarationFilePath);
         baselineRecorder.Close();
-
         const text = baselineRecorder.lines.join("\r\n");
-        sys.writeFile(`${buildInfoPath}.baseline.txt`, text);
+        (originalWriteFile || sys.writeFile).call(sys, `${buildInfoPath}.baseline.txt`, text);
     }
 
     interface VerifyIncrementalCorrectness {
@@ -295,7 +354,7 @@ interface Symbol {
                 const cleanBuildText = sys.readFile(outputFile);
                 const incrementalBuildText = newSys.readFile(outputFile);
                 const descrepancyInClean = discrepancies?.get(outputFile);
-                if (!isBuildInfoFile(outputFile)) {
+                if (!isBuildInfoFile(outputFile) && !fileExtensionIs(outputFile, ".tsbuildinfo.readable.baseline.txt")) {
                     verifyTextEqual(incrementalBuildText, cleanBuildText, descrepancyInClean, `File: ${outputFile}`);
                 }
                 else if (incrementalBuildText !== cleanBuildText) {

src/testRunner/unittests/tsbuild/outputPaths.ts

@@ -60,11 +60,10 @@ namespace ts {
                 noChangeRun,
                 {
                     ...noChangeProject,
-                    cleanBuildDiscrepancies: () => {
-                        const map = new Map<string, CleanBuildDescrepancy>();
-                        map.set("/src/dist/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent); // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
-                        return map;
-                    }
+                    cleanBuildDiscrepancies: () => new Map([
+                        ["/src/dist/tsconfig.tsbuildinfo", CleanBuildDescrepancy.CleanFileTextDifferent], // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
+                        ["/src/dist/tsconfig.tsbuildinfo.readable.baseline.txt", CleanBuildDescrepancy.CleanFileTextDifferent] // tsbuildinfo will have -p setting when built using -p vs no build happens incrementally because of no change.
+                    ]),
                 }
             ],
         }, ["/src/dist/src/index.js", "/src/dist/src/index.d.ts"]);