Skip to content

Commit b823726

Browse files
author
Angular Builds
committed
2dc885304 refactor(@angular-devkit/build-angular): reorganize bundle processing for browser builder (#15776)
1 parent 7348436 commit b823726

File tree

7 files changed

+282
-219
lines changed

7 files changed

+282
-219
lines changed

package.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
{
22
"name": "@angular-devkit/build-angular",
3-
"version": "0.900.0-next.8+17.c0d42e0",
3+
"version": "0.900.0-next.8+18.2dc8853",
44
"description": "Angular Webpack Build Facade",
55
"experimental": true,
66
"main": "src/index.js",
77
"typings": "src/index.d.ts",
88
"builders": "builders.json",
99
"dependencies": {
10-
"@angular-devkit/architect": "github:angular/angular-devkit-architect-builds#c0d42e0c0",
11-
"@angular-devkit/build-optimizer": "github:angular/angular-devkit-build-optimizer-builds#c0d42e0c0",
12-
"@angular-devkit/build-webpack": "github:angular/angular-devkit-build-webpack-builds#c0d42e0c0",
13-
"@angular-devkit/core": "github:angular/angular-devkit-core-builds#c0d42e0c0",
10+
"@angular-devkit/architect": "github:angular/angular-devkit-architect-builds#2dc885304",
11+
"@angular-devkit/build-optimizer": "github:angular/angular-devkit-build-optimizer-builds#2dc885304",
12+
"@angular-devkit/build-webpack": "github:angular/angular-devkit-build-webpack-builds#2dc885304",
13+
"@angular-devkit/core": "github:angular/angular-devkit-core-builds#2dc885304",
1414
"@babel/core": "7.6.3",
1515
"@babel/preset-env": "7.6.3",
16-
"@ngtools/webpack": "github:angular/ngtools-webpack-builds#c0d42e0c0",
16+
"@ngtools/webpack": "github:angular/ngtools-webpack-builds#2dc885304",
1717
"ajv": "6.10.2",
1818
"autoprefixer": "9.6.4",
1919
"browserslist": "4.7.0",

src/browser/action-cache.d.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/// <reference types="node" />
2+
import * as fs from 'fs';
3+
import { ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
4+
export interface CacheEntry {
5+
path: string;
6+
size: number;
7+
integrity?: string;
8+
}
9+
export declare class BundleActionCache {
10+
private readonly integrityAlgorithm?;
11+
constructor(integrityAlgorithm?: string | undefined);
12+
static copyEntryContent(entry: CacheEntry | string, dest: fs.PathLike): void;
13+
generateBaseCacheKey(content: string): string;
14+
generateCacheKeys(action: ProcessBundleOptions): string[];
15+
getCacheEntries(cacheKeys: (string | null)[]): Promise<(CacheEntry | null)[] | false>;
16+
getCachedBundleResult(action: ProcessBundleOptions): Promise<ProcessBundleResult | null>;
17+
}

src/browser/action-cache.js

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
"use strict";
2+
Object.defineProperty(exports, "__esModule", { value: true });
3+
/**
4+
* @license
5+
* Copyright Google Inc. All Rights Reserved.
6+
*
7+
* Use of this source code is governed by an MIT-style license that can be
8+
* found in the LICENSE file at https://angular.io/license
9+
*/
10+
const crypto_1 = require("crypto");
11+
const findCacheDirectory = require("find-cache-dir");
12+
const fs = require("fs");
13+
const mangle_options_1 = require("../utils/mangle-options");
14+
const cacache = require('cacache');
15+
const cacheDownlevelPath = findCacheDirectory({ name: 'angular-build-dl' });
16+
const packageVersion = require('../../package.json').version;
17+
// Workaround Node.js issue prior to 10.16 with copyFile on macOS
18+
// https://github.com/angular/angular-cli/issues/15544 & https://github.com/nodejs/node/pull/27241
19+
let copyFileWorkaround = false;
20+
if (process.platform === 'darwin') {
21+
const version = process.versions.node.split('.').map(part => Number(part));
22+
if (version[0] < 10 || version[0] === 11 || (version[0] === 10 && version[1] < 16)) {
23+
copyFileWorkaround = true;
24+
}
25+
}
26+
class BundleActionCache {
27+
constructor(integrityAlgorithm) {
28+
this.integrityAlgorithm = integrityAlgorithm;
29+
}
30+
static copyEntryContent(entry, dest) {
31+
if (copyFileWorkaround) {
32+
try {
33+
fs.unlinkSync(dest);
34+
}
35+
catch (_a) { }
36+
}
37+
fs.copyFileSync(typeof entry === 'string' ? entry : entry.path, dest, fs.constants.COPYFILE_FICLONE);
38+
if (process.platform !== 'win32') {
39+
// The cache writes entries as readonly and when using copyFile the permissions will also be copied.
40+
// See: https://github.com/npm/cacache/blob/073fbe1a9f789ba42d9a41de7b8429c93cf61579/lib/util/move-file.js#L36
41+
fs.chmodSync(dest, 0o644);
42+
}
43+
}
44+
generateBaseCacheKey(content) {
45+
// Create base cache key with elements:
46+
// * package version - different build-angular versions cause different final outputs
47+
// * code length/hash - ensure cached version matches the same input code
48+
const algorithm = this.integrityAlgorithm || 'sha1';
49+
const codeHash = crypto_1.createHash(algorithm)
50+
.update(content)
51+
.digest('base64');
52+
let baseCacheKey = `${packageVersion}|${content.length}|${algorithm}-${codeHash}`;
53+
if (mangle_options_1.manglingDisabled) {
54+
baseCacheKey += '|MD';
55+
}
56+
return baseCacheKey;
57+
}
58+
generateCacheKeys(action) {
59+
const baseCacheKey = this.generateBaseCacheKey(action.code);
60+
// Postfix added to sourcemap cache keys when vendor sourcemaps are present
61+
// Allows non-destructive caching of both variants
62+
const SourceMapVendorPostfix = !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : '';
63+
// Determine cache entries required based on build settings
64+
const cacheKeys = [];
65+
// If optimizing and the original is not ignored, add original as required
66+
if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) {
67+
cacheKeys[0 /* OriginalCode */] = baseCacheKey + '|orig';
68+
// If sourcemaps are enabled, add original sourcemap as required
69+
if (action.sourceMaps) {
70+
cacheKeys[1 /* OriginalMap */] = baseCacheKey + SourceMapVendorPostfix + '|orig-map';
71+
}
72+
}
73+
// If not only optimizing, add downlevel as required
74+
if (!action.optimizeOnly) {
75+
cacheKeys[2 /* DownlevelCode */] = baseCacheKey + '|dl';
76+
// If sourcemaps are enabled, add downlevel sourcemap as required
77+
if (action.sourceMaps) {
78+
cacheKeys[3 /* DownlevelMap */] = baseCacheKey + SourceMapVendorPostfix + '|dl-map';
79+
}
80+
}
81+
return cacheKeys;
82+
}
83+
async getCacheEntries(cacheKeys) {
84+
// Attempt to get required cache entries
85+
const cacheEntries = [];
86+
for (const key of cacheKeys) {
87+
if (key) {
88+
const entry = await cacache.get.info(cacheDownlevelPath, key);
89+
if (!entry) {
90+
return false;
91+
}
92+
cacheEntries.push({
93+
path: entry.path,
94+
size: entry.size,
95+
integrity: entry.metadata && entry.metadata.integrity,
96+
});
97+
}
98+
else {
99+
cacheEntries.push(null);
100+
}
101+
}
102+
return cacheEntries;
103+
}
104+
async getCachedBundleResult(action) {
105+
const entries = action.cacheKeys && await this.getCacheEntries(action.cacheKeys);
106+
if (!entries) {
107+
return null;
108+
}
109+
const result = { name: action.name };
110+
let cacheEntry = entries[0 /* OriginalCode */];
111+
if (cacheEntry) {
112+
result.original = {
113+
filename: action.filename,
114+
size: cacheEntry.size,
115+
integrity: cacheEntry.integrity,
116+
};
117+
BundleActionCache.copyEntryContent(cacheEntry, result.original.filename);
118+
cacheEntry = entries[1 /* OriginalMap */];
119+
if (cacheEntry) {
120+
result.original.map = {
121+
filename: action.filename + '.map',
122+
size: cacheEntry.size,
123+
};
124+
BundleActionCache.copyEntryContent(cacheEntry, result.original.filename + '.map');
125+
}
126+
}
127+
else if (!action.ignoreOriginal) {
128+
// If the original wasn't processed (and therefore not cached), add info
129+
result.original = {
130+
filename: action.filename,
131+
size: Buffer.byteLength(action.code, 'utf8'),
132+
map: action.map === undefined
133+
? undefined
134+
: {
135+
filename: action.filename + '.map',
136+
size: Buffer.byteLength(action.map, 'utf8'),
137+
},
138+
};
139+
}
140+
cacheEntry = entries[2 /* DownlevelCode */];
141+
if (cacheEntry) {
142+
result.downlevel = {
143+
filename: action.filename.replace('es2015', 'es5'),
144+
size: cacheEntry.size,
145+
integrity: cacheEntry.integrity,
146+
};
147+
BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename);
148+
cacheEntry = entries[3 /* DownlevelMap */];
149+
if (cacheEntry) {
150+
result.downlevel.map = {
151+
filename: action.filename.replace('es2015', 'es5') + '.map',
152+
size: cacheEntry.size,
153+
};
154+
BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename + '.map');
155+
}
156+
}
157+
return result;
158+
}
159+
}
160+
exports.BundleActionCache = BundleActionCache;

src/browser/action-executor.d.ts

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
1-
export declare class ActionExecutor<Input extends {
2-
size: number;
3-
}, Output> {
4-
private readonly actionName;
5-
private largeWorker;
6-
private smallWorker;
7-
private smallThreshold;
8-
constructor(actionFile: string, actionName: string, setupOptions?: unknown);
9-
execute(options: Input): Promise<Output>;
10-
executeAll(options: Input[]): Promise<Output[]>;
1+
import { ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle';
2+
export declare class BundleActionExecutor {
3+
private workerOptions;
4+
private readonly sizeThreshold;
5+
private largeWorker?;
6+
private smallWorker?;
7+
private cache;
8+
constructor(workerOptions: unknown, integrityAlgorithm?: string, sizeThreshold?: number);
9+
private static executeMethod;
10+
private ensureLarge;
11+
private ensureSmall;
12+
private executeAction;
13+
process(action: ProcessBundleOptions): Promise<ProcessBundleResult>;
14+
processAll(actions: Iterable<ProcessBundleOptions>): AsyncIterableIterator<ProcessBundleResult>;
1115
stop(): void;
1216
}

src/browser/action-executor.js

Lines changed: 70 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,39 +9,88 @@ Object.defineProperty(exports, "__esModule", { value: true });
99
*/
1010
const jest_worker_1 = require("jest-worker");
1111
const os = require("os");
12-
class ActionExecutor {
13-
constructor(actionFile, actionName, setupOptions) {
14-
this.actionName = actionName;
15-
this.smallThreshold = 32 * 1024;
12+
const path = require("path");
13+
const action_cache_1 = require("./action-cache");
14+
let workerFile = require.resolve('../utils/process-bundle');
15+
workerFile =
16+
path.extname(workerFile) === '.ts'
17+
? require.resolve('../utils/process-bundle-bootstrap')
18+
: workerFile;
19+
class BundleActionExecutor {
20+
constructor(workerOptions, integrityAlgorithm, sizeThreshold = 32 * 1024) {
21+
this.workerOptions = workerOptions;
22+
this.sizeThreshold = sizeThreshold;
23+
this.cache = new action_cache_1.BundleActionCache(integrityAlgorithm);
24+
}
25+
static executeMethod(worker, method, input) {
26+
return worker[method](input);
27+
}
28+
ensureLarge() {
29+
if (this.largeWorker) {
30+
return this.largeWorker;
31+
}
1632
// larger files are processed in a separate process to limit memory usage in the main process
17-
this.largeWorker = new jest_worker_1.default(actionFile, {
18-
exposedMethods: [actionName],
19-
setupArgs: setupOptions === undefined ? undefined : [setupOptions],
20-
});
33+
return (this.largeWorker = new jest_worker_1.default(workerFile, {
34+
exposedMethods: ['process'],
35+
setupArgs: [this.workerOptions],
36+
}));
37+
}
38+
ensureSmall() {
39+
if (this.smallWorker) {
40+
return this.smallWorker;
41+
}
2142
// small files are processed in a limited number of threads to improve speed
2243
// The limited number also prevents a large increase in memory usage for an otherwise short operation
23-
this.smallWorker = new jest_worker_1.default(actionFile, {
24-
exposedMethods: [actionName],
25-
setupArgs: setupOptions === undefined ? undefined : [setupOptions],
44+
return (this.smallWorker = new jest_worker_1.default(workerFile, {
45+
exposedMethods: ['process'],
46+
setupArgs: [this.workerOptions],
2647
numWorkers: os.cpus().length < 2 ? 1 : 2,
2748
// Will automatically fallback to processes if not supported
2849
enableWorkerThreads: true,
29-
});
50+
}));
3051
}
31-
execute(options) {
32-
if (options.size > this.smallThreshold) {
33-
return this.largeWorker[this.actionName](options);
52+
executeAction(method, action) {
53+
// code.length is not an exact byte count but close enough for this
54+
if (action.code.length > this.sizeThreshold) {
55+
return BundleActionExecutor.executeMethod(this.ensureLarge(), method, action);
3456
}
3557
else {
36-
return this.smallWorker[this.actionName](options);
58+
return BundleActionExecutor.executeMethod(this.ensureSmall(), method, action);
3759
}
3860
}
39-
executeAll(options) {
40-
return Promise.all(options.map(o => this.execute(o)));
61+
async process(action) {
62+
const cacheKeys = this.cache.generateCacheKeys(action);
63+
action.cacheKeys = cacheKeys;
64+
// Try to get cached data, if it fails fallback to processing
65+
try {
66+
const cachedResult = await this.cache.getCachedBundleResult(action);
67+
if (cachedResult) {
68+
return cachedResult;
69+
}
70+
}
71+
catch (_a) { }
72+
return this.executeAction('process', action);
73+
}
74+
async *processAll(actions) {
75+
const executions = new Map();
76+
for (const action of actions) {
77+
const execution = this.process(action);
78+
executions.set(execution, execution.then(result => {
79+
executions.delete(execution);
80+
return result;
81+
}));
82+
}
83+
while (executions.size > 0) {
84+
yield Promise.race(executions.values());
85+
}
4186
}
4287
stop() {
43-
this.largeWorker.end();
44-
this.smallWorker.end();
88+
if (this.largeWorker) {
89+
this.largeWorker.end();
90+
}
91+
if (this.smallWorker) {
92+
this.smallWorker.end();
93+
}
4594
}
4695
}
47-
exports.ActionExecutor = ActionExecutor;
96+
exports.BundleActionExecutor = BundleActionExecutor;

0 commit comments

Comments
 (0)