Skip to content

Commit bf21cfd

Browse files
Alexey AndreevAlexey Andreev
Alexey Andreev
authored and
Alexey Andreev
committed
Use source map remapper in JS inliner
1 parent 9c4ec90 commit bf21cfd

File tree

14 files changed

+345
-90
lines changed

14 files changed

+345
-90
lines changed

‎compiler/tests-common/org/jetbrains/kotlin/test/KotlinTestUtils.java

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ public class KotlinTestUtils {
124124
"(?://\\s*MODULE:\\s*([^()\\n]+)(?:\\(([^()]+(?:" + MODULE_DELIMITER + "[^()]+)*)\\))?\\s*(?:\\(([^()]+(?:" + MODULE_DELIMITER + "[^()]+)*)\\))?\\s*)?" +
125125
"//\\s*FILE:\\s*(.*)$", Pattern.MULTILINE);
126126
private static final Pattern DIRECTIVE_PATTERN = Pattern.compile("^//\\s*!([\\w_]+)(:\\s*(.*)$)?", Pattern.MULTILINE);
127+
private static final Pattern LINE_SEPARATOR_PATTERN = Pattern.compile("\\r\\n|\\r|\\n");
127128

128129
public static final BindingTrace DUMMY_TRACE = new BindingTrace() {
129130
@NotNull
@@ -648,6 +649,12 @@ public Void createModule(@NotNull String name, @NotNull List<String> dependencie
648649

649650
@NotNull
650651
public static <M, F> List<F> createTestFiles(String testFileName, String expectedText, TestFileFactory<M, F> factory) {
652+
return createTestFiles(testFileName, expectedText, factory, false);
653+
}
654+
655+
@NotNull
656+
public static <M, F> List<F> createTestFiles(String testFileName, String expectedText, TestFileFactory<M, F> factory,
657+
boolean preserveLocations) {
651658
Map<String, String> directives = parseDirectives(expectedText);
652659

653660
List<F> testFiles = Lists.newArrayList();
@@ -681,7 +688,9 @@ public static <M, F> List<F> createTestFiles(String testFileName, String expecte
681688
else {
682689
end = expectedText.length();
683690
}
684-
String fileText = expectedText.substring(start, end);
691+
String fileText = preserveLocations ?
692+
substringKeepingLocations(expectedText, start, end) :
693+
expectedText.substring(start,end);
685694
processedChars = end;
686695

687696
testFiles.add(factory.createFile(module, fileName, fileText, directives));
@@ -730,6 +739,26 @@ public static <M, F> List<F> createTestFiles(String testFileName, String expecte
730739
return testFiles;
731740
}
732741

742+
private static String substringKeepingLocations(String string, int start, int end) {
743+
Matcher matcher = LINE_SEPARATOR_PATTERN.matcher(string);
744+
StringBuilder prefix = new StringBuilder();
745+
int lastLineOffset = 0;
746+
while (matcher.find()) {
747+
if (matcher.end() > start) {
748+
break;
749+
}
750+
751+
lastLineOffset = matcher.end();
752+
prefix.append('\n');
753+
}
754+
755+
while (lastLineOffset++ < start) {
756+
prefix.append(' ');
757+
}
758+
759+
return prefix + string.substring(start, end);
760+
}
761+
733762
private static List<String> parseModuleList(@Nullable String dependencies) {
734763
if (dependencies == null) return Collections.emptyList();
735764
return StringsKt.split(dependencies, Pattern.compile(MODULE_DELIMITER), 0);

‎compiler/util/src/org/jetbrains/kotlin/utils/JsLibraryUtils.kt

Lines changed: 50 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import com.intellij.util.Processor
2222
import org.jetbrains.kotlin.utils.fileUtils.withReplacedExtensionOrNull
2323
import java.io.File
2424
import java.io.IOException
25+
import java.util.zip.ZipEntry
2526
import java.util.zip.ZipFile
2627

2728
object JsLibraryUtils {
@@ -43,49 +44,53 @@ object JsLibraryUtils {
4344
}
4445
}
4546

46-
@JvmStatic fun traverseJsLibraries(libs: List<File>, action: (content: String, path: String) -> Unit) {
47+
@JvmStatic fun traverseJsLibraries(libs: List<File>, action: (JsLibrary) -> Unit) {
4748
libs.forEach { traverseJsLibrary(it, action) }
4849
}
4950

50-
@JvmStatic fun traverseJsLibrary(lib: File, action: (content: String, path: String) -> Unit) {
51+
@JvmStatic fun traverseJsLibrary(lib: File, action: (JsLibrary) -> Unit) {
5152
when {
5253
lib.isDirectory -> traverseDirectory(lib, action)
5354
FileUtil.isJarOrZip(lib) -> traverseArchive(lib, action)
5455
lib.name.endsWith(KotlinJavascriptMetadataUtils.JS_EXT) -> {
55-
lib.runIfFileExists(action)
56+
lib.runIfFileExists(lib.path, action)
5657
val jsFile = lib.withReplacedExtensionOrNull(
5758
KotlinJavascriptMetadataUtils.META_JS_SUFFIX, KotlinJavascriptMetadataUtils.JS_EXT
5859
)
59-
jsFile?.runIfFileExists(action)
60+
jsFile?.runIfFileExists(jsFile.path, action)
6061
}
6162
}
6263
}
6364

64-
private fun File.runIfFileExists(action: (content: String, path: String) -> Unit) {
65+
private fun File.runIfFileExists(relativePath: String, action: (JsLibrary) -> Unit) {
6566
if (isFile) {
66-
action(FileUtil.loadFile(this), "")
67+
action(JsLibrary(readText(), relativePath, correspondingSourceMapFile().contentIfExists()))
6768
}
6869
}
6970

7071
private fun copyJsFilesFromDirectory(dir: File, outputLibraryJsPath: String) {
71-
traverseDirectory(dir) { content, relativePath ->
72-
FileUtil.writeToFile(File(outputLibraryJsPath, relativePath), content)
72+
traverseDirectory(dir) { (content, path) ->
73+
FileUtil.writeToFile(File(outputLibraryJsPath, path), content)
7374
}
7475
}
7576

76-
private fun processDirectory(dir: File, action: (content: String, relativePath: String) -> Unit) {
77+
private fun File.contentIfExists(): String? = if (exists()) readText() else null
78+
79+
private fun File.correspondingSourceMapFile(): File = File(parentFile, name + ".map")
80+
81+
private fun processDirectory(dir: File, action: (JsLibrary) -> Unit) {
7782
FileUtil.processFilesRecursively(dir, Processor<File> { file ->
7883
val relativePath = FileUtil.getRelativePath(dir, file)
7984
?: throw IllegalArgumentException("relativePath should not be null $dir $file")
80-
if (file.isFile && relativePath.endsWith(KotlinJavascriptMetadataUtils.JS_EXT)) {
85+
if (relativePath.endsWith(KotlinJavascriptMetadataUtils.JS_EXT)) {
8186
val suggestedRelativePath = getSuggestedPath(relativePath) ?: return@Processor true
82-
action(FileUtil.loadFile(file), suggestedRelativePath)
87+
file.runIfFileExists(suggestedRelativePath, action)
8388
}
8489
true
8590
})
8691
}
8792

88-
private fun traverseDirectory(dir: File, action: (content: String, relativePath: String) -> Unit) {
93+
private fun traverseDirectory(dir: File, action: (JsLibrary) -> Unit) {
8994
try {
9095
processDirectory(dir, action)
9196
}
@@ -95,26 +100,48 @@ object JsLibraryUtils {
95100
}
96101

97102
private fun copyJsFilesFromZip(file: File, outputLibraryJsPath: String) {
98-
traverseArchive(file) { content, relativePath ->
99-
FileUtil.writeToFile(File(outputLibraryJsPath, relativePath), content)
103+
traverseArchive(file) { library ->
104+
FileUtil.writeToFile(File(outputLibraryJsPath, library.path), library.content)
100105
}
101106
}
102107

103-
private fun traverseArchive(file: File, action: (content: String, relativePath: String) -> Unit) {
108+
private fun traverseArchive(file: File, action: (JsLibrary) -> Unit) {
104109
val zipFile = ZipFile(file.path)
105110
try {
106111
val zipEntries = zipFile.entries()
112+
val librariesWithoutSourceMaps = mutableListOf<JsLibrary>()
113+
val possibleMapFiles = mutableMapOf<String, ZipEntry>()
114+
107115
while (zipEntries.hasMoreElements()) {
108116
val entry = zipEntries.nextElement()
109117
val entryName = entry.name
110-
if (!entry.isDirectory && entryName.endsWith(KotlinJavascriptMetadataUtils.JS_EXT)) {
111-
val relativePath = getSuggestedPath(entryName) ?: continue
112-
113-
val stream = zipFile.getInputStream(entry)
114-
val content = FileUtil.loadTextAndClose(stream)
115-
action(content, relativePath)
118+
if (!entry.isDirectory) {
119+
if (entryName.endsWith(KotlinJavascriptMetadataUtils.JS_EXT)) {
120+
val relativePath = getSuggestedPath(entryName) ?: continue
121+
122+
val stream = zipFile.getInputStream(entry)
123+
val content = FileUtil.loadTextAndClose(stream)
124+
librariesWithoutSourceMaps += JsLibrary(content, relativePath, null)
125+
}
126+
else if (entryName.endsWith(KotlinJavascriptMetadataUtils.JS_MAP_EXT)) {
127+
possibleMapFiles[entryName.removeSuffix(KotlinJavascriptMetadataUtils.JS_MAP_EXT)] = entry
128+
}
116129
}
117130
}
131+
132+
librariesWithoutSourceMaps
133+
.map {
134+
val zipEntry = possibleMapFiles[it.path]
135+
if (zipEntry != null) {
136+
val stream = zipFile.getInputStream(zipEntry)
137+
val content = FileUtil.loadTextAndClose(stream)
138+
it.copy(sourceMapContent = content)
139+
}
140+
else {
141+
it
142+
}
143+
}
144+
.forEach(action)
118145
}
119146
catch (ex: IOException) {
120147
LOG.error("Could not extract files from archive ${file.name}: ${ex.message}")
@@ -136,3 +163,5 @@ object JsLibraryUtils {
136163
return path
137164
}
138165
}
166+
167+
data class JsLibrary(val content: String, val path: String, val sourceMapContent: String?)

‎compiler/util/src/org/jetbrains/kotlin/utils/KotlinJavascriptMetadataUtils.kt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ class JsMetadataVersion(vararg numbers: Int) : BinaryVersion(*numbers) {
5757
object KotlinJavascriptMetadataUtils {
5858
const val JS_EXT: String = ".js"
5959
const val META_JS_SUFFIX: String = ".meta.js"
60+
const val JS_MAP_EXT: String = ".js.map"
6061
private val KOTLIN_JAVASCRIPT_METHOD_NAME = "kotlin_module_metadata"
6162
private val KOTLIN_JAVASCRIPT_METHOD_NAME_PATTERN = "\\.kotlin_module_metadata\\(".toPattern()
6263

@@ -78,8 +79,8 @@ object KotlinJavascriptMetadataUtils {
7879
fun loadMetadata(file: File): List<KotlinJavascriptMetadata> {
7980
assert(file.exists()) { "Library $file not found" }
8081
val metadataList = arrayListOf<KotlinJavascriptMetadata>()
81-
JsLibraryUtils.traverseJsLibrary(file) { content, _ ->
82-
parseMetadata(content, metadataList)
82+
JsLibraryUtils.traverseJsLibrary(file) { library ->
83+
parseMetadata(library.content, metadataList)
8384
}
8485

8586
return metadataList

‎js/js.inliner/src/org/jetbrains/kotlin/js/inline/FunctionReader.kt

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,17 @@ import org.jetbrains.kotlin.js.backend.ast.metadata.inlineStrategy
2626
import org.jetbrains.kotlin.js.config.JsConfig
2727
import org.jetbrains.kotlin.js.inline.util.IdentitySet
2828
import org.jetbrains.kotlin.js.inline.util.isCallInvocation
29+
import org.jetbrains.kotlin.js.parser.OffsetToSourceMapping
2930
import org.jetbrains.kotlin.js.parser.parseFunction
31+
import org.jetbrains.kotlin.js.parser.sourcemaps.*
3032
import org.jetbrains.kotlin.js.translate.context.Namer
3133
import org.jetbrains.kotlin.js.translate.utils.JsDescriptorUtils.getModuleName
3234
import org.jetbrains.kotlin.resolve.descriptorUtil.isExtension
3335
import org.jetbrains.kotlin.resolve.inline.InlineStrategy
3436
import org.jetbrains.kotlin.utils.JsLibraryUtils
3537
import org.jetbrains.kotlin.utils.sure
3638
import java.io.File
39+
import java.io.StringReader
3740

3841
// TODO: add hash checksum to defineModule?
3942
/**
@@ -57,7 +60,14 @@ class FunctionReader(private val config: JsConfig, private val currentModuleName
5760
* kotlinVariable: kotlin object variable.
5861
* The default variable is Kotlin, but it can be renamed by minifier.
5962
*/
60-
data class ModuleInfo(val filePath: String, val fileContent: String, val moduleVariable: String, val kotlinVariable: String)
63+
class ModuleInfo(
64+
val filePath: String,
65+
val fileContent: String,
66+
val moduleVariable: String,
67+
val kotlinVariable: String,
68+
val offsetToSourceMapping: OffsetToSourceMapping,
69+
val sourceMap: SourceMap?
70+
)
6171

6272
private val moduleNameToInfo = HashMultimap.create<String, ModuleInfo>()
6373

@@ -68,22 +78,40 @@ class FunctionReader(private val config: JsConfig, private val currentModuleName
6878

6979
moduleNameMap = buildModuleNameMap(fragments)
7080

71-
JsLibraryUtils.traverseJsLibraries(libs) { fileContent, filePath ->
81+
JsLibraryUtils.traverseJsLibraries(libs) { (content, path, sourceMapContent) ->
7282
var current = 0
7383

7484
while (true) {
75-
var index = fileContent.indexOf(DEFINE_MODULE_FIND_PATTERN, current)
85+
var index = content.indexOf(DEFINE_MODULE_FIND_PATTERN, current)
7686
if (index < 0) break
7787

7888
current = index + 1
79-
index = rewindToIdentifierStart(fileContent, index)
80-
val preciseMatcher = DEFINE_MODULE_PATTERN.matcher(offset(fileContent, index))
89+
index = rewindToIdentifierStart(content, index)
90+
val preciseMatcher = DEFINE_MODULE_PATTERN.matcher(offset(content, index))
8191
if (!preciseMatcher.lookingAt()) continue
8292

8393
val moduleName = preciseMatcher.group(3)
8494
val moduleVariable = preciseMatcher.group(4)
8595
val kotlinVariable = preciseMatcher.group(1)
86-
moduleNameToInfo.put(moduleName, ModuleInfo(filePath, fileContent, moduleVariable, kotlinVariable))
96+
97+
val sourceMap = sourceMapContent?.let {
98+
val result = SourceMapParser.parse(StringReader(it))
99+
when (result) {
100+
is SourceMapSuccess -> result.value
101+
is SourceMapError -> throw RuntimeException("Error parsing source map file: ${result.message}\n$it")
102+
}
103+
}
104+
105+
val moduleInfo = ModuleInfo(
106+
filePath = path,
107+
fileContent = content,
108+
moduleVariable = moduleVariable,
109+
kotlinVariable = kotlinVariable,
110+
offsetToSourceMapping = OffsetToSourceMapping(content),
111+
sourceMap = sourceMap
112+
)
113+
114+
moduleNameToInfo.put(moduleName, moduleInfo)
87115
}
88116
}
89117
}
@@ -152,9 +180,16 @@ class FunctionReader(private val config: JsConfig, private val currentModuleName
152180
offset++
153181
}
154182

155-
val function = parseFunction(source, info.filePath, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram()))
183+
val position = info.offsetToSourceMapping[offset]
184+
val function = parseFunction(source, info.filePath, position, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram()))
156185
val moduleReference = moduleNameMap[tag] ?: currentModuleName.makeRef()
157186

187+
val sourceMap = info.sourceMap
188+
if (sourceMap != null) {
189+
val remapper = SourceMapLocationRemapper(mapOf(info.filePath to sourceMap))
190+
remapper.remap(function)
191+
}
192+
158193
val replacements = hashMapOf(info.moduleVariable to moduleReference,
159194
info.kotlinVariable to Namer.kotlinObject())
160195
replaceExternalNames(function, replacements)

‎js/js.parser/src/com/google/gwt/dev/js/rhino/LineBuffer.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,10 @@ final class LineBuffer {
6464
*/
6565
static final int BUFLEN = 256;
6666

67-
LineBuffer(Reader in, int lineno) {
67+
LineBuffer(Reader in, CodePosition position) {
6868
this.in = in;
69-
this.lineno = lineno;
69+
this.lineno = position.getLine();
70+
this.lineStart = -position.getOffset();
7071
}
7172

7273
int read() throws IOException {

‎js/js.parser/src/com/google/gwt/dev/js/rhino/TokenStream.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -495,14 +495,15 @@ private int stringToKeyword(String name) {
495495
}
496496

497497
public TokenStream(Reader in,
498-
String sourceName, int lineno)
498+
String sourceName, CodePosition position)
499499
{
500-
this.in = new LineBuffer(in, lineno);
500+
this.in = new LineBuffer(in, position);
501501
this.pushbackToken = EOF;
502502
this.sourceName = sourceName;
503503
flags = 0;
504-
secondToLastPosition = new CodePosition(lineno, 0);
505-
lastPosition = new CodePosition(lineno, 0);
504+
secondToLastPosition = position;
505+
lastPosition = position;
506+
lastTokenPosition = position;
506507
}
507508

508509
/* return and pop the token from the stream if it matches...

‎js/js.parser/src/org/jetbrains/kotlin/js/parser/OffsetToSourceMapping.kt

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright 2010-2017 JetBrains s.r.o.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.jetbrains.kotlin.js.parser
18+
19+
import com.google.gwt.dev.js.rhino.CodePosition
20+
21+
class OffsetToSourceMapping(text: String) {
22+
private val data: IntArray
23+
24+
init {
25+
val lineSeparators = LINE_SEPARATOR.findAll(text).map { it.range.endInclusive + 1 }
26+
data = (sequenceOf(0) + lineSeparators).toList().toIntArray()
27+
}
28+
29+
operator fun get(offset: Int): CodePosition {
30+
val lineNumber = data.binarySearch(offset).let { if (it >= 0) it else -it - 2 }
31+
return CodePosition(lineNumber, offset - data[lineNumber])
32+
}
33+
34+
private companion object {
35+
private val LINE_SEPARATOR = Regex("\\r\\n|\\r|\\n")
36+
}
37+
}

0 commit comments

Comments
 (0)