Merge pull request tree-sitter#822 from tree-sitter/wasm-fixes

Update to the latest Emscripten and fix some WASM issues

Author: maxbrunsfeld

.travis.yml

@@ -14,8 +14,8 @@ matrix:
 
 before_install:
   # Install node
-  - nvm install 10
-  - nvm use 10
+  - nvm install 12
+  - nvm use 12
 
   # Download emscripten and create a shorthand for adding it to the PATH.
   # Don't add it to the path globally because it overrides the default
@@ -58,8 +58,6 @@ deploy:
   file_glob: true
   file:
     - "tree-sitter-*.gz"
-    - "target/release/tree-sitter.js"
-    - "target/release/tree-sitter.wasm"
   draft: true
   overwrite: true
   skip_cleanup: true

cli/src/wasm.rs

@@ -57,7 +57,7 @@ pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Resu
         }
 
         // Run `emcc` in a container using the `emscripten-slim` image
-        command.args(&["trzeci/emscripten-slim", "emcc"]);
+        command.args(&["emscripten/emsdk", "emcc"]);
     } else {
         return Error::err(
             "You must have either emcc or docker on your PATH to run this command".to_string(),

lib/binding_web/binding.c

@@ -115,18 +115,10 @@ extern void tree_sitter_parse_callback(
 );
 
 extern void tree_sitter_log_callback(
-  void *payload,
-  TSLogType log_type,
+  bool is_lex_message,
   const char *message
 );
 
-void ts_parser_new_wasm() {
-  TSParser *parser = ts_parser_new();
-  char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char));
-  TRANSFER_BUFFER[0] = parser;
-  TRANSFER_BUFFER[1] = input_buffer;
-}
-
 static const char *call_parse_callback(
   void *payload,
   uint32_t byte,
@@ -148,8 +140,23 @@ static const char *call_parse_callback(
   return buffer;
 }
 
+static void call_log_callback(
+  void *payload,
+  TSLogType log_type,
+  const char *message
+) {
+  tree_sitter_log_callback(log_type == TSLogTypeLex, message);
+}
+
+void ts_parser_new_wasm() {
+  TSParser *parser = ts_parser_new();
+  char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char));
+  TRANSFER_BUFFER[0] = parser;
+  TRANSFER_BUFFER[1] = input_buffer;
+}
+
 void ts_parser_enable_logger_wasm(TSParser *self, bool should_log) {
-  TSLogger logger = {self, should_log ? tree_sitter_log_callback : NULL};
+  TSLogger logger = {self, should_log ? call_log_callback : NULL};
   ts_parser_set_logger(self, logger);
 }
 

lib/binding_web/binding.js

@@ -7,7 +7,6 @@ const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT;
 const ZERO_POINT = {row: 0, column: 0};
 const QUERY_WORD_REGEX = /[\w-.]*/g;
 
-const PREDICATE_STEP_TYPE_DONE = 0;
 const PREDICATE_STEP_TYPE_CAPTURE = 1;
 const PREDICATE_STEP_TYPE_STRING = 2;
 
@@ -880,7 +879,7 @@ class Language {
     }
 
     return bytes
-      .then(bytes => loadWebAssemblyModule(bytes, {loadAsync: true}))
+      .then(bytes => loadSideModule(bytes, {loadAsync: true}))
       .then(mod => {
         const symbolNames = Object.keys(mod)
         const functionName = symbolNames.find(key =>
@@ -1140,7 +1139,3 @@ function marshalEdit(edit) {
 }
 
 Parser.Language = Language;
-
-return Parser;
-
-}));

lib/binding_web/imports.js

@@ -16,7 +16,7 @@ mergeInto(LibraryManager.library, {
     }
   },
 
-  tree_sitter_log_callback: function(_payload, isLexMessage, messageAddress) {
+  tree_sitter_log_callback: function(isLexMessage, messageAddress) {
     if (currentLogCallback) {
       const message = UTF8ToString(messageAddress);
       currentLogCallback(message, isLexMessage !== 0);

lib/binding_web/suffix.js

@@ -0,0 +1,2 @@
+return Parser;
+}));

script/build-wasm

@@ -30,7 +30,6 @@ EOF
 set -e
 
 web_dir=lib/binding_web
-exports=$(cat ${web_dir}/exports.json)
 emscripten_flags="-O3"
 minify_js=1
 force_docker=0
@@ -64,11 +63,11 @@ emcc=
 if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then
   emcc=emcc
 elif which docker > /dev/null; then
-  emcc="docker run               \
-    --rm                         \
-    -v $(pwd):/src:Z             \
-    -u $(id -u)                  \
-    trzeci/emscripten-slim       \
+  emcc="docker run    \
+    --rm              \
+    -v $(pwd):/src:Z  \
+    -u $(id -u)       \
+    emscripten/emsdk  \
     emcc"
 else
   echo 'You must have either `docker` or `emcc` on your PATH to run this script'
@@ -79,24 +78,27 @@ mkdir -p target/scratch
 
 # Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm`
 # in the `target/scratch` directory
-$emcc                                \
-  -s WASM=1                          \
-  -s TOTAL_MEMORY=33554432           \
-  -s ALLOW_MEMORY_GROWTH=1           \
-  -s MAIN_MODULE=2                   \
-  -s NO_FILESYSTEM=1                 \
-  -s "EXPORTED_FUNCTIONS=${exports}" \
-  $emscripten_flags                  \
-  -std=c99                           \
-  -D 'fprintf(...)='                 \
-  -D NDEBUG=                         \
-  -I lib/src                         \
-  -I lib/include                     \
-  --js-library ${web_dir}/imports.js \
-  --pre-js ${web_dir}/prefix.js      \
-  --post-js ${web_dir}/binding.js    \
-  lib/src/lib.c                      \
-  ${web_dir}/binding.c               \
+$emcc                                            \
+  -s WASM=1                                      \
+  -s TOTAL_MEMORY=33554432                       \
+  -s ALLOW_MEMORY_GROWTH=1                       \
+  -s MAIN_MODULE=2                               \
+  -s NO_FILESYSTEM=1                             \
+  -s NODEJS_CATCH_EXIT=0                         \
+  -s NODEJS_CATCH_REJECTION=0                    \
+  -s EXPORTED_FUNCTIONS=@${web_dir}/exports.json \
+  $emscripten_flags                              \
+  -std=c99                                       \
+  -D 'fprintf(...)='                             \
+  -D NDEBUG=                                     \
+  -I lib/src                                     \
+  -I lib/include                                 \
+  --js-library ${web_dir}/imports.js             \
+  --pre-js ${web_dir}/prefix.js                  \
+  --post-js ${web_dir}/binding.js                \
+  --post-js ${web_dir}/suffix.js                 \
+  lib/src/lib.c                                  \
+  ${web_dir}/binding.c                           \
   -o target/scratch/tree-sitter.js
 
 # Use terser to write a minified version of `tree-sitter.js` into

script/fetch-emscripten

@@ -2,7 +2,7 @@
 
 set -e
 
-EMSCRIPTEN_VERSION=1.39.15
+EMSCRIPTEN_VERSION=2.0.9
 
 mkdir -p target
 EMSDK_DIR="./target/emsdk"

script/generate-fixtures-wasm

@@ -4,6 +4,12 @@ set -e
 
 cargo build --release
 
+build_wasm_args=
+if [[ $1 == "--docker" ]]; then
+  build_wasm_args="--docker"
+  shift
+fi
+
 filter_grammar_name=$1
 
 root_dir=$PWD
@@ -20,7 +26,7 @@ while read -r grammar_file; do
   fi
 
   echo "Compiling ${grammar_name} parser to wasm"
-  "$tree_sitter" build-wasm $grammar_dir
+  "$tree_sitter" build-wasm $build_wasm_args $grammar_dir
 done <<< "$grammar_files"
 
 mv tree-sitter-*.wasm target/release/