Add a CLI feature flag for wasm runtime support

Author: maxbrunsfeld

cli/Cargo.toml

@@ -20,6 +20,9 @@ doc = false
 name = "benchmark"
 harness = false
 
+[features]
+wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
+
 [dependencies]
 ansi_term = "0.12.1"
 anyhow = "1.0.72"
@@ -49,7 +52,6 @@ which = "4.4.0"
 [dependencies.tree-sitter]
 version = "0.20.10"
 path = "../lib"
-features = ["wasm"]
 
 [dependencies.tree-sitter-config]
 version = "0.19.0"
@@ -62,7 +64,6 @@ path = "../highlight"
 [dependencies.tree-sitter-loader]
 version = "0.20"
 path = "loader"
-features = ["wasm"]
 
 [dependencies.tree-sitter-tags]
 version = "0.20"

cli/loader/src/lib.rs

@@ -354,29 +354,23 @@ impl Loader {
             lib_name.push_str(".debug._");
         }
 
-        let mut library_path = self.parser_lib_path.join(lib_name);
         fs::create_dir_all(&self.parser_lib_path)?;
 
+        let mut library_path = self.parser_lib_path.join(lib_name);
+        library_path.set_extension(DYLIB_EXTENSION);
+
         let parser_path = src_path.join("parser.c");
-        let mut scanner_path = None;
-        let mut try_scanner_path = src_path.join("scanner.c");
-        for extension in ["c", "cc", "cpp"] {
-            try_scanner_path.set_extension(extension);
-            if try_scanner_path.exists() {
-                scanner_path = Some(try_scanner_path);
-                break;
-            }
-        }
+        let scanner_path = self.get_scanner_path(&src_path);
 
+        #[cfg(feature = "wasm")]
         if self.wasm_store.lock().unwrap().is_some() {
             library_path.set_extension("wasm");
-        } else {
-            library_path.set_extension(DYLIB_EXTENSION);
         }
 
         let recompile = needs_recompile(&library_path, &parser_path, scanner_path.as_deref())
             .with_context(|| "Failed to compare source and binary timestamps")?;
 
+        #[cfg(feature = "wasm")]
         if let Some(wasm_store) = self.wasm_store.lock().unwrap().as_mut() {
             if recompile {
                 self.compile_parser_to_wasm(
@@ -391,8 +385,10 @@ impl Loader {
             }
 
             let wasm_bytes = fs::read(&library_path)?;
-            Ok(wasm_store.load_language(name, &wasm_bytes))
-        } else {
+            return Ok(wasm_store.load_language(name, &wasm_bytes));
+        }
+
+        {
             if recompile {
                 self.compile_parser_to_dylib(
                     header_path,
@@ -411,7 +407,7 @@ impl Loader {
                 language_fn()
             };
             mem::forget(library);
-            Ok(language)
+            return Ok(language);
         }
     }
 
@@ -528,7 +524,7 @@ impl Loader {
         Ok(())
     }
 
-    fn compile_parser_to_wasm(
+    pub fn compile_parser_to_wasm(
         &self,
         language_name: &str,
         src_path: &Path,
@@ -874,6 +870,17 @@ impl Loader {
     pub fn use_wasm(&mut self, engine: tree_sitter::wasmtime::Engine) {
         *self.wasm_store.lock().unwrap() = Some(tree_sitter::WasmStore::new(engine))
     }
+
+    pub fn get_scanner_path(&self, src_path: &Path) -> Option<PathBuf> {
+        let mut path = src_path.join("scanner.c");
+        for extension in ["c", "cc", "cpp"] {
+            path.set_extension(extension);
+            if path.exists() {
+                return Some(path);
+            }
+        }
+        None
+    }
 }
 
 impl<'a> LanguageConfiguration<'a> {

cli/src/main.rs

@@ -4,7 +4,7 @@ use glob::glob;
 use std::collections::HashSet;
 use std::path::{Path, PathBuf};
 use std::{env, fs, u64};
-use tree_sitter::{ffi, Parser, Point, WasmStore};
+use tree_sitter::{ffi, Parser, Point};
 use tree_sitter_cli::{
     generate, highlight, logger,
     parse::{self, ParseFileOptions, ParseOutput},
@@ -390,8 +390,6 @@ fn run() -> Result<()> {
             let debug_build = matches.is_present("debug-build");
             let update = matches.is_present("update");
             let filter = matches.value_of("filter");
-            let wasm = matches.is_present("wasm");
-            let mut parser = Parser::new();
             let apply_all_captures = matches.is_present("apply-all-captures");
 
             if debug {
@@ -401,10 +399,13 @@ fn run() -> Result<()> {
 
             loader.use_debug_build(debug_build);
 
-            if wasm {
+            let mut parser = Parser::new();
+
+            #[cfg(feature = "wasm")]
+            if matches.is_present("wasm") {
                 let engine = tree_sitter::wasmtime::Engine::default();
                 parser
-                    .set_wasm_store(WasmStore::new(engine.clone()))
+                    .set_wasm_store(tree_sitter::WasmStore::new(engine.clone()))
                     .unwrap();
                 loader.use_wasm(engine);
             }
@@ -433,33 +434,27 @@ fn run() -> Result<()> {
                 )?;
             }
 
-            let mut store = parser.take_wasm_store();
-
             // Check that all of the queries are valid.
             test::check_queries_at_path(*language, &current_dir.join("queries"))?;
 
             // Run the syntax highlighting tests.
             let test_highlight_dir = test_dir.join("highlight");
             if test_highlight_dir.is_dir() {
                 let mut highlighter = Highlighter::new();
-                if let Some(store) = store.take() {
-                    highlighter.parser().set_wasm_store(store).unwrap();
-                }
+                highlighter.parser = parser;
                 test_highlight::test_highlights(
                     &loader,
                     &mut highlighter,
                     &test_highlight_dir,
                     apply_all_captures,
                 )?;
-                store = highlighter.parser().take_wasm_store();
+                parser = highlighter.parser;
             }
 
             let test_tag_dir = test_dir.join("tags");
             if test_tag_dir.is_dir() {
                 let mut tags_context = TagsContext::new();
-                if let Some(store) = store.take() {
-                    tags_context.parser().set_wasm_store(store).unwrap();
-                }
+                tags_context.parser = parser;
                 test_tags::test_tags(&loader, &mut tags_context, &test_tag_dir)?;
             }
         }
@@ -490,7 +485,6 @@ fn run() -> Result<()> {
                     })?;
 
             let time = matches.is_present("time");
-            let wasm = matches.is_present("wasm");
             let edits = matches
                 .values_of("edits")
                 .map_or(Vec::new(), |e| e.collect());
@@ -504,10 +498,11 @@ fn run() -> Result<()> {
 
             loader.use_debug_build(debug_build);
 
-            if wasm {
+            #[cfg(feature = "wasm")]
+            if matches.is_present("wasm") {
                 let engine = tree_sitter::wasmtime::Engine::default();
                 parser
-                    .set_wasm_store(WasmStore::new(engine.clone()))
+                    .set_wasm_store(tree_sitter::WasmStore::new(engine.clone()))
                     .unwrap();
                 loader.use_wasm(engine);
             }
@@ -744,6 +739,7 @@ fn run() -> Result<()> {
         ("build-wasm", Some(matches)) => {
             let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
             wasm::compile_language_to_wasm(
+                &loader,
                 &grammar_path,
                 &current_dir,
                 matches.is_present("docker"),

cli/src/wasm.rs

@@ -1,14 +1,7 @@
 use super::generate::parse_grammar::GrammarJSON;
-use anyhow::{anyhow, Context, Result};
-use path_slash::PathExt as _;
-use std::{
-    ffi::{OsStr, OsString},
-    fs,
-    path::Path,
-    process::Command,
-};
-use tree_sitter_loader::EMSCRIPTEN_TAG;
-use which::which;
+use anyhow::{Context, Result};
+use std::{fs, path::Path};
+use tree_sitter_loader::Loader;
 
 pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)> {
     let grammar_name = get_grammar_name(&language_dir)
@@ -35,119 +28,23 @@ pub fn get_grammar_name(language_dir: &Path) -> Result<String> {
 }
 
 pub fn compile_language_to_wasm(
+    loader: &Loader,
     language_dir: &Path,
     output_dir: &Path,
     force_docker: bool,
 ) -> Result<()> {
     let grammar_name = get_grammar_name(&language_dir)?;
     let output_filename = output_dir.join(&format!("tree-sitter-{}.wasm", grammar_name));
-
-    let emcc_bin = if cfg!(windows) { "emcc.bat" } else { "emcc" };
-    let emcc_path = which(emcc_bin)
-        .ok()
-        .and_then(|p| Command::new(&p).output().and(Ok(p)).ok());
-
-    let mut command;
-    if !force_docker && emcc_path.is_some() {
-        command = Command::new(emcc_path.unwrap());
-        command.current_dir(&language_dir);
-    } else if Command::new("docker").output().is_ok() {
-        command = Command::new("docker");
-        command.args(&["run", "--rm"]);
-
-        // Mount the parser directory as a volume
-        let mut volume_string;
-        if let (Some(parent), Some(filename)) = (language_dir.parent(), language_dir.file_name()) {
-            volume_string = OsString::from(parent);
-            volume_string.push(":/src:Z");
-            command.arg("--workdir");
-            command.arg(Path::new("/src").join(filename).to_slash_lossy().as_ref());
-        } else {
-            volume_string = OsString::from(language_dir);
-            volume_string.push(":/src:Z");
-            command.args(&["--workdir", "/src"]);
-        }
-
-        command.args(&[OsStr::new("--volume"), &volume_string]);
-
-        // Get the current user id so that files created in the docker container will have
-        // the same owner.
-        if cfg!(unix) {
-            let user_id_output = Command::new("id")
-                .arg("-u")
-                .output()
-                .with_context(|| "Failed to get get current user id")?;
-            let user_id = String::from_utf8_lossy(&user_id_output.stdout);
-            let user_id = user_id.trim();
-            command.args(&["--user", user_id]);
-        }
-
-        // Run `emcc` in a container using the `emscripten-slim` image
-        command.args(&[EMSCRIPTEN_TAG, "emcc"]);
-    } else {
-        if force_docker {
-            return Err(anyhow!(
-                "You must have docker on your PATH to run this command with --docker"
-            ));
-        }
-        return Err(anyhow!(
-            "You must have either emcc or docker on your PATH to run this command"
-        ));
-    }
-
-    command.arg("-o").arg(&output_filename);
-    command.args(&[
-        "-Os",
-        "-s",
-        "WASM=1",
-        "-s",
-        "SIDE_MODULE=1",
-        "-s",
-        "TOTAL_MEMORY=33554432",
-        "-s",
-        "NODEJS_CATCH_EXIT=0",
-        "-s",
-        "NODEJS_CATCH_REJECTION=0",
-        "-s",
-        &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar_name),
-        "-fno-exceptions",
-        "-I",
-        "src",
-    ]);
-
-    let src = Path::new("src");
-    let parser_c_path = src.join("parser.c");
-    let scanner_c_path = src.join("scanner.c");
-    let scanner_cc_path = src.join("scanner.cc");
-    let scanner_cpp_path = src.join("scanner.cpp");
-
-    if language_dir.join(&scanner_cc_path).exists() {
-        command
-            .arg("-xc++")
-            .arg(scanner_cc_path.to_slash_lossy().as_ref());
-    } else if language_dir.join(&scanner_cpp_path).exists() {
-        command
-            .arg("-xc++")
-            .arg(scanner_cpp_path.to_slash_lossy().as_ref());
-    } else if language_dir.join(&scanner_c_path).exists() {
-        command.arg(scanner_c_path.to_slash_lossy().as_ref());
-    }
-
-    command.arg(parser_c_path.to_slash_lossy().as_ref());
-
-    let output = command
-        .output()
-        .with_context(|| "Failed to run emcc command")?;
-    if !output.status.success() {
-        return Err(anyhow!(
-            "emcc command failed - {}",
-            String::from_utf8_lossy(&output.stderr)
-        ));
-    }
-
-    // Move the created `.wasm` file into the current working directory.
-    fs::rename(&language_dir.join(&output_filename), &output_filename)
-        .with_context(|| format!("Couldn't find output file {:?}", output_filename))?;
-
+    let src_path = language_dir.join("src");
+    let scanner_path = loader.get_scanner_path(&src_path);
+    loader.compile_parser_to_wasm(
+        &grammar_name,
+        &src_path,
+        scanner_path
+            .as_ref()
+            .and_then(|p| Some(Path::new(p.file_name()?))),
+        &output_filename,
+        force_docker,
+    )?;
     Ok(())
 }

highlight/src/lib.rs

@@ -127,7 +127,7 @@ pub struct HighlightConfiguration {
 /// syntax highlighting calls. A separate highlighter is needed for each thread that
 /// is performing highlighting.
 pub struct Highlighter {
-    parser: Parser,
+    pub parser: Parser,
     cursors: Vec<QueryCursor>,
 }
 

tags/src/lib.rs

@@ -43,7 +43,7 @@ pub struct NamedCapture {
 }
 
 pub struct TagsContext {
-    parser: Parser,
+    pub parser: Parser,
     cursor: QueryCursor,
 }