Merge pull request tree-sitter#2566 from amaanq/incl-range

fix: do not increment `current_included_range_index` past `included_range_count` in `__do_advance`

Author: ahlinc

.github/workflows/sanitize.yml

@@ -36,14 +36,14 @@ jobs:
       env:
         UBSAN_OPTIONS: halt_on_error=1
         CFLAGS: -fsanitize=undefined
-        RUSTFLAGS: -lubsan
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} -lubsan
       run: cargo test -- --test-threads 1
 
     - name: Run main tests with address sanitizer (ASAN)
       env:
         ASAN_OPTIONS: halt_on_error=1
         CFLAGS: -fsanitize=address
-        RUSTFLAGS: -Zsanitizer=address
+        RUSTFLAGS: ${{ env.RUSTFLAGS }} -Zsanitizer=address --cfg=sanitizing
       run: |
         rustup install nightly
         rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu

cli/src/tests/mod.rs

@@ -5,6 +5,7 @@ mod helpers;
 mod highlight_test;
 mod language_test;
 mod node_test;
+mod parser_hang_test;
 mod parser_test;
 mod pathological_test;
 mod query_test;

cli/src/tests/parser_hang_test.rs

@@ -0,0 +1,99 @@
+// For some reasons `Command::spawn` doesn't work in CI env for many exotic arches.
+#![cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))]
+
+use crate::{
+    generate::{generate_parser_for_grammar, load_grammar_file},
+    tests::helpers::fixtures::{fixtures_dir, get_test_language},
+};
+use std::{
+    env::VarError,
+    process::{Command, Stdio},
+};
+use tree_sitter::Parser;
+
+// The `sanitizing` cfg is required to don't run tests under specific sunitizer
+// because they don't work well with subprocesses _(it's an assumption)_.
+//
+// Bellow are two alternative examples of how to disable tests for some arches
+// if a way with excluding the whole mod from compilation would work well.
+//
+// #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing)))]
+// #[cfg_attr(not(all(any(target_arch = "x86_64", target_arch = "x86"), not(sanitizing))), ignore)]
+//
+#[test]
+fn test_grammar_that_should_hang_and_not_segfault() {
+    let parent_sleep_millis = 1000;
+    let test_name = "test_grammar_that_should_hang_and_not_segfault";
+    let test_var = "CARGO_HANG_TEST";
+
+    eprintln!("  {test_name}");
+
+    let tests_exec_path = std::env::args()
+        .nth(0)
+        .expect("Failed get get tests executable path");
+
+    match std::env::var(test_var) {
+        Ok(v) if v == test_name => {
+            eprintln!("    child process id {}", std::process::id());
+            hang_test();
+        }
+
+        Err(VarError::NotPresent) => {
+            eprintln!("    parent process id {}", std::process::id());
+            if true {
+                let mut command = Command::new(tests_exec_path);
+                command.arg(test_name).env(test_var, test_name);
+                if std::env::args().any(|x| x == "--nocapture") {
+                    command.arg("--nocapture");
+                } else {
+                    command.stdout(Stdio::null()).stderr(Stdio::null());
+                }
+                match command.spawn() {
+                    Ok(mut child) => {
+                        std::thread::sleep(std::time::Duration::from_millis(parent_sleep_millis));
+                        match child.try_wait() {
+                            Ok(Some(status)) if status.success() => {
+                                panic!("Child wasn't hang and exited successfully")
+                            }
+                            Ok(Some(status)) => panic!(
+                                "Child wasn't hang and exited with status code: {:?}",
+                                status.code()
+                            ),
+                            _ => (),
+                        }
+                        if let Err(e) = child.kill() {
+                            eprintln!(
+                                "Failed to kill hang test sub process id: {}, error: {e}",
+                                child.id()
+                            );
+                        }
+                    }
+                    Err(e) => panic!("{e}"),
+                }
+            }
+        }
+
+        Err(e) => panic!("Env var error: {e}"),
+        _ => unreachable!(),
+    }
+
+    fn hang_test() {
+        let test_grammar_dir = fixtures_dir()
+            .join("test_grammars")
+            .join("get_col_should_hang_not_crash");
+
+        let grammar_json = load_grammar_file(&test_grammar_dir.join("grammar.js"), None).unwrap();
+        let (parser_name, parser_code) =
+            generate_parser_for_grammar(grammar_json.as_str()).unwrap();
+
+        let language =
+            get_test_language(&parser_name, &parser_code, Some(test_grammar_dir.as_path()));
+
+        let mut parser = Parser::new();
+        parser.set_language(language).unwrap();
+
+        let code_that_should_hang = "\nHello";
+
+        parser.parse(code_that_should_hang, None).unwrap();
+    }
+}

lib/src/lexer.c

@@ -172,7 +172,9 @@ static void ts_lexer__do_advance(Lexer *self, bool skip) {
     self->current_position.bytes >= current_range->end_byte ||
     current_range->end_byte == current_range->start_byte
   ) {
-    self->current_included_range_index++;
+    if (self->current_included_range_index < self->included_range_count) {
+      self->current_included_range_index++;
+    }
     if (self->current_included_range_index < self->included_range_count) {
       current_range++;
       self->current_position = (Length) {

test/fixtures/test_grammars/get_col_should_hang_not_crash/corpus.txt

@@ -0,0 +1,13 @@
+module.exports = grammar({
+  name: 'get_col_should_hang_not_crash',
+
+  externals: $ => [
+    $.test,
+  ],
+
+  rules: {
+    source_file: $ => seq(
+      $.test
+    ),
+  },
+});

test/fixtures/test_grammars/get_col_should_hang_not_crash/grammar.js

@@ -0,0 +1,17 @@
+#include <tree_sitter/parser.h>
+
+unsigned tree_sitter_get_col_should_hang_not_crash_external_scanner_serialize() { return 0; }
+
+void tree_sitter_get_col_should_hang_not_crash_external_scanner_deserialize() {}
+
+void *tree_sitter_get_col_should_hang_not_crash_external_scanner_create() { return NULL; }
+
+void tree_sitter_get_col_should_hang_not_crash_external_scanner_destroy() {}
+
+bool tree_sitter_get_col_should_hang_not_crash_external_scanner_scan(void *payload, TSLexer *lexer,
+                                            const bool *valid_symbols) {
+    while (true) {
+        lexer->advance(lexer, false);
+        lexer->get_column(lexer);
+    }
+}