fix(lib): advance the lookahead end byte by 4 when there's an invalid code point

This helps in the case where an edit was made in the middle of a code
point, but bytes 1-3 are valid, thus we could advance by at most 4 bytes

Author: amaanq

cli/src/tests/tree_test.rs

@@ -679,6 +679,29 @@ fn test_get_changed_ranges() {
     }
 }
 
+#[test]
+fn test_consistency_with_mid_codepoint_edit() {
+    let mut parser = Parser::new();
+    parser.set_language(&get_language("php/php")).unwrap();
+    let mut source_code =
+        b"\n<?php\n\n<<<'\xE5\xAD\x97\xE6\xBC\xA2'\n  T\n\xE5\xAD\x97\xE6\xBC\xA2;".to_vec();
+    let mut tree = parser.parse(&source_code, None).unwrap();
+
+    let edit = Edit {
+        position: 17,
+        deleted_length: 0,
+        inserted_text: vec![46],
+    };
+    perform_edit(&mut tree, &mut source_code, &edit).unwrap();
+    let mut tree2 = parser.parse(&source_code, Some(&tree)).unwrap();
+
+    let inverted = invert_edit(&source_code, &edit);
+    perform_edit(&mut tree2, &mut source_code, &inverted).unwrap();
+    let tree3 = parser.parse(&source_code, Some(&tree2)).unwrap();
+
+    assert_eq!(tree3.root_node().to_sexp(), tree.root_node().to_sexp());
+}
+
 fn index_of(text: &[u8], substring: &str) -> usize {
     str::from_utf8(text).unwrap().find(substring).unwrap()
 }

lib/src/lexer.c

@@ -365,7 +365,7 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) {
   // Therefore, the next byte *after* the current (invalid) character
   // affects the interpretation of the current character.
   if (self->data.lookahead == TS_DECODE_ERROR) {
-    current_lookahead_end_byte++;
+    current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point
   }
 
   if (current_lookahead_end_byte > *lookahead_end_byte) {