Properly incorporate lookahead bytes when recovering via missing token

Author: maxbrunsfeld

lib/src/parser.c

@@ -1307,9 +1307,11 @@ static void ts_parser__handle_error(
   for (StackVersion v = version; v < version_count;) {
     if (!did_insert_missing_token) {
       TSStateId state = ts_stack_state(self->stack, v);
-      for (TSSymbol missing_symbol = 1;
-           missing_symbol < self->language->token_count;
-           missing_symbol++) {
+      for (
+        TSSymbol missing_symbol = 1;
+        missing_symbol < self->language->token_count;
+        missing_symbol++
+      ) {
         TSStateId state_after_missing_symbol = ts_language_next_state(
           self->language, state, missing_symbol
         );
@@ -1328,10 +1330,13 @@ static void ts_parser__handle_error(
           ts_lexer_reset(&self->lexer, position);
           ts_lexer_mark_end(&self->lexer);
           Length padding = length_sub(self->lexer.token_end_position, position);
+          uint32_t lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead);
 
           StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v);
           Subtree missing_tree = ts_subtree_new_missing_leaf(
-            &self->tree_pool, missing_symbol, padding, self->language
+            &self->tree_pool, missing_symbol,
+            padding, lookahead_bytes,
+            self->language
           );
           ts_stack_push(
             self->stack, version_with_missing_tree,

lib/src/subtree.c

@@ -559,10 +559,11 @@ Subtree ts_subtree_new_missing_leaf(
   SubtreePool *pool,
   TSSymbol symbol,
   Length padding,
+  uint32_t lookahead_bytes,
   const TSLanguage *language
 ) {
   Subtree result = ts_subtree_new_leaf(
-    pool, symbol, padding, length_zero(), 0,
+    pool, symbol, padding, length_zero(), lookahead_bytes,
     0, false, false, false, language
   );
   if (result.data.is_inline) {

lib/src/subtree.h

@@ -193,7 +193,7 @@ Subtree ts_subtree_new_error(
 );
 MutableSubtree ts_subtree_new_node(TSSymbol, SubtreeArray *, unsigned, const TSLanguage *);
 Subtree ts_subtree_new_error_node(SubtreeArray *, bool, const TSLanguage *);
-Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, const TSLanguage *);
+Subtree ts_subtree_new_missing_leaf(SubtreePool *, TSSymbol, Length, uint32_t, const TSLanguage *);
 MutableSubtree ts_subtree_make_mut(SubtreePool *, Subtree);
 void ts_subtree_retain(Subtree);
 void ts_subtree_release(SubtreePool *, Subtree);