Fix code paths that still conflated null characters with EOF

Author: maxbrunsfeld

cli/src/tests/parser_test.rs

@@ -266,6 +266,17 @@ fn test_parsing_invalid_chars_at_eof() {
     assert_eq!(tree.root_node().to_sexp(), "(ERROR (UNEXPECTED INVALID))");
 }
 
+#[test]
+fn test_parsing_unexpected_null_characters_within_source() {
+    let mut parser = Parser::new();
+    parser.set_language(get_language("javascript")).unwrap();
+    let tree = parser.parse(b"var \0 something;", None).unwrap();
+    assert_eq!(
+        tree.root_node().to_sexp(),
+        "(program (variable_declaration (ERROR (UNEXPECTED '\\0')) (variable_declarator name: (identifier))))"
+    );
+}
+
 #[test]
 fn test_parsing_ends_when_input_callback_returns_empty() {
     let mut parser = Parser::new();

lib/src/parser.c

@@ -439,7 +439,7 @@ static Subtree ts_parser__lex(
     }
 
     if (self->lexer.current_position.bytes == error_end_position.bytes) {
-      if (self->lexer.data.lookahead == 0) {
+      if (self->lexer.data.eof(&self->lexer.data)) {
         self->lexer.data.result_symbol = ts_builtin_sym_error;
         break;
       }

lib/src/subtree.c

@@ -766,10 +766,10 @@ Subtree ts_subtree_last_external_token(Subtree tree) {
 }
 
 static size_t ts_subtree__write_char_to_string(char *s, size_t n, int32_t c) {
-  if (c == 0)
-    return snprintf(s, n, "EOF");
   if (c == -1)
     return snprintf(s, n, "INVALID");
+  else if (c == '\0')
+    return snprintf(s, n, "'\\0'");
   else if (c == '\n')
     return snprintf(s, n, "'\\n'");
   else if (c == '\t')

test/fixtures/error_corpus/json_errors.txt

@@ -65,4 +65,4 @@ incomplete tokens at EOF
 nul
 ---
 
-(ERROR (UNEXPECTED EOF))
+(ERROR (UNEXPECTED '\0'))