Merge pull request rescript-lang#54 from nkrkv/test-world

Tests and fixes in the wild

Author: nkrkv

.gitignore

@@ -3,3 +3,4 @@ node_modules/
 src/*
 !src/scanner.c
 log.html
+test_wild/

Makefile

@@ -1,10 +1,27 @@
 
 TS=yarn tree-sitter
 
+# This is an opinionated list of significant ReScript repos on GitHub
+# that are representative and trick-heavy enough to be a subject
+# for acceptance testing. The general idea: if `tree-sitter-rescript`
+# is 100% legit for this codebase, it should satisfy everyone.
+wild_github_repos := rescript-lang/rescript-react \
+										 tinymce/rescript-webapi
+
+wild_sandboxes := $(patsubst %,test_wild/%,$(wild_github_repos))
+
 .PHONY: generate
 generate:
 	$(TS) generate
 
 .PHONY: test
 test: generate
 	$(TS) test
+
+test_wild/%:
+	@mkdir -p test_wild/
+	git clone --depth=1 https://github.com/$* ./$@
+
+.PHONY: test_wild
+test_wild: $(wild_sandboxes)
+	$(TS) parse --quiet --stat '$@/**/*.res*'

README.md

@@ -19,4 +19,4 @@ Contributions are welcome. Here’s how you can help:
 yarn tree-sitter parse /path/to/your/snippet.res
 ```
 
-🤩 Add a failing test case for a snippet which is valid ReScript but produces an incorrect syntax tree. Fix the `grammar.js`. Make sure nothing is broken: `make test` shows 100% test success. Open a pull request.
+🤩 Add a failing test case for a snippet which is valid ReScript but produces an incorrect syntax tree. Fix the `grammar.js`. Make sure nothing is broken: `make test test_wild` shows 100% test success. Open a pull request.

grammar.js

@@ -40,6 +40,8 @@ module.exports = grammar({
       'coercion_relation',
       $.expression,
       $.primary_expression,
+      $.ternary_expression,
+      $.mutation_expression,
       $.function,
       $.let_binding,
     ],
@@ -52,6 +54,9 @@ module.exports = grammar({
     [$.unit, $.formal_parameters],
     [$.pipe_expression, $.expression],
     [$.primary_expression, $._pattern],
+    [$.primary_expression, $.record_pattern],
+    [$.primary_expression, $.spread_pattern],
+    [$.primary_expression, $._literal_pattern],
     [$.tuple_pattern, $._formal_parameter],
     [$.primary_expression, $._formal_parameter],
     [$.primary_expression, $.record_field],
@@ -60,8 +65,6 @@ module.exports = grammar({
     [$.list, $.list_pattern],
     [$.array, $.array_pattern],
     [$.record_field, $.record_pattern],
-    [$.primary_expression, $.spread_pattern],
-    [$.primary_expression, $._literal_pattern],
     [$.expression_statement, $.ternary_expression],
     [$.let_binding, $.ternary_expression],
     [$.variant_identifier, $.module_identifier],
@@ -98,9 +101,7 @@ module.exports = grammar({
       alias($._decorated_statement, $.decorated),
       $.decorator_statement,
       $.expression_statement,
-      $.mutation_statement,
       $.declaration,
-      $.block,
       $.open_statement,
       $.include_statement,
     ),
@@ -213,6 +214,7 @@ module.exports = grammar({
       optional($.type_parameters),
       optional(seq(
         '=',
+        optional('private'),
         $._type,
       ))
     ),
@@ -364,11 +366,11 @@ module.exports = grammar({
     let_binding: $ => seq(
       choice('export', 'let'),
       optional('rec'),
-      $._pattern,
+      choice($._pattern, $.unit),
       optional($.type_annotation),
       optional(seq(
         '=',
-        choice($.expression, $.block),
+        $.expression,
       )),
     ),
 
@@ -382,6 +384,8 @@ module.exports = grammar({
       $.binary_expression,
       $.coercion_expression,
       $.ternary_expression,
+      $.mutation_expression,
+      $.block,
     ),
 
     primary_expression: $ => choice(
@@ -431,10 +435,7 @@ module.exports = grammar({
         $._definition_signature
       ),
       '=>',
-      field('body', choice(
-        $.expression,
-        $.block
-      )),
+      field('body', $.expression),
     )),
 
     record: $ => seq(
@@ -802,6 +803,7 @@ module.exports = grammar({
 
     _jsx_child: $ => choice(
       $.value_identifier,
+      $.value_identifier_path,
       $._jsx_element,
       $.jsx_fragment,
       $.jsx_expression
@@ -862,7 +864,7 @@ module.exports = grammar({
       $.jsx_expression,
     ),
 
-    mutation_statement: $ => seq(
+    mutation_expression: $ => seq(
       $._mutation_lvalue,
       choice('=', ':='),
       $.expression,
@@ -1096,11 +1098,14 @@ module.exports = grammar({
         seq(optional('0'), /[1-9]/, optional(seq(optional('_'), decimal_digits)))
       )
 
-      const decimal_literal = choice(
-        seq(decimal_integer_literal, '.', optional(decimal_digits), optional(exponent_part)),
-        seq('.', decimal_digits, optional(exponent_part)),
-        seq(decimal_integer_literal, exponent_part),
-        seq(decimal_digits),
+      const decimal_literal = seq(
+        optional(choice('-', '+')),
+        choice(
+          seq(decimal_integer_literal, '.', optional(decimal_digits), optional(exponent_part)),
+          seq('.', decimal_digits, optional(exponent_part)),
+          seq(decimal_integer_literal, exponent_part),
+          seq(decimal_digits),
+        )
       )
 
       return token(choice(

queries/highlights.scm

@@ -98,6 +98,7 @@
   "let"
   "module"
   "mutable"
+  "private"
   "rec"
   "type"
 ] @keyword

src/scanner.c

@@ -17,6 +17,7 @@ typedef struct ScannerState {
   int parens_nesting;
   bool in_quotes;
   bool in_backticks;
+  bool eof_reported;
 } ScannerState;
 
 void *tree_sitter_rescript_external_scanner_create() {
@@ -152,6 +153,17 @@ bool tree_sitter_rescript_external_scanner_scan(
     return true;
   }
 
+  // If a source file missing EOL at EOF, give the last statement a chance:
+  // report the statement delimiting EOL at the very end of file. Make sure
+  // it’s done only once, otherwise the scanner will enter dead-lock because
+  // we report NEWLINE again and again, no matter the lexer is exhausted
+  // already.
+  if (valid_symbols[NEWLINE] && lexer->eof(lexer) && !state->eof_reported) {
+    lexer->result_symbol = NEWLINE;
+    state->eof_reported = true;
+    return true;
+  }
+
   // Magic ahead!
   // We have two types of newline in ReScript. The one which ends the current statement,
   // and the one used just for pretty-formatting (e.g. separates variant type values).

test/corpus/expressions.txt

@@ -80,6 +80,14 @@ keep(
   item =>
     item->Foo.bar == 5
 )
+blocky(
+  1,
+  {
+    open Mod
+    qrux
+  },
+  ~third={3},
+)
 
 ---
 
@@ -124,7 +132,18 @@ keep(
               (value_identifier_path
                 (module_identifier)
                 (value_identifier)))
-            right: (number)))))))
+            right: (number))))))
+  (expression_statement
+    (call_expression
+      function: (value_identifier)
+      arguments: (arguments
+        (number)
+        (block
+          (open_statement (module_identifier))
+          (expression_statement (value_identifier)))
+        (labeled_argument
+          label: (value_identifier)
+          value: (block (expression_statement (number))))))))
 
 ===========================================
 Pipe
@@ -620,6 +639,27 @@ switch parseExn(str) {
             (formal_parameters (value_identifier))))
         (expression_statement (number))))))
 
+===========================================
+Switch block
+===========================================
+
+switch { open Mod; foo() } {
+| _ => 42
+}
+
+---
+
+(source_file
+  (expression_statement
+    (switch_expression
+      (block
+        (open_statement (module_identifier))
+        (expression_statement
+          (call_expression (value_identifier) (arguments))))
+      (switch_match
+        (value_identifier)
+        (expression_statement (number))))))
+
 ===========================================
 Math operators
 ===========================================
@@ -887,3 +927,27 @@ try {
           (nested_variant_identifier (module_identifier) (module_identifier) (variant_identifier))
           (formal_parameters (value_identifier)))
         (expression_statement (number))))))
+
+===========================================
+Mutation expressions
+===========================================
+
+foo["bar"] = qux
+foo.bar = qux
+foo.bar := qux
+
+---
+
+(source_file
+  (expression_statement
+    (mutation_expression
+      (subscript_expression (value_identifier) (string (string_fragment)))
+      (value_identifier)))
+  (expression_statement
+    (mutation_expression
+      (member_expression (value_identifier) (property_identifier))
+      (value_identifier)))
+  (expression_statement
+    (mutation_expression
+      (member_expression (value_identifier) (property_identifier))
+      (value_identifier))))

test/corpus/functions.txt

@@ -131,6 +131,24 @@ Record pattern
         (value_identifier)))
     (number))))
 
+===================================================
+Ref assignment as body
+===================================================
+
+reactRef => myRef := Some(reactRef)
+
+---
+
+(source_file
+  (expression_statement
+    (function
+      (value_identifier)
+      (mutation_expression
+        (value_identifier)
+        (variant
+          (variant_identifier)
+          (arguments (value_identifier)))))))
+
 ===================================================
 Operator precendence
 ===================================================

test/corpus/jsx.txt

@@ -2,9 +2,10 @@
 Simple JSX elements
 ==============================================
 
-<div className="b" tabIndex={1} />;
-<Foo.Bar>a <span>b</span> c</Foo.Bar>;
-<Foo.Bar.Baz.Baz></Foo.Bar.Baz.Baz>;
+<div className="b" tabIndex={1} />
+<Foo.Bar>a <span>b</span> c</Foo.Bar>
+<Foo.Bar.Baz.Baz></Foo.Bar.Baz.Baz>
+<div> React.null </div>
 
 ---
 
@@ -37,7 +38,12 @@ Simple JSX elements
           (nested_jsx_identifier
             (nested_jsx_identifier (jsx_identifier) (jsx_identifier))
             (jsx_identifier))
-          (jsx_identifier))))))
+          (jsx_identifier)))))
+  (expression_statement
+    (jsx_element
+      (jsx_opening_element (jsx_identifier))
+      (value_identifier_path (module_identifier) (value_identifier))
+      (jsx_closing_element (jsx_identifier)))))
 
 ==============================================
 Attribute values
@@ -74,8 +80,8 @@ Attribute values
 Anonymous JSX element
 ==============================================
 
-<></>;
-<E><></></E>;
+<></>
+<E><></></E>
 
 ---
 

test/corpus/let_bindings.txt

@@ -93,3 +93,13 @@ let rec foo = n => foo(n-1)
             (binary_expression
               (value_identifier)
               (number)))))))
+
+============================================
+Unit
+============================================
+
+let () = noop
+
+---
+
+(source_file (let_binding (unit) (value_identifier)))