Merge pull request tree-sitter#38 from tree-sitter/changed-ranges

Report text ranges whose syntactic meaning have changed after re-parsing

Author: Max Brunsfeld

include/tree_sitter/runtime.h

@@ -34,17 +34,25 @@ typedef struct {
   void (*log)(void *payload, TSLogType, const char *);
 } TSLogger;
 
-typedef struct {
-  size_t position;
-  size_t chars_inserted;
-  size_t chars_removed;
-} TSInputEdit;
-
 typedef struct {
   size_t row;
   size_t column;
 } TSPoint;
 
+typedef struct {
+  size_t start_byte;
+  size_t bytes_removed;
+  size_t bytes_added;
+  TSPoint start_point;
+  TSPoint extent_removed;
+  TSPoint extent_added;
+} TSInputEdit;
+
+typedef struct {
+  TSPoint start;
+  TSPoint end;
+} TSRange;
+
 typedef struct {
   const void *data;
   size_t offset[3];
@@ -98,6 +106,7 @@ void ts_document_set_logger(TSDocument *, TSLogger);
 void ts_document_print_debugging_graphs(TSDocument *, bool);
 void ts_document_edit(TSDocument *, TSInputEdit);
 int ts_document_parse(TSDocument *);
+int ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, size_t *);
 void ts_document_invalidate(TSDocument *);
 TSNode ts_document_root_node(const TSDocument *);
 size_t ts_document_parse_count(const TSDocument *);

spec/helpers/point_helpers.cc

@@ -1,3 +1,4 @@
+#include "./point_helpers.h"
 #include <string>
 #include <ostream>
 #include "runtime/length.h"
@@ -9,8 +10,12 @@ bool operator==(const TSPoint &left, const TSPoint &right) {
   return left.row == right.row && left.column == right.column;
 }
 
-std::ostream &operator<<(std::ostream &stream, const TSPoint &point) {
-  return stream << "{" << point.row << ", " << point.column << "}";
+bool operator==(const TSRange &left, const TSRange &right) {
+  return left.start == right.start && left.end == right.end;
+}
+
+bool operator==(const TSLength &left, const TSLength &right) {
+  return ts_length_eq(left, right);
 }
 
 bool operator<(const TSPoint &left, const TSPoint &right) {
@@ -23,3 +28,16 @@ bool operator<(const TSPoint &left, const TSPoint &right) {
 bool operator>(const TSPoint &left, const TSPoint &right) {
   return right < left;
 }
+
+std::ostream &operator<<(std::ostream &stream, const TSPoint &point) {
+  return stream << "{" << point.row << ", " << point.column << "}";
+}
+
+std::ostream &operator<<(std::ostream &stream, const TSRange &range) {
+  return stream << "{" << range.start << ", " << range.end << "}";
+}
+
+ostream &operator<<(ostream &stream, const TSLength &length) {
+  return stream << "{chars:" << length.chars << ", bytes:" <<
+    length.bytes << ", extent:" << length.extent << "}";
+}

spec/helpers/point_helpers.h

@@ -1,12 +1,23 @@
 #ifndef HELPERS_POINT_HELPERS_H_
 #define HELPERS_POINT_HELPERS_H_
 
+#include "runtime/length.h"
+#include <ostream>
+
 bool operator==(const TSPoint &left, const TSPoint &right);
 
 bool operator<(const TSPoint &left, const TSPoint &right);
 
 bool operator>(const TSPoint &left, const TSPoint &right);
 
+bool operator==(const TSRange &left, const TSRange &right);
+
+bool operator==(const TSLength &left, const TSLength &right);
+
 std::ostream &operator<<(std::ostream &stream, const TSPoint &point);
 
+std::ostream &operator<<(std::ostream &stream, const TSRange &range);
+
+std::ostream &operator<<(std::ostream &stream, const TSLength &length);
+
 #endif  // HELPERS_POINT_HELPERS_H_

spec/helpers/scope_sequence.cc

@@ -0,0 +1,105 @@
+#include "./scope_sequence.h"
+
+#include "bandit/bandit.h"
+#include <sstream>
+#include "helpers/stream_methods.h"
+#include "helpers/point_helpers.h"
+
+using std::string;
+using std::cout;
+
+static void append_text_to_scope_sequence(ScopeSequence *sequence,
+                                          ScopeStack *current_scopes,
+                                          const std::string &text,
+                                          size_t length) {
+  for (size_t i = 0; i < length; i++) {
+    string character(1, text[sequence->size()]);
+    sequence->push_back(*current_scopes);
+    sequence->back().push_back("'" + character + "'");
+  }
+}
+
+static void append_to_scope_sequence(ScopeSequence *sequence,
+                                     ScopeStack *current_scopes,
+                                     TSNode node, TSDocument *document,
+                                     const std::string &text) {
+  append_text_to_scope_sequence(sequence, current_scopes, text, ts_node_start_byte(node) - sequence->size());
+
+  string scope = ts_node_type(node, document);
+  current_scopes->push_back(scope);
+  size_t child_count = ts_node_child_count(node);
+  if (child_count > 0) {
+    for (size_t i = 0; i < child_count; i++) {
+      TSNode child = ts_node_child(node, i);
+      append_to_scope_sequence(sequence, current_scopes, child, document, text);
+    }
+  } else {
+    size_t length = ts_node_end_byte(node) - ts_node_start_byte(node);
+    append_text_to_scope_sequence(sequence, current_scopes, text, length);
+  }
+  current_scopes->pop_back();
+}
+
+ScopeSequence build_scope_sequence(TSDocument *document, const std::string &text) {
+  ScopeSequence sequence;
+  ScopeStack current_scopes;
+  TSNode node = ts_document_root_node(document);
+  append_to_scope_sequence(&sequence, &current_scopes, node, document, text);
+  return sequence;
+}
+
+bool operator<=(const TSPoint &left, const TSPoint &right) {
+  if (left.row < right.row)
+    return true;
+  else if (left.row == right.row)
+    return left.column <= right.column;
+  else
+    return false;
+}
+
+void verify_changed_ranges(const ScopeSequence &old_sequence, const ScopeSequence &new_sequence,
+                           const string &text, TSRange *ranges, size_t range_count) {
+  TSPoint current_position = {0, 0};
+  for (size_t i = 0; i < old_sequence.size(); i++) {
+    if (text[i] == '\n') {
+      current_position.row++;
+      current_position.column = 0;
+      continue;
+    }
+
+    const ScopeStack &old_scopes = old_sequence[i];
+    const ScopeStack &new_scopes = new_sequence[i];
+    if (old_scopes != new_scopes) {
+      bool found_containing_range = false;
+      for (size_t j = 0; j < range_count; j++) {
+        TSRange range = ranges[j];
+        if (range.start <= current_position && current_position <= range.end) {
+          found_containing_range = true;
+          break;
+        }
+      }
+
+      if (!found_containing_range) {
+        std::stringstream message_stream;
+        message_stream << "Found changed scope outside of any invalidated range;\n";
+        message_stream << "Position: " << current_position << "\n";
+        message_stream << "Byte index: " << i << "\n";
+        size_t line_start_index = i - current_position.column;
+        size_t line_end_index = text.find_first_of('\n', i);
+        message_stream << "Line: " << text.substr(line_start_index, line_end_index - line_start_index) << "\n";
+        for (size_t j = 0; j < current_position.column + string("Line: ").size(); j++)
+          message_stream << " ";
+        message_stream << "^\n";
+        message_stream << "Old scopes: " << old_scopes << "\n";
+        message_stream << "New scopes: " << new_scopes << "\n";
+        message_stream << "Invalidated ranges:\n";
+        for (size_t j = 0; j < range_count; j++) {
+          message_stream << "  " << ranges[j] << "\n";
+        }
+        Assert::Failure(message_stream.str());
+      }
+    }
+
+    current_position.column++;
+  }
+}

spec/helpers/scope_sequence.h

@@ -0,0 +1,16 @@
+#ifndef HELPERS_SCOPE_SEQUENCE_H_
+#define HELPERS_SCOPE_SEQUENCE_H_
+
+#include <string>
+#include <vector>
+#include "tree_sitter/runtime.h"
+
+typedef std::string Scope;
+typedef std::vector<Scope> ScopeStack;
+typedef std::vector<ScopeStack> ScopeSequence;
+
+ScopeSequence build_scope_sequence(TSDocument *document, const std::string &text);
+
+void verify_changed_ranges(const ScopeSequence &old, const ScopeSequence &new_sequence, const std::string &text, TSRange *ranges, size_t range_count);
+
+#endif  // HELPERS_SCOPE_SEQUENCE_H_

spec/helpers/spy_input.cc

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <assert.h>
 
+using std::pair;
 using std::string;
 
 static const size_t UTF8_MAX_CHAR_SIZE = 4;
@@ -68,34 +69,63 @@ TSInput SpyInput::input() {
   return result;
 }
 
-TSInputEdit SpyInput::replace(size_t start_char, size_t chars_removed, string text) {
-  string text_removed = swap_substr(start_char, chars_removed, text);
-  size_t chars_inserted = string_char_count(encoding, text);
-  undo_stack.push_back(SpyInputEdit{start_char, chars_inserted, text_removed});
-  return {start_char, chars_inserted, chars_removed};
+static TSPoint get_extent(string text) {
+  TSPoint result = {0, 0};
+  for (auto i = text.begin(); i != text.end(); i++) {
+    if (*i == '\n') {
+      result.row++;
+      result.column = 0;
+    } else {
+      result.column++;
+    }
+  }
+  return result;
+}
+
+TSInputEdit SpyInput::replace(size_t start_byte, size_t bytes_removed, string text) {
+  auto swap = swap_substr(start_byte, bytes_removed, text);
+  size_t bytes_added = text.size();
+  undo_stack.push_back(SpyInputEdit{start_byte, bytes_added, swap.first});
+  TSInputEdit result = {};
+  result.start_byte = start_byte;
+  result.bytes_added = bytes_added;
+  result.bytes_removed = bytes_removed;
+  result.start_point = swap.second;
+  result.extent_removed = get_extent(swap.first);
+  result.extent_added = get_extent(text);
+  return result;
 }
 
 TSInputEdit SpyInput::undo() {
   SpyInputEdit entry = undo_stack.back();
   undo_stack.pop_back();
-  swap_substr(entry.position, entry.chars_removed, entry.text_inserted);
-  size_t chars_inserted = string_char_count(encoding, entry.text_inserted);
-  return TSInputEdit{entry.position, chars_inserted, entry.chars_removed};
+  auto swap = swap_substr(entry.start_byte, entry.bytes_removed, entry.text_inserted);
+  TSInputEdit result;
+  result.start_byte = entry.start_byte;
+  result.bytes_removed = entry.bytes_removed;
+  result.bytes_added = entry.text_inserted.size();
+  result.start_point = swap.second;
+  result.extent_removed = get_extent(swap.first);
+  result.extent_added = get_extent(entry.text_inserted);
+  return result;
 }
 
-string SpyInput::swap_substr(size_t start_char, size_t chars_removed, string text) {
-  long start_byte = string_byte_for_character(encoding, content, 0, start_char);
-  assert(start_byte >= 0);
-
-  long bytes_removed = string_byte_for_character(encoding, content, start_byte, chars_removed);
-  if (bytes_removed < 0)
-    bytes_removed = content.size() - start_byte;
+pair<string, TSPoint> SpyInput::swap_substr(size_t start_byte, size_t bytes_removed, string text) {
+  TSPoint start_position = {0, 0};
+  for (auto i = content.begin(), n = content.begin() + start_byte; i < n; i++) {
+    if (*i == '\n') {
+      start_position.row++;
+      start_position.column = 0;
+    } else {
+      start_position.column++;
+    }
+  }
 
   string text_removed = content.substr(start_byte, bytes_removed);
   content.erase(start_byte, bytes_removed);
   content.insert(start_byte, text);
 
-  return text_removed;
+  return {text_removed, start_position};
 }
 
 void SpyInput::clear() {

spec/helpers/spy_input.h

@@ -6,8 +6,8 @@
 #include "tree_sitter/runtime.h"
 
 struct SpyInputEdit {
-  size_t position;
-  size_t chars_removed;
+  size_t start_byte;
+  size_t bytes_removed;
   std::string text_inserted;
 };
 
@@ -20,7 +20,7 @@ class SpyInput {
 
   static const char * read(void *, size_t *);
   static int seek(void *, size_t, size_t);
-  std::string swap_substr(size_t, size_t, std::string);
+  std::pair<std::string, TSPoint> swap_substr(size_t, size_t, std::string);
 
  public:
   SpyInput(std::string content, size_t chars_per_chunk);

spec/helpers/tree_helpers.cc

@@ -40,10 +40,6 @@ bool operator==(const TSNode &left, const TSNode &right) {
   return ts_node_eq(left, right);
 }
 
-bool operator==(const TSLength &left, const TSLength &right) {
-  return ts_length_eq(left, right);
-}
-
 bool operator==(const std::vector<TSTree *> &vec, const TreeArray &array) {
   if (vec.size() != array.size)
     return false;
@@ -52,8 +48,3 @@ bool operator==(const std::vector<TSTree *> &vec, const TreeArray &array) {
       return false;
   return true;
 }
-
-ostream &operator<<(ostream &stream, const TSLength &length) {
-  return stream << "{chars:" << length.chars << ", bytes:" <<
-    length.bytes << ", rows:" << length.rows << ", columns:" << length.columns << "}";
-}

spec/helpers/tree_helpers.h

@@ -10,9 +10,7 @@ TSTree ** tree_array(std::vector<TSTree *> trees);
 
 std::ostream &operator<<(std::ostream &stream, const TSTree *tree);
 std::ostream &operator<<(std::ostream &stream, const TSNode &node);
-std::ostream &operator<<(std::ostream &stream, const TSLength &length);
 bool operator==(const TSNode &left, const TSNode &right);
-bool operator==(const TSLength &left, const TSLength &right);
 bool operator==(const std::vector<TSTree *> &right, const TreeArray &array);
 
 #endif  // HELPERS_TREE_HELPERS_H_