feat: add the ability to specify a custom decode function

Author: amaanq

Cargo.lock

@@ -71,6 +71,8 @@ tree-sitter-loader.workspace = true
 tree-sitter-tags.workspace = true
 
 [dev-dependencies]
+encoding_rs = "0.8.35"
+widestring = "1.1.0"
 tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" }
 
 tempfile.workspace = true

cli/Cargo.toml

@@ -4,7 +4,7 @@ use std::{
 };
 
 use tree_sitter::{
-    IncludedRangesError, InputEdit, LogType, ParseOptions, ParseState, Parser, Point, Range,
+    Decode, IncludedRangesError, InputEdit, LogType, ParseOptions, ParseState, Parser, Point, Range,
 };
 use tree_sitter_generate::{generate_parser_for_grammar, load_grammar_file};
 use tree_sitter_proc_macro::retry;
@@ -1646,6 +1646,176 @@ fn test_parsing_by_halting_at_offset() {
     assert!(seen_byte_offsets.len() > 100);
 }
 
+#[test]
+fn test_decode_utf32() {
+    use widestring::u32cstr;
+
+    let mut parser = Parser::new();
+    parser.set_language(&get_language("rust")).unwrap();
+
+    let utf32_text = u32cstr!("pub fn foo() { println!(\"€50\"); }");
+    let utf32_text = unsafe {
+        std::slice::from_raw_parts(utf32_text.as_ptr().cast::<u8>(), utf32_text.len() * 4)
+    };
+
+    struct U32Decoder;
+
+    impl Decode for U32Decoder {
+        fn decode(bytes: &[u8]) -> (i32, u32) {
+            if bytes.len() >= 4 {
+                #[cfg(target_endian = "big")]
+                {
+                    (
+                        i32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]),
+                        4,
+                    )
+                }
+
+                #[cfg(target_endian = "little")]
+                {
+                    (
+                        i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]),
+                        4,
+                    )
+                }
+            } else {
+                println!("bad decode: {bytes:?}");
+                (0, 0)
+            }
+        }
+    }
+
+    let tree = parser
+        .parse_custom_encoding::<U32Decoder, _, _>(
+            &mut |offset, _| {
+                if offset < utf32_text.len() {
+                    &utf32_text[offset..]
+                } else {
+                    &[]
+                }
+            },
+            None,
+            None,
+        )
+        .unwrap();
+
+    assert_eq!(
+        tree.root_node().to_sexp(),
+        "(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (expression_statement (macro_invocation macro: (identifier) (token_tree (string_literal (string_content))))))))"
+    );
+}
+
+#[test]
+fn test_decode_cp1252() {
+    use encoding_rs::WINDOWS_1252;
+
+    let mut parser = Parser::new();
+    parser.set_language(&get_language("rust")).unwrap();
+
+    let windows_1252_text = WINDOWS_1252.encode("pub fn foo() { println!(\"€50\"); }").0;
+
+    struct Cp1252Decoder;
+
+    impl Decode for Cp1252Decoder {
+        fn decode(bytes: &[u8]) -> (i32, u32) {
+            if !bytes.is_empty() {
+                let byte = bytes[0];
+                (byte as i32, 1)
+            } else {
+                (0, 0)
+            }
+        }
+    }
+
+    let tree = parser
+        .parse_custom_encoding::<Cp1252Decoder, _, _>(
+            &mut |offset, _| &windows_1252_text[offset..],
+            None,
+            None,
+        )
+        .unwrap();
+
+    assert_eq!(
+        tree.root_node().to_sexp(),
+        "(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (expression_statement (macro_invocation macro: (identifier) (token_tree (string_literal (string_content))))))))"
+    );
+}
+
+#[test]
+fn test_decode_macintosh() {
+    use encoding_rs::MACINTOSH;
+
+    let mut parser = Parser::new();
+    parser.set_language(&get_language("rust")).unwrap();
+
+    let macintosh_text = MACINTOSH.encode("pub fn foo() { println!(\"€50\"); }").0;
+
+    struct MacintoshDecoder;
+
+    impl Decode for MacintoshDecoder {
+        fn decode(bytes: &[u8]) -> (i32, u32) {
+            if !bytes.is_empty() {
+                let byte = bytes[0];
+                (byte as i32, 1)
+            } else {
+                (0, 0)
+            }
+        }
+    }
+
+    let tree = parser
+        .parse_custom_encoding::<MacintoshDecoder, _, _>(
+            &mut |offset, _| &macintosh_text[offset..],
+            None,
+            None,
+        )
+        .unwrap();
+
+    assert_eq!(
+        tree.root_node().to_sexp(),
+        "(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (expression_statement (macro_invocation macro: (identifier) (token_tree (string_literal (string_content))))))))"
+    );
+}
+
+#[test]
+fn test_decode_utf24le() {
+    let mut parser = Parser::new();
+    parser.set_language(&get_language("rust")).unwrap();
+
+    let mut utf24le_text = Vec::new();
+    for c in "pub fn foo() { println!(\"€50\"); }".chars() {
+        let code_point = c as u32;
+        utf24le_text.push((code_point & 0xFF) as u8);
+        utf24le_text.push(((code_point >> 8) & 0xFF) as u8);
+        utf24le_text.push(((code_point >> 16) & 0xFF) as u8);
+    }
+
+    struct Utf24LeDecoder;
+
+    impl Decode for Utf24LeDecoder {
+        fn decode(bytes: &[u8]) -> (i32, u32) {
+            if bytes.len() >= 3 {
+                (i32::from_le_bytes([bytes[0], bytes[1], bytes[2], 0]), 3)
+            } else {
+                (0, 0)
+            }
+        }
+    }
+
+    let tree = parser
+        .parse_custom_encoding::<Utf24LeDecoder, _, _>(
+            &mut |offset, _| &utf24le_text[offset..],
+            None,
+            None,
+        )
+        .unwrap();
+
+    assert_eq!(
+        tree.root_node().to_sexp(),
+        "(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (expression_statement (macro_invocation macro: (identifier) (token_tree (string_literal (string_content))))))))"
+    );
+}
+
 const fn simple_range(start: usize, end: usize) -> Range {
     Range {
         start_byte: start,

cli/src/tests/parser_test.rs

@@ -149,9 +149,22 @@ typedef struct {
     uint32_t *bytes_read
   );
   TSInputEncoding encoding;
+  DecodeFunction decode;
 } TSInput;
 ```
 
+In the event that you want to decode text that is not encoded in UTF-8 or UTF16, then you can set the `decode` field of the input to your function that will decode text. The signature of the `DecodeFunction` is as follows:
+
+```c
+typedef uint32_t (*DecodeFunction)(
+  const uint8_t *string,
+  uint32_t length,
+  int32_t *code_point
+);
+```
+
+The `string` argument is a pointer to the text to decode, which comes from the `read` function, and the `length` argument is the length of the `string`. The `code_point` argument is a pointer to an integer that represents the decoded code point, and should be written to in your `decode` callback. The function should return the number of bytes decoded.
+
 ### Syntax Nodes
 
 Tree-sitter provides a [DOM](https://en.wikipedia.org/wiki/Document_Object_Model)-style interface for inspecting syntax trees. A syntax node's _type_ is a string that indicates which grammar rule the node represents.

docs/section-2-using-parsers.md

@@ -35,9 +35,13 @@ pub struct TSQueryCursor {
 pub struct TSLookaheadIterator {
     _unused: [u8; 0],
 }
+pub type DecodeFunction = ::core::option::Option<
+    unsafe extern "C" fn(string: *const u8, length: u32, code_point: *mut i32) -> u32,
+>;
 pub const TSInputEncodingUTF8: TSInputEncoding = 0;
 pub const TSInputEncodingUTF16LE: TSInputEncoding = 1;
 pub const TSInputEncodingUTF16BE: TSInputEncoding = 2;
+pub const TSInputEncodingCustom: TSInputEncoding = 3;
 pub type TSInputEncoding = ::core::ffi::c_uint;
 pub const TSSymbolTypeRegular: TSSymbolType = 0;
 pub const TSSymbolTypeAnonymous: TSSymbolType = 1;
@@ -71,6 +75,7 @@ pub struct TSInput {
         ) -> *const ::core::ffi::c_char,
     >,
     pub encoding: TSInputEncoding,
+    pub decode: DecodeFunction,
 }
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
@@ -212,7 +217,7 @@ extern "C" {
     ) -> *mut TSTree;
 }
 extern "C" {
-    #[doc = " Use the parser to parse some source code and create a syntax tree, with some options.\n\n See [`ts_parser_parse`] for more details."]
+    #[doc = " Use the parser to parse some source code and create a syntax tree, with some options.\n\n See [`ts_parser_parse`] for more details.\n\n See [`TSParseOptions`] for more details on the options."]
     pub fn ts_parser_parse_with_options(
         self_: *mut TSParser,
         old_tree: *const TSTree,