feat(lang): implement chomped text blocks

Author: eduardosm

rsjsonnet-lang/src/lexer/mod.rs

@@ -472,6 +472,8 @@ impl<'a, 'p, 'ast> Lexer<'a, 'p, 'ast> {
 
     #[inline]
     fn lex_text_block(&mut self) -> Result<Token<'p, 'ast>, LexError> {
+        let strip_last_lf = self.eat_byte(b'-');
+
         let mut string = String::new();
         let mut prefix;
         while self.eat_byte_if(|b| matches!(b, b' ' | b'\t' | b'\r')) {}
@@ -539,7 +541,15 @@ impl<'a, 'p, 'ast> Lexer<'a, 'p, 'ast> {
             }
         }
 
-        Ok(self.commit_token(TokenKind::TextBlock(self.ast_arena.alloc_str(&string))))
+        let actual_string = if strip_last_lf {
+            string.strip_suffix('\n').unwrap()
+        } else {
+            string.as_str()
+        };
+
+        Ok(self.commit_token(TokenKind::TextBlock(
+            self.ast_arena.alloc_str(actual_string),
+        )))
     }
 
     #[must_use]

rsjsonnet-lang/tests/lexer.rs

@@ -900,6 +900,22 @@ fn test_text_block() {
     test(b"|||\n a\n\t|||", "a\n");
     test(b"|||\n \xFF\n\t|||", "\u{FFFD}\n");
 
+    test(b"|||-\n \n|||", "");
+    test(b"|||-\n a\n|||", "a");
+    test(b"|||-\n \n a\n|||", "\na");
+    test(b"|||- \n a\n|||", "a");
+    test(b"|||-\t\n a\n|||", "a");
+    test(b"|||-\n\n a\n|||", "\na");
+    test(b"|||-\n\n\n a\n|||", "\n\na");
+    test(b"|||-\n a\n b\n|||", "a\nb");
+    test(b"|||-\n  a\n  b\n|||", "a\nb");
+    test(b"|||-\n a\n\n b\n|||", "a\n\nb");
+    test(b"|||-\n a\n \n b\n|||", "a\n\nb");
+    test(b"|||-\n a\n\n\n b\n|||", "a\n\n\nb");
+    test(b"|||-\n a\n b\n\n|||", "a\nb\n");
+    test(b"|||-\n a\n\t|||", "a");
+    test(b"|||-\n \xFF\n\t|||", "\u{FFFD}");
+
     // error: unfinished
     LexerTest {
         input: b"|||\n a",
@@ -915,6 +931,20 @@ fn test_text_block() {
     }
     .run();
 
+    LexerTest {
+        input: b"|||-\n a",
+        check: |session, error, tokens| {
+            assert_eq!(
+                error,
+                Some(LexError::UnfinishedString {
+                    span: session.intern_span(0, 7),
+                }),
+            );
+            assert_eq!(tokens, []);
+        },
+    }
+    .run();
+
     // error: invalid start
     LexerTest {
         input: b"||| x\n y\n|||",
@@ -930,6 +960,20 @@ fn test_text_block() {
     }
     .run();
 
+    LexerTest {
+        input: b"|||- x\n y\n|||",
+        check: |session, error, tokens| {
+            assert_eq!(
+                error,
+                Some(LexError::MissingLineBreakAfterTextBlockStart {
+                    span: session.intern_span(0, 5),
+                }),
+            );
+            assert_eq!(tokens, []);
+        },
+    }
+    .run();
+
     LexerTest {
         input: b"|||\nx\ny\n|||",
         check: |session, error, tokens| {
@@ -943,6 +987,20 @@ fn test_text_block() {
         },
     }
     .run();
+
+    LexerTest {
+        input: b"|||-\nx\ny\n|||",
+        check: |session, error, tokens| {
+            assert_eq!(
+                error,
+                Some(LexError::MissingWhitespaceTextBlockStart {
+                    span: session.intern_span(5, 5),
+                }),
+            );
+            assert_eq!(tokens, []);
+        },
+    }
+    .run();
 }
 
 #[test]

ui-tests/pass/text_block_crlf.jsonnet

@@ -1,6 +1,12 @@
-local value = |||
+local value1 = |||
   first line
   second line
 |||;
 
-std.assertEqual(value, "first line\r\nsecond line\r\n")
+local value2 = |||-
+  first line
+  second line
+|||;
+
+std.assertEqual(value1, "first line\r\nsecond line\r\n") &&
+std.assertEqual(value2, "first line\r\nsecond line\r")

ui-tests/pass/text_block_lf.jsonnet

@@ -1,6 +1,12 @@
-local value = |||
+local value1 = |||
   first line
   second line
 |||;
 
-std.assertEqual(value, "first line\nsecond line\n")
+local value2 = |||-
+  first line
+  second line
+|||;
+
+std.assertEqual(value1, "first line\nsecond line\n") &&
+std.assertEqual(value2, "first line\nsecond line")