Fix scanning of octal sequences in sets

[#86]

Author: jaynetics

CHANGELOG.md

@@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - minor improvements to `Regexp::Scanner` performance
 - overall improvement of parse performance: about 10% for large Regexps
 
+### Fixed
+
+- parsing of octal escape sequences in sets, e.g. `[\141]`
+  * thanks to [Randy Stauner](https://github.com/rwstauner) for the report
+
 ## [2.6.2] - 2023-01-19 - [Janosch Müller](mailto:[email protected])
 
 ### Fixed

lib/regexp_parser/scanner/scanner.rl

@@ -126,7 +126,8 @@
                           keep_mark | sequence_char;
 
   # escapes that also work within a character set
-  set_escape            = backslash | brackets | escaped_ascii | property_char |
+  set_escape            = backslash | brackets | escaped_ascii |
+                          octal_sequence | property_char |
                           sequence_char | single_codepoint_char_type;
 
 
@@ -247,12 +248,22 @@
   # set escapes scanner
   # --------------------------------------------------------------------------
   set_escape_sequence := |*
+    # Special case: in sets, octal sequences have higher priority than backrefs
+    octal_sequence {
+      emit(:escape, :octal, copy(data, ts-1, te))
+      fret;
+    };
+
+    # Scan all other escapes that work in sets with the generic escape scanner
     set_escape > (escaped_set_alpha, 2) {
       fhold;
       fnext character_set;
       fcall escape_sequence;
     };
 
+    # Treat all remaining escapes - those not supported in sets - as literal.
+    # (This currently includes \^, \-, \&, \:, although these could potentially
+    # be meta chars when not escaped, depending on their position in the set.)
     any > (escaped_set_alpha, 1) {
       emit(:escape, :literal, copy(data, ts-1, te))
       fret;

spec/parser/sets_spec.rb

@@ -14,10 +14,18 @@
     [0]    => [:set, :character, CharacterSet, text: '[', count: 3],
     [0, 1] => [:escape, :backspace, EscapeSequence::Backspace, text: '\b']
 
+  include_examples 'parse', '[a\xFz]',
+    [0]    => [:set, :character, CharacterSet, text: '[', count: 3],
+    [0, 1] => [:escape, :hex, EscapeSequence::Hex, text: '\xF']
+
   include_examples 'parse', '[a\x20c]',
     [0]    => [:set, :character, CharacterSet, text: '[', count: 3],
     [0, 1] => [:escape, :hex, EscapeSequence::Hex, text: '\x20']
 
+  include_examples 'parse', '[a\77c]',
+    [0]    => [:set, :character, CharacterSet, text: '[', count: 3],
+    [0, 1] => [:escape, :octal, EscapeSequence::Octal, text: '\77']
+
   include_examples 'parse', '[a\u0640c]',
     [0]    => [:set, :character, CharacterSet, text: '[', count: 3],
     [0, 1] => [:escape, :codepoint, EscapeSequence::Codepoint, text: '\u0640']

spec/scanner/sets_spec.rb

@@ -33,9 +33,13 @@
   include_examples 'scan', '[\u{40}]',              1 => [:escape, :codepoint_list,  '\u{40}',     1, 7]
   include_examples 'scan', '[\c2]',                 1 => [:escape, :control,         '\c2',        1, 4]
   include_examples 'scan', '[\C-C]',                1 => [:escape, :control,         '\C-C',       1, 5]
+  include_examples 'scan', '[\xF]',                 1 => [:escape, :hex,             '\xF',        1, 4]
   include_examples 'scan', '[\x20]',                1 => [:escape, :hex,             '\x20',       1, 5]
   include_examples 'scan', '[\M-Z]',                1 => [:escape, :meta_sequence,   '\M-Z',       1, 5]
   include_examples 'scan', '[\M-\C-X]',             1 => [:escape, :meta_sequence,   '\M-\C-X',    1, 8]
+  include_examples 'scan', '[\7]',                  1 => [:escape, :octal,           '\7',         1, 3]
+  include_examples 'scan', '[\77]',                 1 => [:escape, :octal,           '\77',        1, 4]
+  include_examples 'scan', '[\777]',                1 => [:escape, :octal,           '\777',       1, 5]
   include_examples 'scan', '[\\[]',                 1 => [:escape, :set_open,        '\[',         1, 3]
   include_examples 'scan', '[\\]]',                 1 => [:escape, :set_close,       '\]',         1, 3]
   include_examples 'scan', '[a\-]',                 2 => [:escape, :literal,         '\-',         2, 4]