Added scanning support for extended escapes.

Author: DanielRosenwasser

src/compiler/diagnosticInformationMap.generated.ts

@@ -153,6 +153,8 @@ module ts {
         External_module_0_has_no_default_export_or_export_assignment: { code: 1192, category: DiagnosticCategory.Error, key: "External module '{0}' has no default export or export assignment." },
         An_export_declaration_cannot_have_modifiers: { code: 1193, category: DiagnosticCategory.Error, key: "An export declaration cannot have modifiers." },
         Export_declarations_are_not_permitted_in_an_internal_module: { code: 1194, category: DiagnosticCategory.Error, key: "Export declarations are not permitted in an internal module." },
+        An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive: { code: 1195, category: DiagnosticCategory.Error, key: "An extended Unicode escape value must be between 0x0 and 0x10FFFF inclusive." },
+        expected: { code: 1196, category: DiagnosticCategory.Error, key: "'}' expected." },
         Duplicate_identifier_0: { code: 2300, category: DiagnosticCategory.Error, key: "Duplicate identifier '{0}'." },
         Initializer_of_instance_member_variable_0_cannot_reference_identifier_1_declared_in_the_constructor: { code: 2301, category: DiagnosticCategory.Error, key: "Initializer of instance member variable '{0}' cannot reference identifier '{1}' declared in the constructor." },
         Static_members_cannot_reference_class_type_parameters: { code: 2302, category: DiagnosticCategory.Error, key: "Static members cannot reference class type parameters." },

src/compiler/diagnosticMessages.json

@@ -603,7 +603,14 @@
         "category": "Error",
         "code": 1194
     },
-
+    "An extended Unicode escape value must be between 0x0 and 0x10FFFF inclusive.": {
+        "category": "Error",
+        "code": 1195
+    },
+    "'}' expected.": {
+        "category": "Error",
+        "code":  1196
+    },
     "Duplicate identifier '{0}'.": {
         "category": "Error",
         "code": 2300
@@ -1572,7 +1579,7 @@
     "Exported type alias '{0}' has or is using private name '{1}'.": {
         "category": "Error",
         "code": 4081
-    },
+    },
     "The current host does not support the '{0}' option.": {
         "category": "Error",
         "code": 5001

src/compiler/scanner.ts

@@ -774,11 +774,44 @@ module ts {
                     return "\'";
                 case CharacterCodes.doubleQuote:
                     return "\"";
-                case CharacterCodes.x:
                 case CharacterCodes.u:
-                    var ch = scanExactNumberOfHexDigits(ch === CharacterCodes.x ? 2 : 4);
-                    if (ch >= 0) {
-                        return String.fromCharCode(ch);
+                    if (text.charCodeAt(pos) === CharacterCodes.openBrace) {
+                        pos++;
+                        var escapedValue = scanMinimumNumberOfHexDigits(1);
+                        
+                        if (escapedValue < 0) {
+                            // TODO(drosen): give a proper error message for escaped values that are too large.
+                            error(Diagnostics.Hexadecimal_digit_expected)
+                            return "";
+                        }
+                        
+                        if (escapedValue > 0x10FFFF) {
+                            error(Diagnostics.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive);
+                            return "";
+                        }
+                        
+                        if (pos >= len) {
+                            error(Diagnostics.Unexpected_end_of_text);
+                            return "";
+                        }
+                        
+                        // Only swallow the following character up if it's a '}'.
+                        var escapeTerminator = text.charCodeAt(pos);
+                        if (escapeTerminator == CharacterCodes.closeBrace) {
+                            pos++;
+                        }
+                        else {
+                            // '}' expected.
+                            error(Diagnostics.expected);
+                        }
+                        
+                        return utf16EncodeAsString(escapedValue);
+                    }
+                    // fall through
+                case CharacterCodes.x:
+                    var escapedValue = scanExactNumberOfHexDigits(ch === CharacterCodes.x ? 2 : 4);
+                    if (escapedValue >= 0) {
+                        return String.fromCharCode(escapedValue);
                     }
                     else {
                         error(Diagnostics.Hexadecimal_digit_expected);
@@ -800,6 +833,20 @@ module ts {
                     return String.fromCharCode(ch);
             }
         }
+        
+        // Derived from the 10.1.1 UTF16Encoding of the ES6 Spec.
+        function utf16EncodeAsString(codePoint: number): string {
+            Debug.assert(0x0 <= codePoint && codePoint <= 0x10FFFF);
+            
+            if (codePoint <= 65535) {
+                return String.fromCharCode(codePoint);
+            }
+            
+            var codeUnit1 = Math.floor((codePoint - 65536) / 1024) + 0xD800;
+            var codeUnit2 = ((codePoint - 65536) % 1024) + 0xDC00;
+            
+            return String.fromCharCode(codeUnit1, codeUnit2);
+        }
 
         // Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX'
         // and return code point value if valid Unicode escape is found. Otherwise return -1.

tests/baselines/reference/taggedTemplateStringsWithUnicodeEscapes.errors.txt

@@ -11,5 +11,5 @@ function f() {
         args[_i - 0] = arguments[_i];
     }
 }
-(_a = ["'{1f4a9}'", "'💩'"], _a.raw = ["'\\u{1f4a9}'", "'\\uD83D\\uDCA9'"], f(_a, " should be converted to "));
+(_a = ["'💩'", "'💩'"], _a.raw = ["'\\u{1f4a9}'", "'\\uD83D\\uDCA9'"], f(_a, " should be converted to "));
 var _a;

tests/baselines/reference/taggedTemplateStringsWithUnicodeEscapes.js

@@ -0,0 +1,9 @@
+=== tests/cases/compiler/taggedTemplateStringsWithUnicodeEscapes.ts ===
+function f(...args: any[]) {
+>f : (...args: any[]) => void
+>args : any[]
+}
+
+f `'\u{1f4a9}'${ " should be converted to " }'\uD83D\uDCA9'`;
+>f : (...args: any[]) => void
+

tests/baselines/reference/taggedTemplateStringsWithUnicodeEscapes.types

@@ -0,0 +1,9 @@
+=== tests/cases/compiler/taggedTemplateStringsWithUnicodeEscapesES6.ts ===
+function f(...args: any[]) {
+>f : (...args: any[]) => void
+>args : any[]
+}
+
+f `'\u{1f4a9}'${ " should be converted to " }'\uD83D\uDCA9'`;
+>f : (...args: any[]) => void
+

tests/baselines/reference/taggedTemplateStringsWithUnicodeEscapesES6.errors.txt

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings01_ES5.ts ===
+
+var x = "\u{0}";
+>x : string
+

tests/baselines/reference/taggedTemplateStringsWithUnicodeEscapesES6.types

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings01_ES6.ts ===
+
+var x = "\u{0}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings01_ES5.errors.txt

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings02_ES5.ts ===
+
+var x = "\u{00}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings01_ES5.types

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings02_ES6.ts ===
+
+var x = "\u{00}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings01_ES6.errors.txt

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings03_ES5.ts ===
+
+var x = "\u{0000}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings01_ES6.types

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings03_ES6.ts ===
+
+var x = "\u{0000}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings02_ES5.errors.txt

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings04_ES5.ts ===
+
+var x = "\u{00000000}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings02_ES5.types

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings04_ES6.ts ===
+
+var x = "\u{00000000}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings02_ES6.errors.txt

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings05_ES5.ts ===
+
+var x = "\u{48}\u{65}\u{6c}\u{6c}\u{6f}\u{20}\u{77}\u{6f}\u{72}\u{6c}\u{64}";
+>x : string
+

tests/baselines/reference/unicodeExtendedEscapesInStrings02_ES6.types

@@ -0,0 +1,5 @@
+=== tests/cases/conformance/es6/unicodeExtendedEscapes/unicodeExtendedEscapesInStrings05_ES6.ts ===
+
+var x = "\u{48}\u{65}\u{6c}\u{6c}\u{6f}\u{20}\u{77}\u{6f}\u{72}\u{6c}\u{64}";
+>x : string
+