Issue ESAPI#300 -- Solved the root problem, now have many unit tests to clean up.

Author: xeno6696

src/main/java/org/owasp/esapi/codecs/Codec.java

@@ -64,9 +64,15 @@ public Codec() {
 	 */
 	public String encode(char[] immune, String input) {
 		StringBuilder sb = new StringBuilder();
-		for (int i = 0; i < input.length(); i++) {
-			char c = input.charAt(i);
-			sb.append(encodeCharacter(immune, c));
+		for(int offset  = 0; offset < input.length(); ){
+			final int point = input.codePointAt(offset);
+			if(Character.isBmpCodePoint(point)){
+				//We can then safely cast this to char and maintain legacy behavior.
+				sb.append(encodeCharacter(immune, (char) point));
+			}else{
+				sb.append(encodeCharacter(immune, point));	
+			}
+			offset += Character.charCount(point);
 		}
 		return sb.toString();
 	}
@@ -83,6 +89,19 @@ public String encode(char[] immune, String input) {
 	public String encodeCharacter( char[] immune, Character c ) {
 		return ""+c;
 	}
+	
+	/**
+	 * Default codepoint implementation that should be overridden in specific codecs.
+	 * 
+	 * @param immune
+	 * @param codePoint
+	 * 		the integer to encode
+	 * @return
+	 * 		the encoded Character
+	 */
+	public String encodeCharacter( char[] immune, int codePoint ) {
+		return new StringBuilder().appendCodePoint(codePoint).toString();
+	}
 
 	/**
 	 * Decode a String that was encoded using the encode method in this Class
@@ -131,6 +150,19 @@ public static String getHexForNonAlphanumeric(char c)
 			return hex[c];
 		return toHex(c);
 	}
+	
+	/**
+	 * Lookup the hex value of any character that is not alphanumeric.
+	 * @param c The character to lookup.
+	 * @return, return null if alphanumeric or the character code
+	 * 	in hex.
+	 */
+	public static String getHexForNonAlphanumeric(int c)
+	{
+		if(c<0xFF)
+			return hex[c];
+		return toHex(c);
+	}
 
 	public static String toOctal(char c)
 	{
@@ -141,6 +173,11 @@ public static String toHex(char c)
 	{
 		return Integer.toHexString(c);
 	}
+	
+	public static String toHex(int c)
+	{
+		return Integer.toHexString(c);
+	}
 
 	/**
 	 * Utility to search a char[] for a specific char.

src/main/java/org/owasp/esapi/codecs/HTMLEntityCodec.java

@@ -78,6 +78,42 @@ public String encodeCharacter( char[] immune, Character c ) {
 		return "&#x" + hex + ";";
 	}
 
+	/**
+	 * {@inheritDoc}
+	 * 
+     * Encodes a Character for safe use in an HTML entity field.
+     * @param immune
+     */
+	public String encodeCharacter( char[] immune, int codePoint ) {
+
+		// check for immune characters
+//		if ( containsCharacter(codePoint, immune ) ) {
+//			return ""+codePoint;
+//		}
+		
+//		// check for alphanumeric characters
+		String hex = Codec.getHexForNonAlphanumeric(codePoint);
+//		if ( hex == null ) {
+//			return ""+c;
+//		}
+//		
+//		// check for illegal characters
+//		if ( ( c <= 0x1f && c != '\t' && c != '\n' && c != '\r' ) || ( c >= 0x7f && c <= 0x9f ) )
+//		{
+//			hex = REPLACEMENT_HEX;	// Let's entity encode this instead of returning it
+//			c = REPLACEMENT_CHAR;
+//		}
+//		
+//		// check if there's a defined entity
+//		String entityName = (String) characterToEntityMap.get(c);
+//		if (entityName != null) {
+//			return "&" + entityName + ";";
+//		}
+		
+		// return the hex entity as suggested in the spec
+		return "&#x" + hex + ";";
+	}
+	
 	/**
 	 * {@inheritDoc}
 	 * 

src/test/java/org/owasp/esapi/reference/EncoderTest.java

@@ -32,6 +32,7 @@
 import org.owasp.esapi.EncoderConstants;
 import org.owasp.esapi.codecs.Base64;
 import org.owasp.esapi.codecs.Codec;
+import org.owasp.esapi.codecs.HTMLEntityCodec;
 import org.owasp.esapi.codecs.MySQLCodec;
 import org.owasp.esapi.codecs.OracleCodec;
 import org.owasp.esapi.codecs.PushbackString;
@@ -902,7 +903,27 @@ public void testGetCanonicalizedUriWithMailto() throws Exception {
     	URI uri = new URI(input);
     	System.out.println(uri.toString());
     	assertEquals(expectedUri, e.getCanonicalizedURI(uri));
-    	
+    }
+    
+    public void testHtmlEncodeStrSurrogatePair()
+    {
+    	Encoder enc = ESAPI.encoder();
+        String inStr = new String (new int[]{0x2f804}, 0, 1);
+        assertEquals(false, Character.isBmpCodePoint(inStr.codePointAt(0)));
+        assertEquals(true, Character.isBmpCodePoint(new String(new int[] {0x0a}, 0, 1).codePointAt(0)));
+        String expected = "你";
+        String result;
+
+        result = enc.encodeForHTML(inStr);
+        assertEquals(expected, result);
+    }
+    
+    public void testHtmlDecodeHexEntititesSurrogatePair()
+    {
+    	HTMLEntityCodec htmlCodec = new HTMLEntityCodec();
+        String expected = new String (new int[]{0x2f804}, 0, 1);
+        assertEquals( expected, htmlCodec.decode("你") );
+        assertEquals( expected, htmlCodec.decode("你") );
     }
 }