Skip UTF8 to UTF16 conversion during document indexing

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/ESJsonFactory.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.xcontent.provider.json;
+
+import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonFactoryBuilder;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.io.IOContext;
+import com.fasterxml.jackson.core.json.ByteSourceJsonBootstrapper;
+import com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer;
+
+import java.io.IOException;
+
+public class ESJsonFactory extends JsonFactory {
+    ESJsonFactory(JsonFactoryBuilder b) {
+        super(b);
+    }
+
+    @Override
+    protected JsonParser _createParser(byte[] data, int offset, int len, IOContext ctxt) throws IOException {
+        if (len > 0
+            && Feature.CHARSET_DETECTION.enabledIn(_factoryFeatures)
+            && Feature.CANONICALIZE_FIELD_NAMES.enabledIn(_factoryFeatures)) {
+            var bootstrap = new ByteSourceJsonBootstrapper(ctxt, data, offset, len);
+            var encoding = bootstrap.detectEncoding();
+            if (encoding == JsonEncoding.UTF8) {
+                boolean invalidBom = false;
+                int ptr = offset;
+                // Skip over the BOM if present
+                if ((data[ptr] & 0xFF) == 0xEF) {
+                    if (len < 3) {
+                        invalidBom = true;
+                    } else if ((data[ptr + 1] & 0xFF) != 0xBB) {
+                        invalidBom = true;
+                    } else if ((data[ptr + 2] & 0xFF) != 0xBF) {
+                        invalidBom = true;
+                    } else {
+                        ptr += 3;
+                    }
+                }
+                if (invalidBom == false) {
+                    ByteQuadsCanonicalizer can = _byteSymbolCanonicalizer.makeChild(_factoryFeatures);
+                    return new ESUTF8StreamJsonParser(
+                        ctxt,
+                        _parserFeatures,
+                        null,
+                        _objectCodec,
+                        can,
+                        data,
+                        ptr,
+                        offset + len,
+                        ptr - offset,
+                        false
+                    );
+                }
+            }
+        }
+        return new ByteSourceJsonBootstrapper(ctxt, data, offset, len).constructParser(
+            _parserFeatures,
+            _objectCodec,
+            _byteSymbolCanonicalizer,
+            _rootCharSymbols,
+            _factoryFeatures
+        );
+    }
+}

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/ESJsonFactoryBuilder.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.xcontent.provider.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonFactoryBuilder;
+
+public class ESJsonFactoryBuilder extends JsonFactoryBuilder {
+    @Override
+    public JsonFactory build() {
+        return new ESJsonFactory(this);
+    }
+}

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/ESUTF8StreamJsonParser.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.xcontent.provider.json;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.core.SerializableString;
+import com.fasterxml.jackson.core.io.IOContext;
+import com.fasterxml.jackson.core.json.UTF8StreamJsonParser;
+import com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer;
+
+import org.elasticsearch.xcontent.Text;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+public class ESUTF8StreamJsonParser extends UTF8StreamJsonParser {
+    protected int stringEnd = -1;
+
+    public ESUTF8StreamJsonParser(
+        IOContext ctxt,
+        int features,
+        InputStream in,
+        ObjectCodec codec,
+        ByteQuadsCanonicalizer sym,
+        byte[] inputBuffer,
+        int start,
+        int end,
+        int bytesPreProcessed,
+        boolean bufferRecyclable
+    ) {
+        super(ctxt, features, in, codec, sym, inputBuffer, start, end, bytesPreProcessed, bufferRecyclable);
+    }
+
+    /**
+     * Method that will try to get underlying UTF-8 encoded bytes of the current string token.
+     * This is only a best-effort attempt; if there is some reason the bytes cannot be retrieved, this method will return null.
+     * Currently, this is only implemented for ascii-only strings that do not contain escaped characters.
+     */
+    public Text getValueAsText() throws IOException {
+        if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete) {
+            if (stringEnd > 0) {
+                final int len = stringEnd - 1 - _inputPtr;
+                // For now, we can use `len` for `charCount` because we only support ascii-encoded unescaped strings,
+                // which means each character uses exactly 1 byte.
+                return new Text(ByteBuffer.wrap(_inputBuffer, _inputPtr, len), len);
+            }
+            return _finishAndReturnText();
+        }
+        return null;
+    }
+
+    protected Text _finishAndReturnText() throws IOException {
+        int ptr = _inputPtr;
+        if (ptr >= _inputEnd) {
+            _loadMoreGuaranteed();
+            ptr = _inputPtr;
+        }
+
+        int startPtr = ptr;
+        final int[] codes = INPUT_CODES_UTF8;
+        final int max = _inputEnd;
+        final byte[] inputBuffer = _inputBuffer;
+        while (ptr < max) {
+            int c = inputBuffer[ptr] & 0xFF;
+            if (codes[c] != 0) {
+                if (c == INT_QUOTE) {
+                    stringEnd = ptr + 1;
+                    final int len = ptr - startPtr;
+                    // For now, we can use `len` for `charCount` because we only support ascii-encoded unescaped strings,
+                    // which means each character uses exactly 1 byte.
+                    return new Text(ByteBuffer.wrap(inputBuffer, startPtr, len), len);
+                }
+                return null;
+            }
+            ++ptr;
+        }
+        return null;
+    }
+
+    @Override
+    public JsonToken nextToken() throws IOException {
+        if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete && stringEnd > 0) {
+            _inputPtr = stringEnd;
+            _tokenIncomplete = false;
+        }
+        stringEnd = -1;
+        return super.nextToken();
+    }
+
+    @Override
+    public boolean nextFieldName(SerializableString str) throws IOException {
+        if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete && stringEnd > 0) {
+            _inputPtr = stringEnd;
+            _tokenIncomplete = false;
+        }
+        stringEnd = -1;
+        return super.nextFieldName(str);
+    }
+
+    @Override
+    public String nextFieldName() throws IOException {
+        if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete && stringEnd > 0) {
+            _inputPtr = stringEnd;
+            _tokenIncomplete = false;
+        }
+        stringEnd = -1;
+        return super.nextFieldName();
+    }
+}

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java

@@ -11,7 +11,6 @@
 
 import com.fasterxml.jackson.core.JsonEncoding;
 import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonFactoryBuilder;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 
@@ -47,7 +46,7 @@ public static final XContent jsonXContent() {
     }
 
     static {
-        jsonFactory = XContentImplUtils.configure(new JsonFactoryBuilder());
+        jsonFactory = XContentImplUtils.configure(new ESJsonFactoryBuilder());
         jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
         jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
         jsonFactory.configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java

@@ -18,6 +18,7 @@
 import com.fasterxml.jackson.core.io.JsonEOFException;
 
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.xcontent.Text;
 import org.elasticsearch.xcontent.XContentEOFException;
 import org.elasticsearch.xcontent.XContentLocation;
 import org.elasticsearch.xcontent.XContentParseException;
@@ -115,6 +116,20 @@ public String text() throws IOException {
         }
     }
 
+    @Override
+    public Text optimizedText() throws IOException {
+        if (currentToken().isValue() == false) {
+            throwOnNoText();
+        }
+        if (parser instanceof ESUTF8StreamJsonParser esParser) {
+            var bytesRef = esParser.getValueAsText();
+            if (bytesRef != null) {
+                return bytesRef;
+            }
+        }
+        return new Text(text());
+    }
+
     private void throwOnNoText() {
         throw new IllegalArgumentException("Expected text at " + getTokenLocation() + " but found " + currentToken());
     }

libs/x-content/impl/src/test/java/org/elasticsearch/xcontent/provider/json/ESJsonFactoryTests.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.xcontent.provider.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.json.ReaderBasedJsonParser;
+import com.fasterxml.jackson.core.json.UTF8StreamJsonParser;
+
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+public class ESJsonFactoryTests extends ESTestCase {
+
+    public void testCreateParser() throws IOException {
+        JsonFactory factory = new ESJsonFactoryBuilder().build();
+        assertThat(factory, Matchers.instanceOf(ESJsonFactory.class));
+
+        // \ufeff is the BOM
+        String[] inputs = { "{\"foo\": \"bar\"}", "\ufeff{\"foo\": \"bar\"}" };
+        Charset[] charsets = { StandardCharsets.UTF_8, StandardCharsets.UTF_16LE, StandardCharsets.UTF_16BE };
+        Class<?>[] expectedParsers = { ESUTF8StreamJsonParser.class, ReaderBasedJsonParser.class, ReaderBasedJsonParser.class };
+
+        for (String input : inputs) {
+            for (int i = 0; i < charsets.length; i++) {
+                ByteBuffer encoded = charsets[i].encode(input);
+                JsonParser parser = factory.createParser(encoded.array());
+                assertThat(parser, Matchers.instanceOf(expectedParsers[i]));
+                assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
+                assertThat(parser.nextFieldName(), Matchers.equalTo("foo"));
+                assertThat(parser.nextValue(), Matchers.equalTo(JsonToken.VALUE_STRING));
+                assertThat(parser.getValueAsString(), Matchers.equalTo("bar"));
+            }
+        }
+
+        // Valid BOM
+        {
+            JsonParser parser = factory.createParser(new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF, '{', '}' });
+            assertThat(parser, Matchers.instanceOf(ESUTF8StreamJsonParser.class));
+            assertThat(parser.nextToken(), Matchers.equalTo(JsonToken.START_OBJECT));
+        }
+
+        // Invalid BOMs
+        {
+            JsonParser parser = factory.createParser(new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBB, '{', '}' });
+            assertThat(parser, Matchers.instanceOf(UTF8StreamJsonParser.class));
+            assertThrows("Invalid UTF-8 start byte 0xbb", JsonParseException.class, parser::nextToken);
+        }
+
+        {
+            JsonParser parser = factory.createParser(new byte[] { (byte) 0xEF, '{', '}' });
+            assertThat(parser, Matchers.instanceOf(UTF8StreamJsonParser.class));
+            assertThrows("Invalid UTF-8 start byte 0x7b", JsonParseException.class, parser::nextToken);
+        }
+    }
+}