From 567af8f7b2e025cf7ac3c7f243a1ab8ac58b3f35 Mon Sep 17 00:00:00 2001 From: Piotr Rzysko Date: Sun, 20 Aug 2023 16:26:38 +0200 Subject: [PATCH 1/2] Number parsing --- .gitignore | 1 + README.md | 1 - build.gradle | 17 + .../org/simdjson/NumberParserBenchmark.java | 49 ++ .../org/simdjson/SimdJsonPaddingUtil.java | 12 + src/main/java/org/simdjson/JsonValue.java | 3 + src/main/java/org/simdjson/NumberParser.java | 606 +++++++++++++-- .../java/org/simdjson/NumberParserTables.java | 729 ++++++++++++++++++ src/main/java/org/simdjson/Tape.java | 16 +- src/main/java/org/simdjson/TapeBuilder.java | 10 +- .../simdjson/BenchmarkCorrectnessTest.java | 22 + .../java/org/simdjson/NumberParsingTest.java | 599 ++++++++++++++ .../java/org/simdjson/SimdJsonParserTest.java | 129 +--- src/test/java/org/simdjson/StringUtils.java | 15 +- 14 files changed, 2027 insertions(+), 182 deletions(-) create mode 100644 src/jmh/java/org/simdjson/NumberParserBenchmark.java create mode 100644 src/main/java/org/simdjson/NumberParserTables.java create mode 100644 src/test/java/org/simdjson/NumberParsingTest.java diff --git a/.gitignore b/.gitignore index 89c36aa..5241245 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .gradle build profilers +testdata diff --git a/README.md b/README.md index 1d803bf..a89a527 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ This implementation is still missing several features available in simdsjon. For * Support for Unicode characters * UTF-8 validation -* Full support for parsing floats * Support for 512-bit vectors ## Code Sample diff --git a/build.gradle b/build.gradle index 21cf3c5..edebdff 100644 --- a/build.gradle +++ b/build.gradle @@ -1,10 +1,12 @@ import me.champeau.jmh.JmhBytecodeGeneratorTask import org.gradle.internal.os.OperatingSystem +import org.ajoberstar.grgit.Grgit plugins { id 'java' id 'scala' id 'me.champeau.jmh' version '0.7.1' + id 'org.ajoberstar.grgit' version '5.2.0' } group = 'org.simdjson' @@ -37,7 +39,22 @@ dependencies { testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: junitVersion } +tasks.register('downloadTestData') { + doFirst { + def testDataDir = new File("${project.projectDir.getAbsolutePath()}/testdata") + if (!testDataDir.exists()) { + testDataDir.mkdir() + } + def numbersTestDataDir = new File("${testDataDir}/parse-number-fxx-test-data") + if (!numbersTestDataDir.exists()) { + def grgit = Grgit.clone(dir: numbersTestDataDir, uri: 'https://github.com/nigeltao/parse-number-fxx-test-data.git') + grgit.close() + } + } +} + test { + dependsOn downloadTestData useJUnitPlatform() jvmArgs += [ '--add-modules', 'jdk.incubator.vector', diff --git a/src/jmh/java/org/simdjson/NumberParserBenchmark.java b/src/jmh/java/org/simdjson/NumberParserBenchmark.java new file mode 100644 index 0000000..1b8c9dd --- /dev/null +++ b/src/jmh/java/org/simdjson/NumberParserBenchmark.java @@ -0,0 +1,49 @@ +package org.simdjson; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import static org.simdjson.SimdJsonPaddingUtil.padWithSpaces; + +@State(Scope.Benchmark) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +public class NumberParserBenchmark { + + private final Tape tape = new Tape(100); + private final NumberParser numberParser = new NumberParser(tape); + + @Param({ + "2.2250738585072013e-308", // fast path + "1.00000000000000188558920870223463870174566020691753515394643550663070558368373221972569761144603605635692374830246134201063722058e-309" // slow path + }) + String number; + byte[] numberUtf8Bytes; + + @Setup(Level.Trial) + public void setup() throws IOException { + numberUtf8Bytes = padWithSpaces(number); + } + + @Benchmark + public double baseline() { + return Double.parseDouble(number); + } + + @Benchmark + public double simdjson() { + tape.reset(); + numberParser.parseNumber(numberUtf8Bytes, 0); + return tape.getDouble(0); + } +} diff --git a/src/jmh/java/org/simdjson/SimdJsonPaddingUtil.java b/src/jmh/java/org/simdjson/SimdJsonPaddingUtil.java index 347e59d..74736fa 100644 --- a/src/jmh/java/org/simdjson/SimdJsonPaddingUtil.java +++ b/src/jmh/java/org/simdjson/SimdJsonPaddingUtil.java @@ -1,5 +1,9 @@ package org.simdjson; +import java.util.Arrays; + +import static java.nio.charset.StandardCharsets.UTF_8; + class SimdJsonPaddingUtil { static byte[] padded(byte[] src) { @@ -7,4 +11,12 @@ static byte[] padded(byte[] src) { System.arraycopy(src, 0, bufferPadded, 0, src.length); return bufferPadded; } + + static byte[] padWithSpaces(String str) { + byte[] strBytes = str.getBytes(UTF_8); + byte[] padded = new byte[strBytes.length + 64]; + Arrays.fill(padded, (byte) ' '); + System.arraycopy(strBytes, 0, padded, 0, strBytes.length); + return padded; + } } diff --git a/src/main/java/org/simdjson/JsonValue.java b/src/main/java/org/simdjson/JsonValue.java index 8d08183..ae254b9 100644 --- a/src/main/java/org/simdjson/JsonValue.java +++ b/src/main/java/org/simdjson/JsonValue.java @@ -112,6 +112,9 @@ public String toString() { case INT64 -> { return String.valueOf(asLong()); } + case DOUBLE -> { + return String.valueOf(asDouble()); + } case TRUE_VALUE, FALSE_VALUE -> { return String.valueOf(asBoolean()); } diff --git a/src/main/java/org/simdjson/NumberParser.java b/src/main/java/org/simdjson/NumberParser.java index 6506a95..b83c2ea 100644 --- a/src/main/java/org/simdjson/NumberParser.java +++ b/src/main/java/org/simdjson/NumberParser.java @@ -1,17 +1,59 @@ package org.simdjson; +import static java.lang.Double.NEGATIVE_INFINITY; +import static java.lang.Double.POSITIVE_INFINITY; +import static java.lang.Double.longBitsToDouble; +import static java.lang.Long.compareUnsigned; +import static java.lang.Long.divideUnsigned; +import static java.lang.Long.numberOfLeadingZeros; +import static java.lang.Long.remainderUnsigned; +import static java.lang.Math.abs; +import static java.lang.Math.unsignedMultiplyHigh; import static org.simdjson.JsonCharUtils.isStructuralOrWhitespace; +import static org.simdjson.NumberParserTables.NUMBER_OF_ADDITIONAL_DIGITS_AFTER_LEFT_SHIFT; +import static org.simdjson.NumberParserTables.POWERS_OF_FIVE; +import static org.simdjson.NumberParserTables.POWER_OF_FIVE_DIGITS; class NumberParser { - private static final int SMALLEST_POWER = -342; - private static final int LARGEST_POWER = 308; - private static final double[] POWER_OF_TEN = { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + // When parsing doubles, we assume that a long used to store digits is unsigned. Thus, it can safely accommodate + // up to 19 digits (9999999999999999999 < 2^64). + private static final int FAST_PATH_MAX_DIGIT_COUNT = 19; + // The smallest non-zero number representable in binary64 is 2^-1074, which is about 4.941 * 10^-324. + // If we consider a number in the form of w * 10^q where 1 <= w <= 9999999999999999999, then + // 1 * 10^q <= w * 10^q <= 9.999999999999999999 * 10^18 * 10^q. To ensure w * 10^q < 2^-1074, q must satisfy the + // following inequality: 9.999999999999999999 * 10^(18 + q) < 2^-1074. This condition holds true whenever + // 18 + q < -324. Thus, for q < -342, we can reliably conclude that the number w * 10^q is smaller than 2^-1074, + // and this, in turn means the number is equal to zero. + private static final int FAST_PATH_MIN_POWER_OF_TEN = -342; + // We know that (1 - 2^-53) * 2^1024, which is about 1.798 * 10^308, is the largest number representable in binary64. + // When the parsed number is expressed as w * 10^q, where w >= 1, we are sure that for any q > 308, the number is + // infinite. + private static final int FAST_PATH_MAX_POWER_OF_TEN = 308; + private static final double[] POWERS_OF_TEN = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 }; + private static final long MAX_LONG_REPRESENTED_AS_DOUBLE_EXACTLY = 9007199254740991L; // 2^53 - 1 + private static final int IEEE64_EXPONENT_BIAS = 1023; + private static final int IEEE64_SIGN_BIT_INDEX = 63; + private static final int IEEE64_SIGNIFICAND_EXPLICIT_BIT_COUNT = 52; + private static final int IEEE64_SIGNIFICAND_SIZE_IN_BITS = IEEE64_SIGNIFICAND_EXPLICIT_BIT_COUNT + 1; + private static final int IEEE64_MAX_FINITE_NUMBER_EXPONENT = 1023; + private static final int IEEE64_MIN_FINITE_NUMBER_EXPONENT = -1022; + private static final int IEEE64_SUBNORMAL_EXPONENT = -1023; + private static final int LONG_MAX_DIGIT_COUNT = 19; + // This is the upper limit for the count of decimal digits taken into account in the slow path. All digits exceeding + // this threshold are excluded. + private static final int SLOW_PATH_MAX_DIGIT_COUNT = 800; + private static final int SLOW_PATH_MAX_SHIFT = 60; + private static final byte[] SLOW_PATH_SHIFTS = { + 0, 3, 6, 9, 13, 16, 19, 23, 26, 29, + 33, 36, 39, 43, 46, 49, 53, 56, 59, + }; private final Tape tape; + private final SlowPathDecimal slowPathDecimal = new SlowPathDecimal(); private int currentIdx; @@ -19,16 +61,19 @@ class NumberParser { this.tape = tape; } - void parseNumber(byte[] buffer, int buffIdx, int jsonIdx) { - boolean negative = buffer[buffIdx] == '-'; + void parseNumber(byte[] buffer, int offset) { + boolean negative = buffer[offset] == '-'; - currentIdx = negative ? buffIdx + 1 : buffIdx; + currentIdx = negative ? offset + 1 : offset; - int startDigitsIdx = currentIdx; - long i = parseDigits(buffer, 0); - int digitCount = currentIdx - startDigitsIdx; - if (digitCount == 0 || ('0' == buffer[startDigitsIdx] && digitCount > 1)) { - throwException(jsonIdx); + int digitsStartIdx = currentIdx; + long digits = parseDigits(buffer, 0); + int digitCount = currentIdx - digitsStartIdx; + if (digitCount == 0) { + throw new JsonParsingException("Invalid number. Minus has to be followed by a digit."); + } + if ('0' == buffer[digitsStartIdx] && digitCount > 1) { + throw new JsonParsingException("Invalid number. Leading zeroes are not allowed."); } long exponent = 0; @@ -36,102 +81,535 @@ void parseNumber(byte[] buffer, int buffIdx, int jsonIdx) { if ('.' == buffer[currentIdx]) { isDouble = true; currentIdx++; - int firstAfterPeriod = currentIdx; - i = parseDigits(buffer, i); - exponent = firstAfterPeriod - currentIdx; + int firstIdxAfterPeriod = currentIdx; + digits = parseDigits(buffer, digits); + exponent = firstIdxAfterPeriod - currentIdx; if (exponent == 0) { - throwException(jsonIdx); + throw new JsonParsingException("Invalid number. Decimal point has to be followed by a digit."); } + digitCount = currentIdx - digitsStartIdx; } - if ('e' == buffer[currentIdx] || 'E' == buffer[currentIdx]) { + if (isExponentIndicator(buffer[currentIdx])) { isDouble = true; currentIdx++; - exponent = parseExponent(buffer, jsonIdx, exponent); + exponent = parseExponent(buffer, exponent); } if (!isStructuralOrWhitespace(buffer[currentIdx])) { - throwException(jsonIdx); + throw new JsonParsingException("Number has to be followed by a structural character or whitespace."); } if (isDouble) { - if (digitCount > 19 && significantDigits(buffer, startDigitsIdx, digitCount) > 19) { - throw new UnsupportedOperationException("Not implemented yet"); - } else if (exponent < SMALLEST_POWER || exponent > LARGEST_POWER) { - if (exponent < SMALLEST_POWER || i == 0) { - tape.appendDouble(negative ? -0.0 : 0.0); - } else { - throwException(jsonIdx); - } + double d; + if (shouldBeHandledBySlowPath(buffer, digitsStartIdx, digitCount)) { + d = slowlyParseDouble(buffer, offset); } else { - double computed = computeDouble(exponent, i, negative); - tape.appendDouble(computed); + d = computeDouble(negative, digits, exponent); } + tape.appendDouble(d); } else { - tape.appendInt64(negative ? (~i + 1) : i); + if (isOutOfLongRange(negative, digits, digitCount)) { + throw new JsonParsingException("Number value is out of long range ([" + Long.MIN_VALUE + ", " + Long.MAX_VALUE + "])."); + } + tape.appendInt64(negative ? (~digits + 1) : digits); + } + } + + private static boolean isOutOfLongRange(boolean negative, long digits, int digitCount) { + if (digitCount < LONG_MAX_DIGIT_COUNT) { + return false; } + if (digitCount > LONG_MAX_DIGIT_COUNT) { + return true; + } + if (negative && digits == Long.MIN_VALUE) { + // The maximum value we can store in a long is 9223372036854775807. When we try to store 9223372036854775808, + // a long wraps around, resulting in -9223372036854775808 (Long.MIN_VALUE). If the number we are parsing is + // negative, and we've attempted to store 9223372036854775808 in "digits", we can be sure that we are + // dealing with Long.MIN_VALUE, which obviously does not fall outside the acceptable range. + return false; + } + return digits < 0; } - private double computeDouble(long power, long i, boolean negative) { - if (-22 < power && power <= 22 && i <= 9007199254740991L) { - double d = i; - if (power < 0) { - d = d / POWER_OF_TEN[(int) -power]; + private static double computeDouble(boolean negative, long significand10, long exp10) { + if (abs(exp10) < POWERS_OF_TEN.length && compareUnsigned(significand10, MAX_LONG_REPRESENTED_AS_DOUBLE_EXACTLY) <= 0) { + // This path has been described in https://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/. + double d = significand10; + if (exp10 < 0) { + d = d / POWERS_OF_TEN[(int) -exp10]; } else { - d = d * POWER_OF_TEN[(int) power]; + d = d * POWERS_OF_TEN[(int) exp10]; } - if (negative) { - d = -d; + return negative ? -d : d; + } + + // The following path is an implementation of the Eisel-Lemire algorithm described by Daniel Lemire in + // "Number Parsing at a Gigabyte per Second" (https://arxiv.org/abs/2101.11408). + + if (exp10 < FAST_PATH_MIN_POWER_OF_TEN || significand10 == 0) { + return zero(negative); + } else if (exp10 > FAST_PATH_MAX_POWER_OF_TEN) { + return infinity(negative); + } + + // We start by normalizing the decimal significand so that it is within the range of [2^63, 2^64). + int lz = numberOfLeadingZeros(significand10); + significand10 <<= lz; + + // Initially, the number we are parsing is in the form of w * 10^q = w * 5^q * 2^q, and our objective is to + // convert it to m * 2^p. We can represent w * 10^q as w * 5^q * 2^r * 2^p, where w * 5^q * 2^r = m. + // As a result, the next step involves computing w * 5^q. The implementation of this multiplication is optimized + // to minimize necessary operations while ensuring precise results. For further insight, refer to the + // aforementioned paper. + int powersOfFiveTableIndex = 2 * (int) (exp10 - FAST_PATH_MIN_POWER_OF_TEN); + long upper = unsignedMultiplyHigh(significand10, POWERS_OF_FIVE[powersOfFiveTableIndex]); + long lower = significand10 * POWERS_OF_FIVE[powersOfFiveTableIndex]; + if ((upper & 0x1FF) == 0x1FF) { + long secondUpper = unsignedMultiplyHigh(significand10, POWERS_OF_FIVE[powersOfFiveTableIndex + 1]); + lower += secondUpper; + if (compareUnsigned(secondUpper, lower) > 0) { + upper++; } - return d; + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without Fallback" + // (https://arxiv.org/abs/2212.06644), at this point we are sure that the product is sufficiently accurate, + // and more computation is not needed. } - if (i == 0) { - return negative ? -0.0 : 0.0; + + // Here, we extract the binary significand from the product. Although in binary64 the significand has 53 bits, + // we extract 54 bits to use the least significant bit for rounding. Since both the decimal significand and the + // values stored in POWERS_OF_FIVE are normalized, ensuring that their most significant bits are set, the product + // has either 0 or 1 leading zeros. As a result, we need to perform a right shift of either 9 or 10 bits. + long upperBit = upper >>> 63; + long significand2 = upper >>> (upperBit + 9); + + long exp2 = ((217706 * exp10) >> 16) + 63 - lz + upperBit; + if (exp2 < IEEE64_MIN_FINITE_NUMBER_EXPONENT) { + // In the next step, we right-shift the binary significand by the difference between the minimum exponent + // and the binary exponent. In Java, the shift distance is limited to the range of 0 to 63, inclusive. + // Thus, we need to handle the case when the distance is >= 64 separately and always return zero. + if (exp2 <= IEEE64_MIN_FINITE_NUMBER_EXPONENT - 64) { + return zero(negative); + } + + // In this branch, it is likely that we are handling a subnormal number. Therefore, we adjust the significand + // to conform to the formula representing subnormal numbers: (significand2 * 2^(1 - IEEE64_EXPONENT_BIAS)) / 2^52. + significand2 >>= 1 - IEEE64_EXPONENT_BIAS - exp2; + // Round up if the significand is odd and remove the least significant bit that we've left for rounding. + significand2 += significand2 & 1; + significand2 >>= 1; + + // Here, we are addressing a scenario in which the original number was subnormal, but it became normal after + // rounding up. For example, when we are parsing 2.2250738585072013e-308 before rounding and removing the + // least significant bit significand2 = 0x3fffffffffffff and exp2 = -1023. After rounding, we get + // significand2 = 0x10000000000000, which is the significand of the smallest normal number. + exp2 = (significand2 < (1L << 52)) ? IEEE64_SUBNORMAL_EXPONENT : IEEE64_MIN_FINITE_NUMBER_EXPONENT; + return toDouble(negative, significand2, exp2); + } + + if ((compareUnsigned(lower, 1) <= 0) && (exp10 >= -4) && (exp10 <= 23) && ((significand2 & 3) == 1)) { + if (significand2 << (upperBit + 64 - 53 - 2) == upper) { + significand2 &= ~1; + } } - throw new UnsupportedOperationException("Not implemented yet"); + + // Round up if the significand is odd and remove the least significant bit that we've left for rounding. + significand2 += significand2 & 1; + significand2 >>= 1; + + if (significand2 == (1L << IEEE64_SIGNIFICAND_SIZE_IN_BITS)) { + // If we've reached here, it means that rounding has caused an overflow. We need to divide the significand + // by 2 and update the exponent accordingly. + significand2 >>= 1; + exp2++; + } + + if (exp2 > IEEE64_MAX_FINITE_NUMBER_EXPONENT) { + return infinity(negative); + } + return toDouble(negative, significand2, exp2); } - private long parseExponent(byte[] buffer, int jsonIdx, long exponent) { - boolean negExp = '-' == buffer[currentIdx]; - if (negExp || '+' == buffer[currentIdx]) { + // The following parser is based on the idea described in + // https://nigeltao.github.io/blog/2020/parse-number-f64-simple.html and implemented in + // https://github.com/simdjson/simdjson/blob/caff09cafceb0f5f6fc9109236d6dd09ac4bc0d8/src/from_chars.cpp + private double slowlyParseDouble(byte[] buffer, int offset) { + SlowPathDecimal decimal = slowPathDecimal; + decimal.reset(); + + decimal.negative = buffer[offset] == '-'; + currentIdx = decimal.negative ? offset + 1 : offset; + long exp10 = 0; + + skipZeros(buffer); + parseDigits(buffer, decimal); + if (buffer[currentIdx] == '.') { currentIdx++; + int firstIdxAfterPeriod = currentIdx; + if (decimal.digitCount == 0) { + skipZeros(buffer); + } + parseDigits(buffer, decimal); + exp10 = firstIdxAfterPeriod - currentIdx; + } + + int currentIdxMovingBackwards = currentIdx - 1; + int trailingZeros = 0; + // Here, we also skip the period to handle cases like 100000000000000000000.000000 + while (buffer[currentIdxMovingBackwards] == '0' || buffer[currentIdxMovingBackwards] == '.') { + if (buffer[currentIdxMovingBackwards] == '0') { + trailingZeros++; + } + currentIdxMovingBackwards--; + } + exp10 += decimal.digitCount; + decimal.digitCount -= trailingZeros; + + if (decimal.digitCount > SLOW_PATH_MAX_DIGIT_COUNT) { + decimal.digitCount = SLOW_PATH_MAX_DIGIT_COUNT; + decimal.truncated = true; + } + + if (isExponentIndicator(buffer[currentIdx])) { + currentIdx++; + exp10 = parseExponent(buffer, exp10); + } + + // At this point, the number we are parsing is represented in the following way: w * 10^exp10, where -1 < w < 1. + if (exp10 <= -324) { + // We know that -1e-324 < w * 10^exp10 < 1e-324. In binary64 -1e-324 = -0.0 and 1e-324 = +0.0, so we can + // safely return +/-0.0. + return zero(decimal.negative); + } else if (exp10 >= 310) { + // We know that either w * 10^exp10 <= -0.1e310 or w * 10^exp10 >= 0.1e310. + // In binary64 -0.1e310 = -inf and 0.1e310 = +inf, so we can safely return +/-inf. + return infinity(decimal.negative); + } + + decimal.exp10 = (int) exp10; + int exp2 = 0; + + // We start the following loop with the decimal in the form of w * 10^exp10. After a series of + // right-shifts (dividing by a power of 2), we transform the decimal into w' * 2^exp2 * 10^exp10, + // where exp10 is <= 0. Resultantly, w' * 10^exp10 is in the range of [0, 1). + while (decimal.exp10 > 0) { + int shift = resolveShiftDistanceBasedOnExponent10(decimal.exp10); + decimal.shiftRight(shift); + exp2 += shift; + } + + // Now, we are left-shifting to get to the point where w'' * 10^exp10 is within the range of [1/2, 1). + while (decimal.exp10 <= 0) { + int shift; + if (decimal.exp10 == 0) { + if (decimal.digits[0] >= 5) { + break; + } + shift = (decimal.digits[0] < 2) ? 2 : 1; + } else { + shift = resolveShiftDistanceBasedOnExponent10(-decimal.exp10); + } + decimal.shiftLeft(shift); + exp2 -= shift; + } + + // Here, w'' * 10^exp10 falls within the range of [1/2, 1). In binary64, the significand must be within the + // range of [1, 2). We can get to the target range by decreasing the binary exponent. Resultantly, the decimal + // is represented as w'' * 10^exp10 * 2^exp2, where w'' * 10^exp10 is in the range of [1, 2). + exp2--; + + while (IEEE64_MIN_FINITE_NUMBER_EXPONENT > exp2) { + int n = IEEE64_MIN_FINITE_NUMBER_EXPONENT - exp2; + if (n > SLOW_PATH_MAX_SHIFT) { + n = SLOW_PATH_MAX_SHIFT; + } + decimal.shiftRight(n); + exp2 += n; } - int startExp = currentIdx; - long expNumber = parseDigits(buffer, 0); + // To conform to the IEEE 754 standard, the binary significand must fall within the range of [2^52, 2^53). Hence, + // we perform the following multiplication. If, after this step, the significand is less than 2^52, we have a + // subnormal number, which we will address later. + decimal.shiftLeft(IEEE64_SIGNIFICAND_SIZE_IN_BITS); - if (startExp == currentIdx) { - throwException(jsonIdx); + long significand2 = decimal.computeSignificand(); + if (significand2 >= (1L << IEEE64_SIGNIFICAND_SIZE_IN_BITS)) { + // If we've reached here, it means that rounding has caused an overflow. We need to divide the significand + // by 2 and update the exponent accordingly. + significand2 >>= 1; + exp2++; } - if (currentIdx > startExp + 18) { - while (buffer[startExp] == '0') { - startExp++; + if (significand2 < (1L << IEEE64_SIGNIFICAND_EXPLICIT_BIT_COUNT)) { + exp2 = IEEE64_SUBNORMAL_EXPONENT; + } + if (exp2 > IEEE64_MAX_FINITE_NUMBER_EXPONENT) { + return infinity(decimal.negative); + } + return toDouble(decimal.negative, significand2, exp2); + } + + private static int resolveShiftDistanceBasedOnExponent10(int exp10) { + return (exp10 < SLOW_PATH_SHIFTS.length) ? SLOW_PATH_SHIFTS[exp10] : SLOW_PATH_MAX_SHIFT; + } + + private long parseExponent(byte[] buffer, long exponent) { + boolean negative = '-' == buffer[currentIdx]; + if (negative || '+' == buffer[currentIdx]) { + currentIdx++; + } + int exponentStartIdx = currentIdx; + long parsedExponent = parseDigits(buffer, 0); + if (exponentStartIdx == currentIdx) { + throw new JsonParsingException("Invalid number. Exponent indicator has to be followed by a digit."); + } + // Long.MAX_VALUE = 9223372036854775807 (19 digits). Therefore, any number with <= 18 digits can be safely + // stored in a long without causing an overflow. + int maxDigitCountLongCanAccommodate = 18; + if (currentIdx > exponentStartIdx + maxDigitCountLongCanAccommodate) { + // Potentially, we have an overflow here. We try to skip leading zeros. + while (buffer[exponentStartIdx] == '0') { + exponentStartIdx++; } - if (currentIdx > startExp + 18) { - expNumber = 999999999999999999L; + if (currentIdx > exponentStartIdx + maxDigitCountLongCanAccommodate) { + // We still have more digits than a long can safely accommodate. + // + // The largest finite number that can be represented in binary64 is (1-2^-53) * 2^1024, which is about + // 1.798e308, and the smallest non-zero number is 2^-1074, roughly 4.941e-324. So, we might, potentially, + // care only about numbers with explicit exponents falling within the range of [-324, 308], and return + // either zero or infinity for everything outside of this range.However, we have to take into account + // the fractional part of the parsed number. This part can potentially cancel out the value of the + // explicit exponent. For example, 1000e-325 (1 * 10^3 * 10^-325 = 1 * 10^-322) is not equal to zero + // despite the explicit exponent being less than -324. + // + // Let's consider a scenario where the explicit exponent is greater than 999999999999999999. As long as + // the fractional part has <= 999999999999999690 digits, it doesn't matter whether we take + // 999999999999999999 or its actual value as the explicit exponent. This is due to the fact that the + // parsed number is infinite anyway (w * 10^-q * 10^999999999999999999 > (1-2^-53) * 2^1024, 0 < w < 10, + // 0 <= q <= 999999999999999690). Similarly, in a scenario where the explicit exponent is less than + // -999999999999999999, as long as the fractional part has <= 999999999999999674 digits, we can safely + // take 999999999999999999 as the explicit exponent, given that the parsed number is zero anyway + // (w * 10^q * 10^-999999999999999999 < 2^-1074, 0 < w < 10, 0 <= q <= 999999999999999674) + // + // Note that if the fractional part had 999999999999999674 digits, the JSON size would need to be + // 999999999999999674 bytes, which is approximately ~888 PiB. Consequently, it's reasonable to assume + // that the fractional part contains no more than 999999999999999674 digits. + parsedExponent = 999999999999999999L; } } - exponent += (negExp ? -expNumber : expNumber); + // Note that we don't check if 'exponent' has overflowed after the following addition. This is because we + // know that the parsed exponent falls within the range of [-999999999999999999, 999999999999999999]. We also + // assume that 'exponent' before the addition is within the range of [-9223372036854775808, 9223372036854775807]. + // This assumption should always be valid as the value of 'exponent' is constrained by the size of the JSON input. + exponent += negative ? -parsedExponent : parsedExponent; return exponent; } - private long parseDigits(byte[] buffer, long acc) { - byte digit = (byte) (buffer[currentIdx] - '0'); + private long parseDigits(byte[] buffer, long digits) { + byte digit = convertCharacterToDigit(buffer[currentIdx]); while (digit >= 0 && digit <= 9) { - acc = 10 * acc + digit; + digits = 10 * digits + digit; currentIdx++; - digit = (byte) (buffer[currentIdx] - '0'); + digit = convertCharacterToDigit(buffer[currentIdx]); } - return acc; + return digits; } - private int significantDigits(byte[] buffer, int startDigitsIdx, int digitCount) { + private static boolean shouldBeHandledBySlowPath(byte[] buffer, int startDigitsIdx, int digitCount) { + if (digitCount <= FAST_PATH_MAX_DIGIT_COUNT) { + return false; + } int start = startDigitsIdx; while (buffer[start] == '0' || buffer[start] == '.') { start++; } - return digitCount - start - startDigitsIdx; + int significantDigitCount = digitCount - (start - startDigitsIdx); + return significantDigitCount > FAST_PATH_MAX_DIGIT_COUNT; + } + + private void skipZeros(byte[] buffer) { + while (buffer[currentIdx] == '0') { + currentIdx++; + } + } + + private void parseDigits(byte[] buffer, SlowPathDecimal decimal) { + while (isDigit(buffer[currentIdx])) { + if (decimal.digitCount < SLOW_PATH_MAX_DIGIT_COUNT) { + decimal.digits[decimal.digitCount] = convertCharacterToDigit(buffer[currentIdx]); + } + decimal.digitCount++; + currentIdx++; + } + } + + private static boolean isDigit(byte b) { + return b >= '0' && b <= '9'; + } + + private static boolean isExponentIndicator(byte b) { + return 'e' == b || 'E' == b; } - private void throwException(int idx) { - throw new JsonParsingException("Invalid number starting at " + idx); + private static double toDouble(boolean negative, long significand2, long exp2) { + long bits = significand2; + bits &= ~(1L << IEEE64_SIGNIFICAND_EXPLICIT_BIT_COUNT); // clear the implicit bit + bits |= (exp2 + IEEE64_EXPONENT_BIAS) << IEEE64_SIGNIFICAND_EXPLICIT_BIT_COUNT; + bits = negative ? (bits | (1L << IEEE64_SIGN_BIT_INDEX)) : bits; + return longBitsToDouble(bits); + } + + private static double infinity(boolean negative) { + return negative ? NEGATIVE_INFINITY : POSITIVE_INFINITY; + } + + private static double zero(boolean negative) { + return negative ? -0.0 : 0.0; + } + + private static byte convertCharacterToDigit(byte b) { + return (byte) (b - '0'); + } + + private static class SlowPathDecimal { + + final byte[] digits = new byte[SLOW_PATH_MAX_DIGIT_COUNT]; + int digitCount; + int exp10; + boolean truncated; + boolean negative; + + // Before calling this method we have to make sure that the significand is within the range of [0, 2^53 - 1]. + long computeSignificand() { + if (digitCount == 0 || exp10 < 0) { + return 0; + } + long significand = 0; + for (int i = 0; i < exp10; i++) { + significand = (10 * significand) + ((i < digitCount) ? digits[i] : 0); + } + boolean roundUp = false; + if (exp10 < digitCount) { + roundUp = digits[exp10] >= 5; + if ((digits[exp10] == 5) && (exp10 + 1 == digitCount)) { + // If the digits haven't been truncated, then we are exactly half-way between two integers. In such + // cases, we round to even, otherwise we round up. + roundUp = truncated || (significand & 1) == 1; + } + } + return roundUp ? ++significand : significand; + } + + void shiftLeft(int shift) { + if (digitCount == 0) { + return; + } + + int numberOfAdditionalDigits = calculateNumberOfAdditionalDigitsAfterLeftShift(shift); + int readIndex = digitCount - 1; + int writeIndex = digitCount - 1 + numberOfAdditionalDigits; + long n = 0; + + while (readIndex >= 0) { + n += (long) digits[readIndex] << shift; + long quotient = divideUnsigned(n, 10); + long remainder = remainderUnsigned(n, 10); + if (writeIndex < SLOW_PATH_MAX_DIGIT_COUNT) { + digits[writeIndex] = (byte) remainder; + } else if (remainder > 0) { + truncated = true; + } + n = quotient; + writeIndex--; + readIndex--; + } + + while (compareUnsigned(n, 0) > 0) { + long quotient = divideUnsigned(n, 10); + long remainder = remainderUnsigned(n, 10); + if (writeIndex < SLOW_PATH_MAX_DIGIT_COUNT) { + digits[writeIndex] = (byte) remainder; + } else if (remainder > 0) { + truncated = true; + } + n = quotient; + writeIndex--; + } + digitCount += numberOfAdditionalDigits; + if (digitCount > SLOW_PATH_MAX_DIGIT_COUNT) { + digitCount = SLOW_PATH_MAX_DIGIT_COUNT; + } + exp10 += numberOfAdditionalDigits; + trimTrailingZeros(); + } + + // See https://nigeltao.github.io/blog/2020/parse-number-f64-simple.html#hpd-shifts + private int calculateNumberOfAdditionalDigitsAfterLeftShift(int shift) { + int a = NUMBER_OF_ADDITIONAL_DIGITS_AFTER_LEFT_SHIFT[shift]; + int b = NUMBER_OF_ADDITIONAL_DIGITS_AFTER_LEFT_SHIFT[shift + 1]; + int newDigitCount = a >> 11; + int pow5OffsetA = 0x7FF & a; + int pow5OffsetB = 0x7FF & b; + + int n = pow5OffsetB - pow5OffsetA; + for (int i = 0; i < n; i++) { + if (i >= digitCount) { + return newDigitCount - 1; + } else if (digits[i] < POWER_OF_FIVE_DIGITS[pow5OffsetA + i]) { + return newDigitCount - 1; + } else if (digits[i] > POWER_OF_FIVE_DIGITS[pow5OffsetA + i]) { + return newDigitCount; + } + } + return newDigitCount; + } + + void shiftRight(int shift) { + int readIndex = 0; + int writeIndex = 0; + long n = 0; + + while ((n >>> shift) == 0) { + if (readIndex < digitCount) { + n = (10 * n) + digits[readIndex++]; + } else if (n == 0) { + return; + } else { + while ((n >>> shift) == 0) { + n = 10 * n; + readIndex++; + } + break; + } + } + exp10 -= (readIndex - 1); + long mask = (1L << shift) - 1; + while (readIndex < digitCount) { + byte newDigit = (byte) (n >>> shift); + n = (10 * (n & mask)) + digits[readIndex++]; + digits[writeIndex++] = newDigit; + } + while (compareUnsigned(n, 0) > 0) { + byte newDigit = (byte) (n >>> shift); + n = 10 * (n & mask); + if (writeIndex < SLOW_PATH_MAX_DIGIT_COUNT) { + digits[writeIndex++] = newDigit; + } else if (newDigit > 0) { + truncated = true; + } + } + digitCount = writeIndex; + trimTrailingZeros(); + } + + private void trimTrailingZeros() { + while ((digitCount > 0) && (digits[digitCount - 1] == 0)) { + digitCount--; + } + } + + private void reset() { + digitCount = 0; + exp10 = 0; + truncated = false; + } } } diff --git a/src/main/java/org/simdjson/NumberParserTables.java b/src/main/java/org/simdjson/NumberParserTables.java new file mode 100644 index 0000000..58f1fa0 --- /dev/null +++ b/src/main/java/org/simdjson/NumberParserTables.java @@ -0,0 +1,729 @@ +package org.simdjson; + +class NumberParserTables { + + static final int[] NUMBER_OF_ADDITIONAL_DIGITS_AFTER_LEFT_SHIFT = { + 0x0000, 0x0800, 0x0801, 0x0803, 0x1006, 0x1009, 0x100D, 0x1812, 0x1817, + 0x181D, 0x2024, 0x202B, 0x2033, 0x203C, 0x2846, 0x2850, 0x285B, 0x3067, + 0x3073, 0x3080, 0x388E, 0x389C, 0x38AB, 0x38BB, 0x40CC, 0x40DD, 0x40EF, + 0x4902, 0x4915, 0x4929, 0x513E, 0x5153, 0x5169, 0x5180, 0x5998, 0x59B0, + 0x59C9, 0x61E3, 0x61FD, 0x6218, 0x6A34, 0x6A50, 0x6A6D, 0x6A8B, 0x72AA, + 0x72C9, 0x72E9, 0x7B0A, 0x7B2B, 0x7B4D, 0x8370, 0x8393, 0x83B7, 0x83DC, + 0x8C02, 0x8C28, 0x8C4F, 0x9477, 0x949F, 0x94C8, 0x9CF2, 0x051C, 0x051C, + 0x051C, 0x051C + }; + + static final byte[] POWER_OF_FIVE_DIGITS = { + 5, 2, 5, 1, 2, 5, 6, 2, 5, 3, 1, 2, 5, 1, 5, 6, 2, 5, 7, 8, 1, 2, 5, + 3, 9, 0, 6, 2, 5, 1, 9, 5, 3, 1, 2, 5, 9, 7, 6, 5, 6, 2, 5, 4, 8, 8, + 2, 8, 1, 2, 5, 2, 4, 4, 1, 4, 0, 6, 2, 5, 1, 2, 2, 0, 7, 0, 3, 1, 2, + 5, 6, 1, 0, 3, 5, 1, 5, 6, 2, 5, 3, 0, 5, 1, 7, 5, 7, 8, 1, 2, 5, 1, + 5, 2, 5, 8, 7, 8, 9, 0, 6, 2, 5, 7, 6, 2, 9, 3, 9, 4, 5, 3, 1, 2, 5, + 3, 8, 1, 4, 6, 9, 7, 2, 6, 5, 6, 2, 5, 1, 9, 0, 7, 3, 4, 8, 6, 3, 2, + 8, 1, 2, 5, 9, 5, 3, 6, 7, 4, 3, 1, 6, 4, 0, 6, 2, 5, 4, 7, 6, 8, 3, + 7, 1, 5, 8, 2, 0, 3, 1, 2, 5, 2, 3, 8, 4, 1, 8, 5, 7, 9, 1, 0, 1, 5, + 6, 2, 5, 1, 1, 9, 2, 0, 9, 2, 8, 9, 5, 5, 0, 7, 8, 1, 2, 5, 5, 9, 6, + 0, 4, 6, 4, 4, 7, 7, 5, 3, 9, 0, 6, 2, 5, 2, 9, 8, 0, 2, 3, 2, 2, 3, + 8, 7, 6, 9, 5, 3, 1, 2, 5, 1, 4, 9, 0, 1, 1, 6, 1, 1, 9, 3, 8, 4, 7, + 6, 5, 6, 2, 5, 7, 4, 5, 0, 5, 8, 0, 5, 9, 6, 9, 2, 3, 8, 2, 8, 1, 2, + 5, 3, 7, 2, 5, 2, 9, 0, 2, 9, 8, 4, 6, 1, 9, 1, 4, 0, 6, 2, 5, 1, 8, + 6, 2, 6, 4, 5, 1, 4, 9, 2, 3, 0, 9, 5, 7, 0, 3, 1, 2, 5, 9, 3, 1, 3, + 2, 2, 5, 7, 4, 6, 1, 5, 4, 7, 8, 5, 1, 5, 6, 2, 5, 4, 6, 5, 6, 6, 1, + 2, 8, 7, 3, 0, 7, 7, 3, 9, 2, 5, 7, 8, 1, 2, 5, 2, 3, 2, 8, 3, 0, 6, + 4, 3, 6, 5, 3, 8, 6, 9, 6, 2, 8, 9, 0, 6, 2, 5, 1, 1, 6, 4, 1, 5, 3, + 2, 1, 8, 2, 6, 9, 3, 4, 8, 1, 4, 4, 5, 3, 1, 2, 5, 5, 8, 2, 0, 7, 6, + 6, 0, 9, 1, 3, 4, 6, 7, 4, 0, 7, 2, 2, 6, 5, 6, 2, 5, 2, 9, 1, 0, 3, + 8, 3, 0, 4, 5, 6, 7, 3, 3, 7, 0, 3, 6, 1, 3, 2, 8, 1, 2, 5, 1, 4, 5, + 5, 1, 9, 1, 5, 2, 2, 8, 3, 6, 6, 8, 5, 1, 8, 0, 6, 6, 4, 0, 6, 2, 5, + 7, 2, 7, 5, 9, 5, 7, 6, 1, 4, 1, 8, 3, 4, 2, 5, 9, 0, 3, 3, 2, 0, 3, + 1, 2, 5, 3, 6, 3, 7, 9, 7, 8, 8, 0, 7, 0, 9, 1, 7, 1, 2, 9, 5, 1, 6, + 6, 0, 1, 5, 6, 2, 5, 1, 8, 1, 8, 9, 8, 9, 4, 0, 3, 5, 4, 5, 8, 5, 6, + 4, 7, 5, 8, 3, 0, 0, 7, 8, 1, 2, 5, 9, 0, 9, 4, 9, 4, 7, 0, 1, 7, 7, + 2, 9, 2, 8, 2, 3, 7, 9, 1, 5, 0, 3, 9, 0, 6, 2, 5, 4, 5, 4, 7, 4, 7, + 3, 5, 0, 8, 8, 6, 4, 6, 4, 1, 1, 8, 9, 5, 7, 5, 1, 9, 5, 3, 1, 2, 5, + 2, 2, 7, 3, 7, 3, 6, 7, 5, 4, 4, 3, 2, 3, 2, 0, 5, 9, 4, 7, 8, 7, 5, + 9, 7, 6, 5, 6, 2, 5, 1, 1, 3, 6, 8, 6, 8, 3, 7, 7, 2, 1, 6, 1, 6, 0, + 2, 9, 7, 3, 9, 3, 7, 9, 8, 8, 2, 8, 1, 2, 5, 5, 6, 8, 4, 3, 4, 1, 8, + 8, 6, 0, 8, 0, 8, 0, 1, 4, 8, 6, 9, 6, 8, 9, 9, 4, 1, 4, 0, 6, 2, 5, + 2, 8, 4, 2, 1, 7, 0, 9, 4, 3, 0, 4, 0, 4, 0, 0, 7, 4, 3, 4, 8, 4, 4, + 9, 7, 0, 7, 0, 3, 1, 2, 5, 1, 4, 2, 1, 0, 8, 5, 4, 7, 1, 5, 2, 0, 2, + 0, 0, 3, 7, 1, 7, 4, 2, 2, 4, 8, 5, 3, 5, 1, 5, 6, 2, 5, 7, 1, 0, 5, + 4, 2, 7, 3, 5, 7, 6, 0, 1, 0, 0, 1, 8, 5, 8, 7, 1, 1, 2, 4, 2, 6, 7, + 5, 7, 8, 1, 2, 5, 3, 5, 5, 2, 7, 1, 3, 6, 7, 8, 8, 0, 0, 5, 0, 0, 9, + 2, 9, 3, 5, 5, 6, 2, 1, 3, 3, 7, 8, 9, 0, 6, 2, 5, 1, 7, 7, 6, 3, 5, + 6, 8, 3, 9, 4, 0, 0, 2, 5, 0, 4, 6, 4, 6, 7, 7, 8, 1, 0, 6, 6, 8, 9, + 4, 5, 3, 1, 2, 5, 8, 8, 8, 1, 7, 8, 4, 1, 9, 7, 0, 0, 1, 2, 5, 2, 3, + 2, 3, 3, 8, 9, 0, 5, 3, 3, 4, 4, 7, 2, 6, 5, 6, 2, 5, 4, 4, 4, 0, 8, + 9, 2, 0, 9, 8, 5, 0, 0, 6, 2, 6, 1, 6, 1, 6, 9, 4, 5, 2, 6, 6, 7, 2, + 3, 6, 3, 2, 8, 1, 2, 5, 2, 2, 2, 0, 4, 4, 6, 0, 4, 9, 2, 5, 0, 3, 1, + 3, 0, 8, 0, 8, 4, 7, 2, 6, 3, 3, 3, 6, 1, 8, 1, 6, 4, 0, 6, 2, 5, 1, + 1, 1, 0, 2, 2, 3, 0, 2, 4, 6, 2, 5, 1, 5, 6, 5, 4, 0, 4, 2, 3, 6, 3, + 1, 6, 6, 8, 0, 9, 0, 8, 2, 0, 3, 1, 2, 5, 5, 5, 5, 1, 1, 1, 5, 1, 2, + 3, 1, 2, 5, 7, 8, 2, 7, 0, 2, 1, 1, 8, 1, 5, 8, 3, 4, 0, 4, 5, 4, 1, + 0, 1, 5, 6, 2, 5, 2, 7, 7, 5, 5, 5, 7, 5, 6, 1, 5, 6, 2, 8, 9, 1, 3, + 5, 1, 0, 5, 9, 0, 7, 9, 1, 7, 0, 2, 2, 7, 0, 5, 0, 7, 8, 1, 2, 5, 1, + 3, 8, 7, 7, 7, 8, 7, 8, 0, 7, 8, 1, 4, 4, 5, 6, 7, 5, 5, 2, 9, 5, 3, + 9, 5, 8, 5, 1, 1, 3, 5, 2, 5, 3, 9, 0, 6, 2, 5, 6, 9, 3, 8, 8, 9, 3, + 9, 0, 3, 9, 0, 7, 2, 2, 8, 3, 7, 7, 6, 4, 7, 6, 9, 7, 9, 2, 5, 5, 6, + 7, 6, 2, 6, 9, 5, 3, 1, 2, 5, 3, 4, 6, 9, 4, 4, 6, 9, 5, 1, 9, 5, 3, + 6, 1, 4, 1, 8, 8, 8, 2, 3, 8, 4, 8, 9, 6, 2, 7, 8, 3, 8, 1, 3, 4, 7, + 6, 5, 6, 2, 5, 1, 7, 3, 4, 7, 2, 3, 4, 7, 5, 9, 7, 6, 8, 0, 7, 0, 9, + 4, 4, 1, 1, 9, 2, 4, 4, 8, 1, 3, 9, 1, 9, 0, 6, 7, 3, 8, 2, 8, 1, 2, + 5, 8, 6, 7, 3, 6, 1, 7, 3, 7, 9, 8, 8, 4, 0, 3, 5, 4, 7, 2, 0, 5, 9, + 6, 2, 2, 4, 0, 6, 9, 5, 9, 5, 3, 3, 6, 9, 1, 4, 0, 6, 2, 5 + }; + + static final long[] POWERS_OF_FIVE = { + 0xeef453d6923bd65aL, 0x113faa2906a13b3fL, + 0x9558b4661b6565f8L, 0x4ac7ca59a424c507L, + 0xbaaee17fa23ebf76L, 0x5d79bcf00d2df649L, + 0xe95a99df8ace6f53L, 0xf4d82c2c107973dcL, + 0x91d8a02bb6c10594L, 0x79071b9b8a4be869L, + 0xb64ec836a47146f9L, 0x9748e2826cdee284L, + 0xe3e27a444d8d98b7L, 0xfd1b1b2308169b25L, + 0x8e6d8c6ab0787f72L, 0xfe30f0f5e50e20f7L, + 0xb208ef855c969f4fL, 0xbdbd2d335e51a935L, + 0xde8b2b66b3bc4723L, 0xad2c788035e61382L, + 0x8b16fb203055ac76L, 0x4c3bcb5021afcc31L, + 0xaddcb9e83c6b1793L, 0xdf4abe242a1bbf3dL, + 0xd953e8624b85dd78L, 0xd71d6dad34a2af0dL, + 0x87d4713d6f33aa6bL, 0x8672648c40e5ad68L, + 0xa9c98d8ccb009506L, 0x680efdaf511f18c2L, + 0xd43bf0effdc0ba48L, 0x212bd1b2566def2L, + 0x84a57695fe98746dL, 0x14bb630f7604b57L, + 0xa5ced43b7e3e9188L, 0x419ea3bd35385e2dL, + 0xcf42894a5dce35eaL, 0x52064cac828675b9L, + 0x818995ce7aa0e1b2L, 0x7343efebd1940993L, + 0xa1ebfb4219491a1fL, 0x1014ebe6c5f90bf8L, + 0xca66fa129f9b60a6L, 0xd41a26e077774ef6L, + 0xfd00b897478238d0L, 0x8920b098955522b4L, + 0x9e20735e8cb16382L, 0x55b46e5f5d5535b0L, + 0xc5a890362fddbc62L, 0xeb2189f734aa831dL, + 0xf712b443bbd52b7bL, 0xa5e9ec7501d523e4L, + 0x9a6bb0aa55653b2dL, 0x47b233c92125366eL, + 0xc1069cd4eabe89f8L, 0x999ec0bb696e840aL, + 0xf148440a256e2c76L, 0xc00670ea43ca250dL, + 0x96cd2a865764dbcaL, 0x380406926a5e5728L, + 0xbc807527ed3e12bcL, 0xc605083704f5ecf2L, + 0xeba09271e88d976bL, 0xf7864a44c633682eL, + 0x93445b8731587ea3L, 0x7ab3ee6afbe0211dL, + 0xb8157268fdae9e4cL, 0x5960ea05bad82964L, + 0xe61acf033d1a45dfL, 0x6fb92487298e33bdL, + 0x8fd0c16206306babL, 0xa5d3b6d479f8e056L, + 0xb3c4f1ba87bc8696L, 0x8f48a4899877186cL, + 0xe0b62e2929aba83cL, 0x331acdabfe94de87L, + 0x8c71dcd9ba0b4925L, 0x9ff0c08b7f1d0b14L, + 0xaf8e5410288e1b6fL, 0x7ecf0ae5ee44dd9L, + 0xdb71e91432b1a24aL, 0xc9e82cd9f69d6150L, + 0x892731ac9faf056eL, 0xbe311c083a225cd2L, + 0xab70fe17c79ac6caL, 0x6dbd630a48aaf406L, + 0xd64d3d9db981787dL, 0x92cbbccdad5b108L, + 0x85f0468293f0eb4eL, 0x25bbf56008c58ea5L, + 0xa76c582338ed2621L, 0xaf2af2b80af6f24eL, + 0xd1476e2c07286faaL, 0x1af5af660db4aee1L, + 0x82cca4db847945caL, 0x50d98d9fc890ed4dL, + 0xa37fce126597973cL, 0xe50ff107bab528a0L, + 0xcc5fc196fefd7d0cL, 0x1e53ed49a96272c8L, + 0xff77b1fcbebcdc4fL, 0x25e8e89c13bb0f7aL, + 0x9faacf3df73609b1L, 0x77b191618c54e9acL, + 0xc795830d75038c1dL, 0xd59df5b9ef6a2417L, + 0xf97ae3d0d2446f25L, 0x4b0573286b44ad1dL, + 0x9becce62836ac577L, 0x4ee367f9430aec32L, + 0xc2e801fb244576d5L, 0x229c41f793cda73fL, + 0xf3a20279ed56d48aL, 0x6b43527578c1110fL, + 0x9845418c345644d6L, 0x830a13896b78aaa9L, + 0xbe5691ef416bd60cL, 0x23cc986bc656d553L, + 0xedec366b11c6cb8fL, 0x2cbfbe86b7ec8aa8L, + 0x94b3a202eb1c3f39L, 0x7bf7d71432f3d6a9L, + 0xb9e08a83a5e34f07L, 0xdaf5ccd93fb0cc53L, + 0xe858ad248f5c22c9L, 0xd1b3400f8f9cff68L, + 0x91376c36d99995beL, 0x23100809b9c21fa1L, + 0xb58547448ffffb2dL, 0xabd40a0c2832a78aL, + 0xe2e69915b3fff9f9L, 0x16c90c8f323f516cL, + 0x8dd01fad907ffc3bL, 0xae3da7d97f6792e3L, + 0xb1442798f49ffb4aL, 0x99cd11cfdf41779cL, + 0xdd95317f31c7fa1dL, 0x40405643d711d583L, + 0x8a7d3eef7f1cfc52L, 0x482835ea666b2572L, + 0xad1c8eab5ee43b66L, 0xda3243650005eecfL, + 0xd863b256369d4a40L, 0x90bed43e40076a82L, + 0x873e4f75e2224e68L, 0x5a7744a6e804a291L, + 0xa90de3535aaae202L, 0x711515d0a205cb36L, + 0xd3515c2831559a83L, 0xd5a5b44ca873e03L, + 0x8412d9991ed58091L, 0xe858790afe9486c2L, + 0xa5178fff668ae0b6L, 0x626e974dbe39a872L, + 0xce5d73ff402d98e3L, 0xfb0a3d212dc8128fL, + 0x80fa687f881c7f8eL, 0x7ce66634bc9d0b99L, + 0xa139029f6a239f72L, 0x1c1fffc1ebc44e80L, + 0xc987434744ac874eL, 0xa327ffb266b56220L, + 0xfbe9141915d7a922L, 0x4bf1ff9f0062baa8L, + 0x9d71ac8fada6c9b5L, 0x6f773fc3603db4a9L, + 0xc4ce17b399107c22L, 0xcb550fb4384d21d3L, + 0xf6019da07f549b2bL, 0x7e2a53a146606a48L, + 0x99c102844f94e0fbL, 0x2eda7444cbfc426dL, + 0xc0314325637a1939L, 0xfa911155fefb5308L, + 0xf03d93eebc589f88L, 0x793555ab7eba27caL, + 0x96267c7535b763b5L, 0x4bc1558b2f3458deL, + 0xbbb01b9283253ca2L, 0x9eb1aaedfb016f16L, + 0xea9c227723ee8bcbL, 0x465e15a979c1cadcL, + 0x92a1958a7675175fL, 0xbfacd89ec191ec9L, + 0xb749faed14125d36L, 0xcef980ec671f667bL, + 0xe51c79a85916f484L, 0x82b7e12780e7401aL, + 0x8f31cc0937ae58d2L, 0xd1b2ecb8b0908810L, + 0xb2fe3f0b8599ef07L, 0x861fa7e6dcb4aa15L, + 0xdfbdcece67006ac9L, 0x67a791e093e1d49aL, + 0x8bd6a141006042bdL, 0xe0c8bb2c5c6d24e0L, + 0xaecc49914078536dL, 0x58fae9f773886e18L, + 0xda7f5bf590966848L, 0xaf39a475506a899eL, + 0x888f99797a5e012dL, 0x6d8406c952429603L, + 0xaab37fd7d8f58178L, 0xc8e5087ba6d33b83L, + 0xd5605fcdcf32e1d6L, 0xfb1e4a9a90880a64L, + 0x855c3be0a17fcd26L, 0x5cf2eea09a55067fL, + 0xa6b34ad8c9dfc06fL, 0xf42faa48c0ea481eL, + 0xd0601d8efc57b08bL, 0xf13b94daf124da26L, + 0x823c12795db6ce57L, 0x76c53d08d6b70858L, + 0xa2cb1717b52481edL, 0x54768c4b0c64ca6eL, + 0xcb7ddcdda26da268L, 0xa9942f5dcf7dfd09L, + 0xfe5d54150b090b02L, 0xd3f93b35435d7c4cL, + 0x9efa548d26e5a6e1L, 0xc47bc5014a1a6dafL, + 0xc6b8e9b0709f109aL, 0x359ab6419ca1091bL, + 0xf867241c8cc6d4c0L, 0xc30163d203c94b62L, + 0x9b407691d7fc44f8L, 0x79e0de63425dcf1dL, + 0xc21094364dfb5636L, 0x985915fc12f542e4L, + 0xf294b943e17a2bc4L, 0x3e6f5b7b17b2939dL, + 0x979cf3ca6cec5b5aL, 0xa705992ceecf9c42L, + 0xbd8430bd08277231L, 0x50c6ff782a838353L, + 0xece53cec4a314ebdL, 0xa4f8bf5635246428L, + 0x940f4613ae5ed136L, 0x871b7795e136be99L, + 0xb913179899f68584L, 0x28e2557b59846e3fL, + 0xe757dd7ec07426e5L, 0x331aeada2fe589cfL, + 0x9096ea6f3848984fL, 0x3ff0d2c85def7621L, + 0xb4bca50b065abe63L, 0xfed077a756b53a9L, + 0xe1ebce4dc7f16dfbL, 0xd3e8495912c62894L, + 0x8d3360f09cf6e4bdL, 0x64712dd7abbbd95cL, + 0xb080392cc4349decL, 0xbd8d794d96aacfb3L, + 0xdca04777f541c567L, 0xecf0d7a0fc5583a0L, + 0x89e42caaf9491b60L, 0xf41686c49db57244L, + 0xac5d37d5b79b6239L, 0x311c2875c522ced5L, + 0xd77485cb25823ac7L, 0x7d633293366b828bL, + 0x86a8d39ef77164bcL, 0xae5dff9c02033197L, + 0xa8530886b54dbdebL, 0xd9f57f830283fdfcL, + 0xd267caa862a12d66L, 0xd072df63c324fd7bL, + 0x8380dea93da4bc60L, 0x4247cb9e59f71e6dL, + 0xa46116538d0deb78L, 0x52d9be85f074e608L, + 0xcd795be870516656L, 0x67902e276c921f8bL, + 0x806bd9714632dff6L, 0xba1cd8a3db53b6L, + 0xa086cfcd97bf97f3L, 0x80e8a40eccd228a4L, + 0xc8a883c0fdaf7df0L, 0x6122cd128006b2cdL, + 0xfad2a4b13d1b5d6cL, 0x796b805720085f81L, + 0x9cc3a6eec6311a63L, 0xcbe3303674053bb0L, + 0xc3f490aa77bd60fcL, 0xbedbfc4411068a9cL, + 0xf4f1b4d515acb93bL, 0xee92fb5515482d44L, + 0x991711052d8bf3c5L, 0x751bdd152d4d1c4aL, + 0xbf5cd54678eef0b6L, 0xd262d45a78a0635dL, + 0xef340a98172aace4L, 0x86fb897116c87c34L, + 0x9580869f0e7aac0eL, 0xd45d35e6ae3d4da0L, + 0xbae0a846d2195712L, 0x8974836059cca109L, + 0xe998d258869facd7L, 0x2bd1a438703fc94bL, + 0x91ff83775423cc06L, 0x7b6306a34627ddcfL, + 0xb67f6455292cbf08L, 0x1a3bc84c17b1d542L, + 0xe41f3d6a7377eecaL, 0x20caba5f1d9e4a93L, + 0x8e938662882af53eL, 0x547eb47b7282ee9cL, + 0xb23867fb2a35b28dL, 0xe99e619a4f23aa43L, + 0xdec681f9f4c31f31L, 0x6405fa00e2ec94d4L, + 0x8b3c113c38f9f37eL, 0xde83bc408dd3dd04L, + 0xae0b158b4738705eL, 0x9624ab50b148d445L, + 0xd98ddaee19068c76L, 0x3badd624dd9b0957L, + 0x87f8a8d4cfa417c9L, 0xe54ca5d70a80e5d6L, + 0xa9f6d30a038d1dbcL, 0x5e9fcf4ccd211f4cL, + 0xd47487cc8470652bL, 0x7647c3200069671fL, + 0x84c8d4dfd2c63f3bL, 0x29ecd9f40041e073L, + 0xa5fb0a17c777cf09L, 0xf468107100525890L, + 0xcf79cc9db955c2ccL, 0x7182148d4066eeb4L, + 0x81ac1fe293d599bfL, 0xc6f14cd848405530L, + 0xa21727db38cb002fL, 0xb8ada00e5a506a7cL, + 0xca9cf1d206fdc03bL, 0xa6d90811f0e4851cL, + 0xfd442e4688bd304aL, 0x908f4a166d1da663L, + 0x9e4a9cec15763e2eL, 0x9a598e4e043287feL, + 0xc5dd44271ad3cdbaL, 0x40eff1e1853f29fdL, + 0xf7549530e188c128L, 0xd12bee59e68ef47cL, + 0x9a94dd3e8cf578b9L, 0x82bb74f8301958ceL, + 0xc13a148e3032d6e7L, 0xe36a52363c1faf01L, + 0xf18899b1bc3f8ca1L, 0xdc44e6c3cb279ac1L, + 0x96f5600f15a7b7e5L, 0x29ab103a5ef8c0b9L, + 0xbcb2b812db11a5deL, 0x7415d448f6b6f0e7L, + 0xebdf661791d60f56L, 0x111b495b3464ad21L, + 0x936b9fcebb25c995L, 0xcab10dd900beec34L, + 0xb84687c269ef3bfbL, 0x3d5d514f40eea742L, + 0xe65829b3046b0afaL, 0xcb4a5a3112a5112L, + 0x8ff71a0fe2c2e6dcL, 0x47f0e785eaba72abL, + 0xb3f4e093db73a093L, 0x59ed216765690f56L, + 0xe0f218b8d25088b8L, 0x306869c13ec3532cL, + 0x8c974f7383725573L, 0x1e414218c73a13fbL, + 0xafbd2350644eeacfL, 0xe5d1929ef90898faL, + 0xdbac6c247d62a583L, 0xdf45f746b74abf39L, + 0x894bc396ce5da772L, 0x6b8bba8c328eb783L, + 0xab9eb47c81f5114fL, 0x66ea92f3f326564L, + 0xd686619ba27255a2L, 0xc80a537b0efefebdL, + 0x8613fd0145877585L, 0xbd06742ce95f5f36L, + 0xa798fc4196e952e7L, 0x2c48113823b73704L, + 0xd17f3b51fca3a7a0L, 0xf75a15862ca504c5L, + 0x82ef85133de648c4L, 0x9a984d73dbe722fbL, + 0xa3ab66580d5fdaf5L, 0xc13e60d0d2e0ebbaL, + 0xcc963fee10b7d1b3L, 0x318df905079926a8L, + 0xffbbcfe994e5c61fL, 0xfdf17746497f7052L, + 0x9fd561f1fd0f9bd3L, 0xfeb6ea8bedefa633L, + 0xc7caba6e7c5382c8L, 0xfe64a52ee96b8fc0L, + 0xf9bd690a1b68637bL, 0x3dfdce7aa3c673b0L, + 0x9c1661a651213e2dL, 0x6bea10ca65c084eL, + 0xc31bfa0fe5698db8L, 0x486e494fcff30a62L, + 0xf3e2f893dec3f126L, 0x5a89dba3c3efccfaL, + 0x986ddb5c6b3a76b7L, 0xf89629465a75e01cL, + 0xbe89523386091465L, 0xf6bbb397f1135823L, + 0xee2ba6c0678b597fL, 0x746aa07ded582e2cL, + 0x94db483840b717efL, 0xa8c2a44eb4571cdcL, + 0xba121a4650e4ddebL, 0x92f34d62616ce413L, + 0xe896a0d7e51e1566L, 0x77b020baf9c81d17L, + 0x915e2486ef32cd60L, 0xace1474dc1d122eL, + 0xb5b5ada8aaff80b8L, 0xd819992132456baL, + 0xe3231912d5bf60e6L, 0x10e1fff697ed6c69L, + 0x8df5efabc5979c8fL, 0xca8d3ffa1ef463c1L, + 0xb1736b96b6fd83b3L, 0xbd308ff8a6b17cb2L, + 0xddd0467c64bce4a0L, 0xac7cb3f6d05ddbdeL, + 0x8aa22c0dbef60ee4L, 0x6bcdf07a423aa96bL, + 0xad4ab7112eb3929dL, 0x86c16c98d2c953c6L, + 0xd89d64d57a607744L, 0xe871c7bf077ba8b7L, + 0x87625f056c7c4a8bL, 0x11471cd764ad4972L, + 0xa93af6c6c79b5d2dL, 0xd598e40d3dd89bcfL, + 0xd389b47879823479L, 0x4aff1d108d4ec2c3L, + 0x843610cb4bf160cbL, 0xcedf722a585139baL, + 0xa54394fe1eedb8feL, 0xc2974eb4ee658828L, + 0xce947a3da6a9273eL, 0x733d226229feea32L, + 0x811ccc668829b887L, 0x806357d5a3f525fL, + 0xa163ff802a3426a8L, 0xca07c2dcb0cf26f7L, + 0xc9bcff6034c13052L, 0xfc89b393dd02f0b5L, + 0xfc2c3f3841f17c67L, 0xbbac2078d443ace2L, + 0x9d9ba7832936edc0L, 0xd54b944b84aa4c0dL, + 0xc5029163f384a931L, 0xa9e795e65d4df11L, + 0xf64335bcf065d37dL, 0x4d4617b5ff4a16d5L, + 0x99ea0196163fa42eL, 0x504bced1bf8e4e45L, + 0xc06481fb9bcf8d39L, 0xe45ec2862f71e1d6L, + 0xf07da27a82c37088L, 0x5d767327bb4e5a4cL, + 0x964e858c91ba2655L, 0x3a6a07f8d510f86fL, + 0xbbe226efb628afeaL, 0x890489f70a55368bL, + 0xeadab0aba3b2dbe5L, 0x2b45ac74ccea842eL, + 0x92c8ae6b464fc96fL, 0x3b0b8bc90012929dL, + 0xb77ada0617e3bbcbL, 0x9ce6ebb40173744L, + 0xe55990879ddcaabdL, 0xcc420a6a101d0515L, + 0x8f57fa54c2a9eab6L, 0x9fa946824a12232dL, + 0xb32df8e9f3546564L, 0x47939822dc96abf9L, + 0xdff9772470297ebdL, 0x59787e2b93bc56f7L, + 0x8bfbea76c619ef36L, 0x57eb4edb3c55b65aL, + 0xaefae51477a06b03L, 0xede622920b6b23f1L, + 0xdab99e59958885c4L, 0xe95fab368e45ecedL, + 0x88b402f7fd75539bL, 0x11dbcb0218ebb414L, + 0xaae103b5fcd2a881L, 0xd652bdc29f26a119L, + 0xd59944a37c0752a2L, 0x4be76d3346f0495fL, + 0x857fcae62d8493a5L, 0x6f70a4400c562ddbL, + 0xa6dfbd9fb8e5b88eL, 0xcb4ccd500f6bb952L, + 0xd097ad07a71f26b2L, 0x7e2000a41346a7a7L, + 0x825ecc24c873782fL, 0x8ed400668c0c28c8L, + 0xa2f67f2dfa90563bL, 0x728900802f0f32faL, + 0xcbb41ef979346bcaL, 0x4f2b40a03ad2ffb9L, + 0xfea126b7d78186bcL, 0xe2f610c84987bfa8L, + 0x9f24b832e6b0f436L, 0xdd9ca7d2df4d7c9L, + 0xc6ede63fa05d3143L, 0x91503d1c79720dbbL, + 0xf8a95fcf88747d94L, 0x75a44c6397ce912aL, + 0x9b69dbe1b548ce7cL, 0xc986afbe3ee11abaL, + 0xc24452da229b021bL, 0xfbe85badce996168L, + 0xf2d56790ab41c2a2L, 0xfae27299423fb9c3L, + 0x97c560ba6b0919a5L, 0xdccd879fc967d41aL, + 0xbdb6b8e905cb600fL, 0x5400e987bbc1c920L, + 0xed246723473e3813L, 0x290123e9aab23b68L, + 0x9436c0760c86e30bL, 0xf9a0b6720aaf6521L, + 0xb94470938fa89bceL, 0xf808e40e8d5b3e69L, + 0xe7958cb87392c2c2L, 0xb60b1d1230b20e04L, + 0x90bd77f3483bb9b9L, 0xb1c6f22b5e6f48c2L, + 0xb4ecd5f01a4aa828L, 0x1e38aeb6360b1af3L, + 0xe2280b6c20dd5232L, 0x25c6da63c38de1b0L, + 0x8d590723948a535fL, 0x579c487e5a38ad0eL, + 0xb0af48ec79ace837L, 0x2d835a9df0c6d851L, + 0xdcdb1b2798182244L, 0xf8e431456cf88e65L, + 0x8a08f0f8bf0f156bL, 0x1b8e9ecb641b58ffL, + 0xac8b2d36eed2dac5L, 0xe272467e3d222f3fL, + 0xd7adf884aa879177L, 0x5b0ed81dcc6abb0fL, + 0x86ccbb52ea94baeaL, 0x98e947129fc2b4e9L, + 0xa87fea27a539e9a5L, 0x3f2398d747b36224L, + 0xd29fe4b18e88640eL, 0x8eec7f0d19a03aadL, + 0x83a3eeeef9153e89L, 0x1953cf68300424acL, + 0xa48ceaaab75a8e2bL, 0x5fa8c3423c052dd7L, + 0xcdb02555653131b6L, 0x3792f412cb06794dL, + 0x808e17555f3ebf11L, 0xe2bbd88bbee40bd0L, + 0xa0b19d2ab70e6ed6L, 0x5b6aceaeae9d0ec4L, + 0xc8de047564d20a8bL, 0xf245825a5a445275L, + 0xfb158592be068d2eL, 0xeed6e2f0f0d56712L, + 0x9ced737bb6c4183dL, 0x55464dd69685606bL, + 0xc428d05aa4751e4cL, 0xaa97e14c3c26b886L, + 0xf53304714d9265dfL, 0xd53dd99f4b3066a8L, + 0x993fe2c6d07b7fabL, 0xe546a8038efe4029L, + 0xbf8fdb78849a5f96L, 0xde98520472bdd033L, + 0xef73d256a5c0f77cL, 0x963e66858f6d4440L, + 0x95a8637627989aadL, 0xdde7001379a44aa8L, + 0xbb127c53b17ec159L, 0x5560c018580d5d52L, + 0xe9d71b689dde71afL, 0xaab8f01e6e10b4a6L, + 0x9226712162ab070dL, 0xcab3961304ca70e8L, + 0xb6b00d69bb55c8d1L, 0x3d607b97c5fd0d22L, + 0xe45c10c42a2b3b05L, 0x8cb89a7db77c506aL, + 0x8eb98a7a9a5b04e3L, 0x77f3608e92adb242L, + 0xb267ed1940f1c61cL, 0x55f038b237591ed3L, + 0xdf01e85f912e37a3L, 0x6b6c46dec52f6688L, + 0x8b61313bbabce2c6L, 0x2323ac4b3b3da015L, + 0xae397d8aa96c1b77L, 0xabec975e0a0d081aL, + 0xd9c7dced53c72255L, 0x96e7bd358c904a21L, + 0x881cea14545c7575L, 0x7e50d64177da2e54L, + 0xaa242499697392d2L, 0xdde50bd1d5d0b9e9L, + 0xd4ad2dbfc3d07787L, 0x955e4ec64b44e864L, + 0x84ec3c97da624ab4L, 0xbd5af13bef0b113eL, + 0xa6274bbdd0fadd61L, 0xecb1ad8aeacdd58eL, + 0xcfb11ead453994baL, 0x67de18eda5814af2L, + 0x81ceb32c4b43fcf4L, 0x80eacf948770ced7L, + 0xa2425ff75e14fc31L, 0xa1258379a94d028dL, + 0xcad2f7f5359a3b3eL, 0x96ee45813a04330L, + 0xfd87b5f28300ca0dL, 0x8bca9d6e188853fcL, + 0x9e74d1b791e07e48L, 0x775ea264cf55347eL, + 0xc612062576589ddaL, 0x95364afe032a81a0L, + 0xf79687aed3eec551L, 0x3a83ddbd83f52210L, + 0x9abe14cd44753b52L, 0xc4926a9672793580L, + 0xc16d9a0095928a27L, 0x75b7053c0f178400L, + 0xf1c90080baf72cb1L, 0x5324c68b12dd6800L, + 0x971da05074da7beeL, 0xd3f6fc16ebca8000L, + 0xbce5086492111aeaL, 0x88f4bb1ca6bd0000L, + 0xec1e4a7db69561a5L, 0x2b31e9e3d0700000L, + 0x9392ee8e921d5d07L, 0x3aff322e62600000L, + 0xb877aa3236a4b449L, 0x9befeb9fad487c3L, + 0xe69594bec44de15bL, 0x4c2ebe687989a9b4L, + 0x901d7cf73ab0acd9L, 0xf9d37014bf60a11L, + 0xb424dc35095cd80fL, 0x538484c19ef38c95L, + 0xe12e13424bb40e13L, 0x2865a5f206b06fbaL, + 0x8cbccc096f5088cbL, 0xf93f87b7442e45d4L, + 0xafebff0bcb24aafeL, 0xf78f69a51539d749L, + 0xdbe6fecebdedd5beL, 0xb573440e5a884d1cL, + 0x89705f4136b4a597L, 0x31680a88f8953031L, + 0xabcc77118461cefcL, 0xfdc20d2b36ba7c3eL, + 0xd6bf94d5e57a42bcL, 0x3d32907604691b4dL, + 0x8637bd05af6c69b5L, 0xa63f9a49c2c1b110L, + 0xa7c5ac471b478423L, 0xfcf80dc33721d54L, + 0xd1b71758e219652bL, 0xd3c36113404ea4a9L, + 0x83126e978d4fdf3bL, 0x645a1cac083126eaL, + 0xa3d70a3d70a3d70aL, 0x3d70a3d70a3d70a4L, + 0xccccccccccccccccL, 0xcccccccccccccccdL, + 0x8000000000000000L, 0x0L, + 0xa000000000000000L, 0x0L, + 0xc800000000000000L, 0x0L, + 0xfa00000000000000L, 0x0L, + 0x9c40000000000000L, 0x0L, + 0xc350000000000000L, 0x0L, + 0xf424000000000000L, 0x0L, + 0x9896800000000000L, 0x0L, + 0xbebc200000000000L, 0x0L, + 0xee6b280000000000L, 0x0L, + 0x9502f90000000000L, 0x0L, + 0xba43b74000000000L, 0x0L, + 0xe8d4a51000000000L, 0x0L, + 0x9184e72a00000000L, 0x0L, + 0xb5e620f480000000L, 0x0L, + 0xe35fa931a0000000L, 0x0L, + 0x8e1bc9bf04000000L, 0x0L, + 0xb1a2bc2ec5000000L, 0x0L, + 0xde0b6b3a76400000L, 0x0L, + 0x8ac7230489e80000L, 0x0L, + 0xad78ebc5ac620000L, 0x0L, + 0xd8d726b7177a8000L, 0x0L, + 0x878678326eac9000L, 0x0L, + 0xa968163f0a57b400L, 0x0L, + 0xd3c21bcecceda100L, 0x0L, + 0x84595161401484a0L, 0x0L, + 0xa56fa5b99019a5c8L, 0x0L, + 0xcecb8f27f4200f3aL, 0x0L, + 0x813f3978f8940984L, 0x4000000000000000L, + 0xa18f07d736b90be5L, 0x5000000000000000L, + 0xc9f2c9cd04674edeL, 0xa400000000000000L, + 0xfc6f7c4045812296L, 0x4d00000000000000L, + 0x9dc5ada82b70b59dL, 0xf020000000000000L, + 0xc5371912364ce305L, 0x6c28000000000000L, + 0xf684df56c3e01bc6L, 0xc732000000000000L, + 0x9a130b963a6c115cL, 0x3c7f400000000000L, + 0xc097ce7bc90715b3L, 0x4b9f100000000000L, + 0xf0bdc21abb48db20L, 0x1e86d40000000000L, + 0x96769950b50d88f4L, 0x1314448000000000L, + 0xbc143fa4e250eb31L, 0x17d955a000000000L, + 0xeb194f8e1ae525fdL, 0x5dcfab0800000000L, + 0x92efd1b8d0cf37beL, 0x5aa1cae500000000L, + 0xb7abc627050305adL, 0xf14a3d9e40000000L, + 0xe596b7b0c643c719L, 0x6d9ccd05d0000000L, + 0x8f7e32ce7bea5c6fL, 0xe4820023a2000000L, + 0xb35dbf821ae4f38bL, 0xdda2802c8a800000L, + 0xe0352f62a19e306eL, 0xd50b2037ad200000L, + 0x8c213d9da502de45L, 0x4526f422cc340000L, + 0xaf298d050e4395d6L, 0x9670b12b7f410000L, + 0xdaf3f04651d47b4cL, 0x3c0cdd765f114000L, + 0x88d8762bf324cd0fL, 0xa5880a69fb6ac800L, + 0xab0e93b6efee0053L, 0x8eea0d047a457a00L, + 0xd5d238a4abe98068L, 0x72a4904598d6d880L, + 0x85a36366eb71f041L, 0x47a6da2b7f864750L, + 0xa70c3c40a64e6c51L, 0x999090b65f67d924L, + 0xd0cf4b50cfe20765L, 0xfff4b4e3f741cf6dL, + 0x82818f1281ed449fL, 0xbff8f10e7a8921a4L, + 0xa321f2d7226895c7L, 0xaff72d52192b6a0dL, + 0xcbea6f8ceb02bb39L, 0x9bf4f8a69f764490L, + 0xfee50b7025c36a08L, 0x2f236d04753d5b4L, + 0x9f4f2726179a2245L, 0x1d762422c946590L, + 0xc722f0ef9d80aad6L, 0x424d3ad2b7b97ef5L, + 0xf8ebad2b84e0d58bL, 0xd2e0898765a7deb2L, + 0x9b934c3b330c8577L, 0x63cc55f49f88eb2fL, + 0xc2781f49ffcfa6d5L, 0x3cbf6b71c76b25fbL, + 0xf316271c7fc3908aL, 0x8bef464e3945ef7aL, + 0x97edd871cfda3a56L, 0x97758bf0e3cbb5acL, + 0xbde94e8e43d0c8ecL, 0x3d52eeed1cbea317L, + 0xed63a231d4c4fb27L, 0x4ca7aaa863ee4bddL, + 0x945e455f24fb1cf8L, 0x8fe8caa93e74ef6aL, + 0xb975d6b6ee39e436L, 0xb3e2fd538e122b44L, + 0xe7d34c64a9c85d44L, 0x60dbbca87196b616L, + 0x90e40fbeea1d3a4aL, 0xbc8955e946fe31cdL, + 0xb51d13aea4a488ddL, 0x6babab6398bdbe41L, + 0xe264589a4dcdab14L, 0xc696963c7eed2dd1L, + 0x8d7eb76070a08aecL, 0xfc1e1de5cf543ca2L, + 0xb0de65388cc8ada8L, 0x3b25a55f43294bcbL, + 0xdd15fe86affad912L, 0x49ef0eb713f39ebeL, + 0x8a2dbf142dfcc7abL, 0x6e3569326c784337L, + 0xacb92ed9397bf996L, 0x49c2c37f07965404L, + 0xd7e77a8f87daf7fbL, 0xdc33745ec97be906L, + 0x86f0ac99b4e8dafdL, 0x69a028bb3ded71a3L, + 0xa8acd7c0222311bcL, 0xc40832ea0d68ce0cL, + 0xd2d80db02aabd62bL, 0xf50a3fa490c30190L, + 0x83c7088e1aab65dbL, 0x792667c6da79e0faL, + 0xa4b8cab1a1563f52L, 0x577001b891185938L, + 0xcde6fd5e09abcf26L, 0xed4c0226b55e6f86L, + 0x80b05e5ac60b6178L, 0x544f8158315b05b4L, + 0xa0dc75f1778e39d6L, 0x696361ae3db1c721L, + 0xc913936dd571c84cL, 0x3bc3a19cd1e38e9L, + 0xfb5878494ace3a5fL, 0x4ab48a04065c723L, + 0x9d174b2dcec0e47bL, 0x62eb0d64283f9c76L, + 0xc45d1df942711d9aL, 0x3ba5d0bd324f8394L, + 0xf5746577930d6500L, 0xca8f44ec7ee36479L, + 0x9968bf6abbe85f20L, 0x7e998b13cf4e1ecbL, + 0xbfc2ef456ae276e8L, 0x9e3fedd8c321a67eL, + 0xefb3ab16c59b14a2L, 0xc5cfe94ef3ea101eL, + 0x95d04aee3b80ece5L, 0xbba1f1d158724a12L, + 0xbb445da9ca61281fL, 0x2a8a6e45ae8edc97L, + 0xea1575143cf97226L, 0xf52d09d71a3293bdL, + 0x924d692ca61be758L, 0x593c2626705f9c56L, + 0xb6e0c377cfa2e12eL, 0x6f8b2fb00c77836cL, + 0xe498f455c38b997aL, 0xb6dfb9c0f956447L, + 0x8edf98b59a373fecL, 0x4724bd4189bd5eacL, + 0xb2977ee300c50fe7L, 0x58edec91ec2cb657L, + 0xdf3d5e9bc0f653e1L, 0x2f2967b66737e3edL, + 0x8b865b215899f46cL, 0xbd79e0d20082ee74L, + 0xae67f1e9aec07187L, 0xecd8590680a3aa11L, + 0xda01ee641a708de9L, 0xe80e6f4820cc9495L, + 0x884134fe908658b2L, 0x3109058d147fdcddL, + 0xaa51823e34a7eedeL, 0xbd4b46f0599fd415L, + 0xd4e5e2cdc1d1ea96L, 0x6c9e18ac7007c91aL, + 0x850fadc09923329eL, 0x3e2cf6bc604ddb0L, + 0xa6539930bf6bff45L, 0x84db8346b786151cL, + 0xcfe87f7cef46ff16L, 0xe612641865679a63L, + 0x81f14fae158c5f6eL, 0x4fcb7e8f3f60c07eL, + 0xa26da3999aef7749L, 0xe3be5e330f38f09dL, + 0xcb090c8001ab551cL, 0x5cadf5bfd3072cc5L, + 0xfdcb4fa002162a63L, 0x73d9732fc7c8f7f6L, + 0x9e9f11c4014dda7eL, 0x2867e7fddcdd9afaL, + 0xc646d63501a1511dL, 0xb281e1fd541501b8L, + 0xf7d88bc24209a565L, 0x1f225a7ca91a4226L, + 0x9ae757596946075fL, 0x3375788de9b06958L, + 0xc1a12d2fc3978937L, 0x52d6b1641c83aeL, + 0xf209787bb47d6b84L, 0xc0678c5dbd23a49aL, + 0x9745eb4d50ce6332L, 0xf840b7ba963646e0L, + 0xbd176620a501fbffL, 0xb650e5a93bc3d898L, + 0xec5d3fa8ce427affL, 0xa3e51f138ab4cebeL, + 0x93ba47c980e98cdfL, 0xc66f336c36b10137L, + 0xb8a8d9bbe123f017L, 0xb80b0047445d4184L, + 0xe6d3102ad96cec1dL, 0xa60dc059157491e5L, + 0x9043ea1ac7e41392L, 0x87c89837ad68db2fL, + 0xb454e4a179dd1877L, 0x29babe4598c311fbL, + 0xe16a1dc9d8545e94L, 0xf4296dd6fef3d67aL, + 0x8ce2529e2734bb1dL, 0x1899e4a65f58660cL, + 0xb01ae745b101e9e4L, 0x5ec05dcff72e7f8fL, + 0xdc21a1171d42645dL, 0x76707543f4fa1f73L, + 0x899504ae72497ebaL, 0x6a06494a791c53a8L, + 0xabfa45da0edbde69L, 0x487db9d17636892L, + 0xd6f8d7509292d603L, 0x45a9d2845d3c42b6L, + 0x865b86925b9bc5c2L, 0xb8a2392ba45a9b2L, + 0xa7f26836f282b732L, 0x8e6cac7768d7141eL, + 0xd1ef0244af2364ffL, 0x3207d795430cd926L, + 0x8335616aed761f1fL, 0x7f44e6bd49e807b8L, + 0xa402b9c5a8d3a6e7L, 0x5f16206c9c6209a6L, + 0xcd036837130890a1L, 0x36dba887c37a8c0fL, + 0x802221226be55a64L, 0xc2494954da2c9789L, + 0xa02aa96b06deb0fdL, 0xf2db9baa10b7bd6cL, + 0xc83553c5c8965d3dL, 0x6f92829494e5acc7L, + 0xfa42a8b73abbf48cL, 0xcb772339ba1f17f9L, + 0x9c69a97284b578d7L, 0xff2a760414536efbL, + 0xc38413cf25e2d70dL, 0xfef5138519684abaL, + 0xf46518c2ef5b8cd1L, 0x7eb258665fc25d69L, + 0x98bf2f79d5993802L, 0xef2f773ffbd97a61L, + 0xbeeefb584aff8603L, 0xaafb550ffacfd8faL, + 0xeeaaba2e5dbf6784L, 0x95ba2a53f983cf38L, + 0x952ab45cfa97a0b2L, 0xdd945a747bf26183L, + 0xba756174393d88dfL, 0x94f971119aeef9e4L, + 0xe912b9d1478ceb17L, 0x7a37cd5601aab85dL, + 0x91abb422ccb812eeL, 0xac62e055c10ab33aL, + 0xb616a12b7fe617aaL, 0x577b986b314d6009L, + 0xe39c49765fdf9d94L, 0xed5a7e85fda0b80bL, + 0x8e41ade9fbebc27dL, 0x14588f13be847307L, + 0xb1d219647ae6b31cL, 0x596eb2d8ae258fc8L, + 0xde469fbd99a05fe3L, 0x6fca5f8ed9aef3bbL, + 0x8aec23d680043beeL, 0x25de7bb9480d5854L, + 0xada72ccc20054ae9L, 0xaf561aa79a10ae6aL, + 0xd910f7ff28069da4L, 0x1b2ba1518094da04L, + 0x87aa9aff79042286L, 0x90fb44d2f05d0842L, + 0xa99541bf57452b28L, 0x353a1607ac744a53L, + 0xd3fa922f2d1675f2L, 0x42889b8997915ce8L, + 0x847c9b5d7c2e09b7L, 0x69956135febada11L, + 0xa59bc234db398c25L, 0x43fab9837e699095L, + 0xcf02b2c21207ef2eL, 0x94f967e45e03f4bbL, + 0x8161afb94b44f57dL, 0x1d1be0eebac278f5L, + 0xa1ba1ba79e1632dcL, 0x6462d92a69731732L, + 0xca28a291859bbf93L, 0x7d7b8f7503cfdcfeL, + 0xfcb2cb35e702af78L, 0x5cda735244c3d43eL, + 0x9defbf01b061adabL, 0x3a0888136afa64a7L, + 0xc56baec21c7a1916L, 0x88aaa1845b8fdd0L, + 0xf6c69a72a3989f5bL, 0x8aad549e57273d45L, + 0x9a3c2087a63f6399L, 0x36ac54e2f678864bL, + 0xc0cb28a98fcf3c7fL, 0x84576a1bb416a7ddL, + 0xf0fdf2d3f3c30b9fL, 0x656d44a2a11c51d5L, + 0x969eb7c47859e743L, 0x9f644ae5a4b1b325L, + 0xbc4665b596706114L, 0x873d5d9f0dde1feeL, + 0xeb57ff22fc0c7959L, 0xa90cb506d155a7eaL, + 0x9316ff75dd87cbd8L, 0x9a7f12442d588f2L, + 0xb7dcbf5354e9beceL, 0xc11ed6d538aeb2fL, + 0xe5d3ef282a242e81L, 0x8f1668c8a86da5faL, + 0x8fa475791a569d10L, 0xf96e017d694487bcL, + 0xb38d92d760ec4455L, 0x37c981dcc395a9acL, + 0xe070f78d3927556aL, 0x85bbe253f47b1417L, + 0x8c469ab843b89562L, 0x93956d7478ccec8eL, + 0xaf58416654a6babbL, 0x387ac8d1970027b2L, + 0xdb2e51bfe9d0696aL, 0x6997b05fcc0319eL, + 0x88fcf317f22241e2L, 0x441fece3bdf81f03L, + 0xab3c2fddeeaad25aL, 0xd527e81cad7626c3L, + 0xd60b3bd56a5586f1L, 0x8a71e223d8d3b074L, + 0x85c7056562757456L, 0xf6872d5667844e49L, + 0xa738c6bebb12d16cL, 0xb428f8ac016561dbL, + 0xd106f86e69d785c7L, 0xe13336d701beba52L, + 0x82a45b450226b39cL, 0xecc0024661173473L, + 0xa34d721642b06084L, 0x27f002d7f95d0190L, + 0xcc20ce9bd35c78a5L, 0x31ec038df7b441f4L, + 0xff290242c83396ceL, 0x7e67047175a15271L, + 0x9f79a169bd203e41L, 0xf0062c6e984d386L, + 0xc75809c42c684dd1L, 0x52c07b78a3e60868L, + 0xf92e0c3537826145L, 0xa7709a56ccdf8a82L, + 0x9bbcc7a142b17ccbL, 0x88a66076400bb691L, + 0xc2abf989935ddbfeL, 0x6acff893d00ea435L, + 0xf356f7ebf83552feL, 0x583f6b8c4124d43L, + 0x98165af37b2153deL, 0xc3727a337a8b704aL, + 0xbe1bf1b059e9a8d6L, 0x744f18c0592e4c5cL, + 0xeda2ee1c7064130cL, 0x1162def06f79df73L, + 0x9485d4d1c63e8be7L, 0x8addcb5645ac2ba8L, + 0xb9a74a0637ce2ee1L, 0x6d953e2bd7173692L, + 0xe8111c87c5c1ba99L, 0xc8fa8db6ccdd0437L, + 0x910ab1d4db9914a0L, 0x1d9c9892400a22a2L, + 0xb54d5e4a127f59c8L, 0x2503beb6d00cab4bL, + 0xe2a0b5dc971f303aL, 0x2e44ae64840fd61dL, + 0x8da471a9de737e24L, 0x5ceaecfed289e5d2L, + 0xb10d8e1456105dadL, 0x7425a83e872c5f47L, + 0xdd50f1996b947518L, 0xd12f124e28f77719L, + 0x8a5296ffe33cc92fL, 0x82bd6b70d99aaa6fL, + 0xace73cbfdc0bfb7bL, 0x636cc64d1001550bL, + 0xd8210befd30efa5aL, 0x3c47f7e05401aa4eL, + 0x8714a775e3e95c78L, 0x65acfaec34810a71L, + 0xa8d9d1535ce3b396L, 0x7f1839a741a14d0dL, + 0xd31045a8341ca07cL, 0x1ede48111209a050L, + 0x83ea2b892091e44dL, 0x934aed0aab460432L, + 0xa4e4b66b68b65d60L, 0xf81da84d5617853fL, + 0xce1de40642e3f4b9L, 0x36251260ab9d668eL, + 0x80d2ae83e9ce78f3L, 0xc1d72b7c6b426019L, + 0xa1075a24e4421730L, 0xb24cf65b8612f81fL, + 0xc94930ae1d529cfcL, 0xdee033f26797b627L, + 0xfb9b7cd9a4a7443cL, 0x169840ef017da3b1L, + 0x9d412e0806e88aa5L, 0x8e1f289560ee864eL, + 0xc491798a08a2ad4eL, 0xf1a6f2bab92a27e2L, + 0xf5b5d7ec8acb58a2L, 0xae10af696774b1dbL, + 0x9991a6f3d6bf1765L, 0xacca6da1e0a8ef29L, + 0xbff610b0cc6edd3fL, 0x17fd090a58d32af3L, + 0xeff394dcff8a948eL, 0xddfc4b4cef07f5b0L, + 0x95f83d0a1fb69cd9L, 0x4abdaf101564f98eL, + 0xbb764c4ca7a4440fL, 0x9d6d1ad41abe37f1L, + 0xea53df5fd18d5513L, 0x84c86189216dc5edL, + 0x92746b9be2f8552cL, 0x32fd3cf5b4e49bb4L, + 0xb7118682dbb66a77L, 0x3fbc8c33221dc2a1L, + 0xe4d5e82392a40515L, 0xfabaf3feaa5334aL, + 0x8f05b1163ba6832dL, 0x29cb4d87f2a7400eL, + 0xb2c71d5bca9023f8L, 0x743e20e9ef511012L, + 0xdf78e4b2bd342cf6L, 0x914da9246b255416L, + 0x8bab8eefb6409c1aL, 0x1ad089b6c2f7548eL, + 0xae9672aba3d0c320L, 0xa184ac2473b529b1L, + 0xda3c0f568cc4f3e8L, 0xc9e5d72d90a2741eL, + 0x8865899617fb1871L, 0x7e2fa67c7a658892L, + 0xaa7eebfb9df9de8dL, 0xddbb901b98feeab7L, + 0xd51ea6fa85785631L, 0x552a74227f3ea565L, + 0x8533285c936b35deL, 0xd53a88958f87275fL, + 0xa67ff273b8460356L, 0x8a892abaf368f137L, + 0xd01fef10a657842cL, 0x2d2b7569b0432d85L, + 0x8213f56a67f6b29bL, 0x9c3b29620e29fc73L, + 0xa298f2c501f45f42L, 0x8349f3ba91b47b8fL, + 0xcb3f2f7642717713L, 0x241c70a936219a73L, + 0xfe0efb53d30dd4d7L, 0xed238cd383aa0110L, + 0x9ec95d1463e8a506L, 0xf4363804324a40aaL, + 0xc67bb4597ce2ce48L, 0xb143c6053edcd0d5L, + 0xf81aa16fdc1b81daL, 0xdd94b7868e94050aL, + 0x9b10a4e5e9913128L, 0xca7cf2b4191c8326L, + 0xc1d4ce1f63f57d72L, 0xfd1c2f611f63a3f0L, + 0xf24a01a73cf2dccfL, 0xbc633b39673c8cecL, + 0x976e41088617ca01L, 0xd5be0503e085d813L, + 0xbd49d14aa79dbc82L, 0x4b2d8644d8a74e18L, + 0xec9c459d51852ba2L, 0xddf8e7d60ed1219eL, + 0x93e1ab8252f33b45L, 0xcabb90e5c942b503L, + 0xb8da1662e7b00a17L, 0x3d6a751f3b936243L, + 0xe7109bfba19c0c9dL, 0xcc512670a783ad4L, + 0x906a617d450187e2L, 0x27fb2b80668b24c5L, + 0xb484f9dc9641e9daL, 0xb1f9f660802dedf6L, + 0xe1a63853bbd26451L, 0x5e7873f8a0396973L, + 0x8d07e33455637eb2L, 0xdb0b487b6423e1e8L, + 0xb049dc016abc5e5fL, 0x91ce1a9a3d2cda62L, + 0xdc5c5301c56b75f7L, 0x7641a140cc7810fbL, + 0x89b9b3e11b6329baL, 0xa9e904c87fcb0a9dL, + 0xac2820d9623bf429L, 0x546345fa9fbdcd44L, + 0xd732290fbacaf133L, 0xa97c177947ad4095L, + 0x867f59a9d4bed6c0L, 0x49ed8eabcccc485dL, + 0xa81f301449ee8c70L, 0x5c68f256bfff5a74L, + 0xd226fc195c6a2f8cL, 0x73832eec6fff3111L, + 0x83585d8fd9c25db7L, 0xc831fd53c5ff7eabL, + 0xa42e74f3d032f525L, 0xba3e7ca8b77f5e55L, + 0xcd3a1230c43fb26fL, 0x28ce1bd2e55f35ebL, + 0x80444b5e7aa7cf85L, 0x7980d163cf5b81b3L, + 0xa0555e361951c366L, 0xd7e105bcc332621fL, + 0xc86ab5c39fa63440L, 0x8dd9472bf3fefaa7L, + 0xfa856334878fc150L, 0xb14f98f6f0feb951L, + 0x9c935e00d4b9d8d2L, 0x6ed1bf9a569f33d3L, + 0xc3b8358109e84f07L, 0xa862f80ec4700c8L, + 0xf4a642e14c6262c8L, 0xcd27bb612758c0faL, + 0x98e7e9cccfbd7dbdL, 0x8038d51cb897789cL, + 0xbf21e44003acdd2cL, 0xe0470a63e6bd56c3L, + 0xeeea5d5004981478L, 0x1858ccfce06cac74L, + 0x95527a5202df0ccbL, 0xf37801e0c43ebc8L, + 0xbaa718e68396cffdL, 0xd30560258f54e6baL, + 0xe950df20247c83fdL, 0x47c6b82ef32a2069L, + 0x91d28b7416cdd27eL, 0x4cdc331d57fa5441L, + 0xb6472e511c81471dL, 0xe0133fe4adf8e952L, + 0xe3d8f9e563a198e5L, 0x58180fddd97723a6L, + 0x8e679c2f5e44ff8fL, 0x570f09eaa7ea7648L + }; +} diff --git a/src/main/java/org/simdjson/Tape.java b/src/main/java/org/simdjson/Tape.java index 2d7af22..1fc7663 100644 --- a/src/main/java/org/simdjson/Tape.java +++ b/src/main/java/org/simdjson/Tape.java @@ -18,14 +18,11 @@ class Tape { private static final int JSON_COUNT_MASK = 0xFFFFFF; private final long[] tape; - private final double[] doubleTape; private int tapeIdx; - private int doubleTapeIdx; Tape(int capacity) { tape = new long[capacity]; - doubleTape = new double[capacity]; } void append(long val, char type) { @@ -40,9 +37,9 @@ void appendInt64(long val) { } void appendDouble(double val) { - append(doubleTapeIdx, DOUBLE); - doubleTape[doubleTapeIdx] = val; - doubleTapeIdx++; + append(0, DOUBLE); + tape[tapeIdx] = Double.doubleToRawLongBits(val); + tapeIdx++; } void write(int idx, long val, char type) { @@ -55,7 +52,6 @@ void skip() { void reset() { tapeIdx = 0; - doubleTapeIdx = 0; } int getCurrentIdx() { @@ -75,8 +71,8 @@ long getInt64Value(int idx) { } double getDouble(int idx) { - int doubleTapeIdx = (int) getValue(idx); - return doubleTape[doubleTapeIdx]; + long bits = getInt64Value(idx); + return Double.longBitsToDouble(bits); } int getMatchingBraceIndex(int idx) { @@ -92,7 +88,7 @@ int computeNextIndex(int idx) { case START_ARRAY, START_OBJECT -> { return getMatchingBraceIndex(idx); } - case INT64 -> { + case INT64, DOUBLE -> { return idx + 2; } default -> { diff --git a/src/main/java/org/simdjson/TapeBuilder.java b/src/main/java/org/simdjson/TapeBuilder.java index e835f5e..934a3b6 100644 --- a/src/main/java/org/simdjson/TapeBuilder.java +++ b/src/main/java/org/simdjson/TapeBuilder.java @@ -87,7 +87,7 @@ void visitRootPrimitive(byte[] buffer, int idx, int len) { case 'f' -> visitRootFalseAtom(buffer, idx); case 'n' -> visitRootNullAtom(buffer, idx); case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' -> visitRootNumber(buffer, idx, len); - default -> throw new JsonParsingException("Document starts with a non-value character"); + default -> throw new JsonParsingException("Unrecognized primitive. Expected: string, number, 'true', 'false' or 'null'."); } } @@ -98,7 +98,7 @@ void visitPrimitive(byte[] buffer, int idx) { case 'f' -> visitFalseAtom(buffer, idx); case 'n' -> visitNullAtom(buffer, idx); case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' -> visitNumber(buffer, idx); - default -> throw new JsonParsingException("Non-value found when value was expected!"); + default -> throw new JsonParsingException("Unrecognized primitive. Expected: string, number, 'true', 'false' or 'null'."); } } @@ -247,7 +247,7 @@ private boolean hasBackslash(long backslashBits, long quoteBits) { } private void visitNumber(byte[] buffer, int idx) { - numberParser.parseNumber(buffer, idx, idx); + numberParser.parseNumber(buffer, idx); } private void visitRootNumber(byte[] buffer, int idx, int len) { @@ -255,7 +255,7 @@ private void visitRootNumber(byte[] buffer, int idx, int len) { byte[] copy = new byte[remainingLen + padding]; System.arraycopy(buffer, idx, copy, 0, remainingLen); Arrays.fill(copy, remainingLen, remainingLen + padding, SPACE); - numberParser.parseNumber(copy, 0, idx); + numberParser.parseNumber(copy, 0); } private void startContainer(int depth) { @@ -282,7 +282,7 @@ void reset() { stringBufferIdx = 0; } - public JsonValue createJsonValue(byte[] buffer) { + JsonValue createJsonValue(byte[] buffer) { return new JsonValue(tape, 1, stringBuffer, buffer); } diff --git a/src/test/java/org/simdjson/BenchmarkCorrectnessTest.java b/src/test/java/org/simdjson/BenchmarkCorrectnessTest.java index a5fbfc1..4957381 100644 --- a/src/test/java/org/simdjson/BenchmarkCorrectnessTest.java +++ b/src/test/java/org/simdjson/BenchmarkCorrectnessTest.java @@ -1,6 +1,8 @@ package org.simdjson; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.io.IOException; import java.io.InputStream; @@ -9,6 +11,8 @@ import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; +import static org.simdjson.StringUtils.padWithSpaces; +import static org.simdjson.StringUtils.toUtf8; public class BenchmarkCorrectnessTest { @@ -34,6 +38,24 @@ public void countUniqueTwitterUsersWithDefaultProfile() throws IOException { assertThat(defaultUsers.size()).isEqualTo(86); } + @ParameterizedTest + @CsvSource({ + "2.2250738585072013e-308, 0x1p-1022", // fast path + "1.00000000000000188558920870223463870174566020691753515394643550663070558368373221972569761144603605635692374830246134201063722058e-309, 1e-309" // slow path + }) + public void numberParserTest(String input, Double expected) { + // given + Tape tape = new Tape(100); + NumberParser numberParser = new NumberParser(tape); + byte[] numberUtf8Bytes = toUtf8(padWithSpaces(input)); + + // when + numberParser.parseNumber(numberUtf8Bytes, 0); + + // then + assertThat(tape.getDouble(0)).isEqualTo(expected); + } + private static byte[] loadTwitterJson() throws IOException { try (InputStream is = BenchmarkCorrectnessTest.class.getResourceAsStream("/twitter.json")) { return is.readAllBytes(); diff --git a/src/test/java/org/simdjson/NumberParsingTest.java b/src/test/java/org/simdjson/NumberParsingTest.java new file mode 100644 index 0000000..f518b48 --- /dev/null +++ b/src/test/java/org/simdjson/NumberParsingTest.java @@ -0,0 +1,599 @@ +package org.simdjson; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.simdjson.StringUtils.toUtf8; + +public class NumberParsingTest { + + @ParameterizedTest + @ValueSource(strings = { + "123.", + "1..1", + "1.e1", + "1.E1" + }) + public void decimalPointHasToBeFollowedByAtLeastOneDigit(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Invalid number. Decimal point has to be followed by a digit."); + } + + @ParameterizedTest + @ValueSource(strings = { + "1e+-2", + "1E+-2", + "1e--23", + "1E--23", + "1ea", + "1Ea", + "1e", + "1E", + "1e+", + "1E+" + }) + public void exponentIndicatorHasToBeFollowedByAtLeastOneDigit(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Invalid number. Exponent indicator has to be followed by a digit."); + } + + @ParameterizedTest + @ValueSource(strings = { + "012", + "-012", + "012e34", + "01.2", + "000", + "-000" + }) + public void leadingZerosAreNotAllowed(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Invalid number. Leading zeroes are not allowed."); + } + + @ParameterizedTest + @ValueSource(strings = { + "-a123", + "--123", + "-+123", + "-.123", + "-e123", + "[-]", + "{\"a\":-}" + }) + public void minusHasToBeFollowedByAtLeastOneDigit(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Invalid number. Minus has to be followed by a digit."); + } + + @ParameterizedTest + @ValueSource(strings = { + "-1-2", + "1.0.1", + "12E12.12", + "1e2e3", + "1a" + }) + public void numberHasToBeFollowedByStructuralCharacterOrWhitespace(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Number has to be followed by a structural character or whitespace."); + } + + @Test + public void minusZeroIsTreatedAsIntegerZero() { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8("-0"); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertLong(value, 0); + } + + @Test + public void startingWithPlusIsNotAllowed() { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8("+1"); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Unrecognized primitive. Expected: string, number, 'true', 'false' or 'null'."); + } + + @ParameterizedTest + @ValueSource(strings = { + "a123", + "{\"num\": a123}", + "a-123", + "{\"num\": a-123}" + }) + public void numberHasToStartWithMinusOrDigit(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Unrecognized primitive. Expected: string, number, 'true', 'false' or 'null'."); + } + + @ParameterizedTest + @ValueSource(longs = { + Long.MAX_VALUE, + Long.MIN_VALUE + }) + public void minMaxLongValue(long input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(String.valueOf(input)); + + // when + JsonValue jsonValue = parser.parse(json, json.length); + + // then + assertLong(jsonValue, input); + } + + @ParameterizedTest + @ValueSource(strings = { + "9223372036854775808", + "9999999999999999999", + "10000000000000000000", + "-9223372036854775809", + "-9999999999999999999", + "-10000000000000000000" + }) + public void outOfRangeLongIsNotAllowed(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); + + // then + assertThat(ex.getMessage()).isEqualTo("Number value is out of long range ([-9223372036854775808, 9223372036854775807])."); + } + + @ParameterizedTest + @CsvSource({ + "1e000000000000000000001, 10.0", + "1e-000000000000000000001, 0.1" + }) + public void exponentWithMoreDigitsThanLongCanAccommodateAndLeadingZeros(String input, double expected) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, expected); + } + + @ParameterizedTest + @CsvSource({ + "0e999999999999999999999, 0.0", + "0e-999999999999999999999, 0.0", + "1e999999999999999999999, Infinity", + "1e-999999999999999999999, 0.0", + "9999999999999999999999999999999999999999e-999999999999999999999, 0.0", + "0.9999999999999999999999999999999999999999e999999999999999999999, Infinity" + }) + public void exponentWithMoreDigitsThanLongCanAccommodate(String input, double expected) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, expected); + } + + @ParameterizedTest + @ValueSource(strings = { + "1.9e308", + "1.8e308", + "1234456789012345678901234567890e9999999999999999999999999999", + "1.832312213213213232132132143451234453123412321321312e308", + "2139879401095466344511101915470454744.9813888656856943E+272", + "2e30000000000000000", + "2e3000", + "1234456789012345678901234567890e999999999999999999999999999", + "1.7976931348623159e308", + "1438456663141390273526118207642235581183227845246331231162636653790368152091394196930365828634687637948157940776599182791387527135353034738357134110310609455693900824193549772792016543182680519740580354365467985440183598701312257624545562331397018329928613196125590274187720073914818062530830316533158098624984118889298281371812288789537310599037529113415438738954894752124724983067241108764488346454376699018673078404751121414804937224240805993123816932326223683090770561597570457793932985826162604255884529134126396282202126526253389383421806727954588525596114379801269094096329805054803089299736996870951258573010877404407451953846698609198213926882692078557033228265259305481198526059813164469187586693257335779522020407645498684263339921905227556616698129967412891282231685504660671277927198290009824680186319750978665734576683784255802269708917361719466043175201158849097881370477111850171579869056016061666173029059588433776015644439705050377554277696143928278093453792803846252715966016733222646442382892123940052441346822429721593884378212558701004356924243030059517489346646577724622498919752597382095222500311124181823512251071356181769376577651390028297796156208815375089159128394945710515861334486267101797497111125909272505194792870889617179758703442608016143343262159998149700606597792535574457560429226974273443630323818747730771316763398572110874959981923732463076884528677392654150010269822239401993427482376513231389212353583573566376915572650916866553612366187378959554983566712767093372906030188976220169058025354973622211666504549316958271880975697143546564469806791358707318873075708383345004090151974068325838177531266954177406661392229801349994695941509935655355652985723782153570084089560139142231.738475042362596875449154552392299548947138162081694168675340677843807613129780449323363759027012972466987370921816813162658754726545121090545507240267000456594786540949605260722461937870630634874991729398208026467698131898691830012167897399682179601734569071423681e-733" + }) + public void positiveInfinity(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, Double.POSITIVE_INFINITY); + } + + @ParameterizedTest + @ValueSource(strings = { + "-1.9e308", + "-1.8e308", + "-1234456789012345678901234567890e9999999999999999999999999999", + "-1.832312213213213232132132143451234453123412321321312e308", + "-2139879401095466344511101915470454744.9813888656856943E+272", + "-2e30000000000000000", + "-2e3000", + "-1234456789012345678901234567890e999999999999999999999999999", + "-1.7976931348623159e308", + "-1438456663141390273526118207642235581183227845246331231162636653790368152091394196930365828634687637948157940776599182791387527135353034738357134110310609455693900824193549772792016543182680519740580354365467985440183598701312257624545562331397018329928613196125590274187720073914818062530830316533158098624984118889298281371812288789537310599037529113415438738954894752124724983067241108764488346454376699018673078404751121414804937224240805993123816932326223683090770561597570457793932985826162604255884529134126396282202126526253389383421806727954588525596114379801269094096329805054803089299736996870951258573010877404407451953846698609198213926882692078557033228265259305481198526059813164469187586693257335779522020407645498684263339921905227556616698129967412891282231685504660671277927198290009824680186319750978665734576683784255802269708917361719466043175201158849097881370477111850171579869056016061666173029059588433776015644439705050377554277696143928278093453792803846252715966016733222646442382892123940052441346822429721593884378212558701004356924243030059517489346646577724622498919752597382095222500311124181823512251071356181769376577651390028297796156208815375089159128394945710515861334486267101797497111125909272505194792870889617179758703442608016143343262159998149700606597792535574457560429226974273443630323818747730771316763398572110874959981923732463076884528677392654150010269822239401993427482376513231389212353583573566376915572650916866553612366187378959554983566712767093372906030188976220169058025354973622211666504549316958271880975697143546564469806791358707318873075708383345004090151974068325838177531266954177406661392229801349994695941509935655355652985723782153570084089560139142231.738475042362596875449154552392299548947138162081694168675340677843807613129780449323363759027012972466987370921816813162658754726545121090545507240267000456594786540949605260722461937870630634874991729398208026467698131898691830012167897399682179601734569071423681e-733" + }) + public void negativeInfinity(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, Double.NEGATIVE_INFINITY); + } + + @ParameterizedTest + @ValueSource(strings = { + "0.0", + "2251799813685248e-342", + "9999999999999999999e-343", + "1.23e-341", + "123e-343", + "0.0e-999", + "0e9999999999999999999999999999", + "18446744073709551615e-343", + "0.099999999999999999999e-323", + "0.99999999999999999999e-324", + "0.9999999999999999999e-324" + }) + public void positiveZero(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, 0.0); + } + + @ParameterizedTest + @ValueSource(strings = { + "-0.0", + "-2251799813685248e-342", + "-9999999999999999999e-343", + "-1.23e-341", + "-123e-343", + "-0.0e-999", + "-0e9999999999999999999999999999", + "-18446744073709551615e-343", + "-0.099999999999999999999e-323", + "-0.99999999999999999999e-324", + "-0.9999999999999999999e-324" + }) + public void negativeZero(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, -0.0); + } + + @ParameterizedTest + @ValueSource(strings = { + // In this case the binary significand after rounding up is equal to 9007199254740992 (2^53), + // which is more than we can store (2^53 - 1). + "7.2057594037927933e16", + "72057594037927933.0000000000000000", + }) + public void roundingOverflow(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, 7.2057594037927933e16); + assertDouble(value, 7.2057594037927936e16); + } + + @ParameterizedTest + @ValueSource(strings = { + "2.2250738585072016e-308", + "2.2250738585072015e-308", + "2.2250738585072014e-308", + "2.2250738585072013e-308", + "2.2250738585072012e-308" + }) + public void minNormalDouble(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, 0x1p-1022); + } + + @ParameterizedTest + @ValueSource(strings = { + "2.2250738585072011e-308", + "2.2250738585072010e-308", + "2.2250738585072009e-308", + "2.2250738585072008e-308", + "2.2250738585072007e-308", + "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072008890245868760858598876504231122409594654935248025624400092282356951787758888037591552642309780950434312085877387158357291821993020294379224223559819827501242041788969571311791082261043971979604000454897391938079198936081525613113376149842043271751033627391549782731594143828136275113838604094249464942286316695429105080201815926642134996606517803095075913058719846423906068637102005108723282784678843631944515866135041223479014792369585208321597621066375401613736583044193603714778355306682834535634005074073040135602968046375918583163124224521599262546494300836851861719422417646455137135420132217031370496583210154654068035397417906022589503023501937519773030945763173210852507299305089761582519159720757232455434770912461317493580281734466552734375", + }) + public void maxSubnormalDouble(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, 0x0.fffffffffffffp-1022); + } + + @ParameterizedTest + @ValueSource(strings = { + "3e-324", + "4.9e-324", + "4.9406564584124654e-324", + "4.94065645841246544176568792868e-324", + }) + public void minSubnormalDouble(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, 0x0.0000000000001p-1022); + } + + @ParameterizedTest + @ValueSource(strings = { + "1.7976931348623157e308", + "1.7976931348623158e308", + }) + public void maxDouble(String input) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, 0x1.fffffffffffffp+1023); + } + + @ParameterizedTest + @CsvSource({ + "2251799813685803.75, 2251799813685804", + "4503599627370497.5, 4503599627370498", + "4503599627475353.5, 4503599627475354", + "9007199254740993.0, 9007199254740992.0", + "4503599627370496.5, 4503599627370496", + "4503599627475352.5, 4503599627475352", + "2251799813685248.25, 2251799813685248", + "2.22507385850720212418870147920222032907240528279439037814303133837435107319244194686754406432563881851382188218502438069999947733013005649884107791928741341929297200970481951993067993290969042784064731682041565926728632933630474670123316852983422152744517260835859654566319282835244787787799894310779783833699159288594555213714181128458251145584319223079897504395086859412457230891738946169368372321191373658977977723286698840356390251044443035457396733706583981055420456693824658413747607155981176573877626747665912387199931904006317334709003012790188175203447190250028061277777916798391090578584006464715943810511489154282775041174682194133952466682503431306181587829379004205392375072083366693241580002758391118854188641513168478436313080237596295773983001708984375e-308, 2.2250738585072024e-308", + "1125899906842624.125, 1125899906842624", + "1125899906842901.875, 1125899906842902", + "9007199254740993.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000, 9007199254740992", + }) + public void roundTiesToEven(String input, double expected) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, expected); + } + + @ParameterizedTest + @CsvSource({ + "12045e23, 12045e23", + "2251799813685803.85, 2251799813685804", + "4503599627370497.6, 4503599627370498", + "45035996.273704995, 45035996.273705", + "4503599627475353.6, 4503599627475354", + "1.797693134862315700000000000000001e308, 1.7976931348623157e308", + "860228122.6654514319E+90, 8.602281226654515e98", + "-42823146028335318693e-128, -4.282314602833532e-109", + "-2402844368454405395.2, -2402844368454405600", + "2402844368454405395.2, 2402844368454405600", + "-2240084132271013504.131248280843119943687942846658579428, -2240084132271013600", + }) + public void roundUpToNearest(String input, double expected) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, expected); + } + + @ParameterizedTest + @CsvSource({ + "2251799813685803.15, 2251799813685803", + "4503599627370497.2, 4503599627370497", + "45035996.273704985, 45035996.27370498", + "4503599627475353.2, 4503599627475353", + "9355950000000000000.00000000000000000000000000000000001844674407370955161600000184467440737095516161844674407370955161407370955161618446744073709551616000184467440737095516166000001844674407370955161618446744073709551614073709551616184467440737095516160001844674407370955161601844674407370955674451616184467440737095516140737095516161844674407370955161600018446744073709551616018446744073709551611616000184467440737095001844674407370955161600184467440737095516160018446744073709551168164467440737095516160001844073709551616018446744073709551616184467440737095516160001844674407536910751601611616000184467440737095001844674407370955161600184467440737095516160018446744073709551616184467440737095516160001844955161618446744073709551616000184467440753691075160018446744073709, 9355950000000000000", + "1.0000000000000006661338147750939242541790008544921875, 1.0000000000000007", + "-92666518056446206563e3, -9.26665180564462e22", + "90054602635948575728e72, 9.005460263594858e91", + "7.0420557077594588669468784357561207962098443483187940792729600000e59, 7.042055707759459e59", + }) + public void roundDownToNearest(String input, double expected) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, expected); + } + + @ParameterizedTest + @CsvSource({ + "9007199254740991.0, 9007199254740991" + }) + public void exactDouble(String input, double expected) { + // given + SimdJsonParser parser = new SimdJsonParser(); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, expected); + } + + @ParameterizedTest + @MethodSource("listTestFiles") + // This test assumes that input files are formatted as described in: https://github.com/nigeltao/parse-number-fxx-test-data + public void testFiles(File file) throws IOException { + // given + SimdJsonParser parser = new SimdJsonParser(); + + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + String line; + while ((line = br.readLine()) != null) { + String[] cells = line.split(" "); + Double expected = Double.longBitsToDouble(Long.decode("0x" + cells[2])); + String input = readInputNumber(cells[3]); + byte[] json = toUtf8(input); + + // when + JsonValue value = parser.parse(json, json.length); + + // then + assertDouble(value, expected); + } + } + } + + private static String readInputNumber(String input) { + boolean isDouble = input.indexOf('e') >= 0 || input.indexOf('E') >= 0 || input.indexOf('.') >= 0; + if (isDouble) { + if (input.startsWith(".")) { + input = "0" + input; + } + return input.replaceFirst("\\.[eE]", ".0e"); + } + return input + ".0"; + } + + private static List listTestFiles() throws IOException { + String testDataDir = System.getProperty("org.simdjson.testdata.dir", System.getProperty("user.dir") + "/testdata"); + File[] testFiles = Path.of(testDataDir, "parse-number-fxx-test-data", "data").toFile().listFiles(); + if (testFiles == null) { + File emptyFile = new File(testDataDir, "empty.txt"); + emptyFile.createNewFile(); + return List.of(emptyFile); + } + return Stream.of(testFiles) + .filter(File::isFile) + .toList(); + } + + private static void assertLong(JsonValue actual, long expected) { + assertThat(actual.isLong()).isTrue(); + assertThat(actual.asLong()).isEqualTo(expected); + } + + private static void assertDouble(JsonValue actual, Double expected) { + assertThat(actual.isDouble()).isTrue(); + assertThat(actual.asDouble()).isEqualTo(expected); + } +} diff --git a/src/test/java/org/simdjson/SimdJsonParserTest.java b/src/test/java/org/simdjson/SimdJsonParserTest.java index 86c3ff9..9d15c5c 100644 --- a/src/test/java/org/simdjson/SimdJsonParserTest.java +++ b/src/test/java/org/simdjson/SimdJsonParserTest.java @@ -1,6 +1,5 @@ package org.simdjson; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -12,6 +11,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.simdjson.StringUtils.toUtf8; public class SimdJsonParserTest { @@ -19,7 +19,7 @@ public class SimdJsonParserTest { public void testEmptyArray() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[]"); + byte[] json = toUtf8("[]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -37,7 +37,7 @@ public void testEmptyArray() { public void testEmptyObject() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("{}"); + byte[] json = toUtf8("{}"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -55,7 +55,7 @@ public void testEmptyObject() { public void testArrayIterator() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[1, 2, 3]"); + byte[] json = toUtf8("[1, 2, 3]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -78,7 +78,7 @@ public void testArrayIterator() { public void testObjectIterator() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("{\"a\": 1, \"b\": 2, \"c\": 3}"); + byte[] json = toUtf8("{\"a\": 1, \"b\": 2, \"c\": 3}"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -103,7 +103,7 @@ public void testObjectIterator() { public void testBooleanValues() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[true, false]"); + byte[] json = toUtf8("[true, false]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -122,7 +122,7 @@ public void testBooleanValues() { public void testBooleanValuesAsRoot(boolean booleanVal) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(Boolean.toString(booleanVal)); + byte[] json = toUtf8(Boolean.toString(booleanVal)); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -135,7 +135,7 @@ public void testBooleanValuesAsRoot(boolean booleanVal) { public void testNullValue() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[null]"); + byte[] json = toUtf8("[null]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -153,7 +153,7 @@ public void testNullValue() { public void testNullValueAsRoot() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("null"); + byte[] json = toUtf8("null"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -166,7 +166,7 @@ public void testNullValueAsRoot() { public void testStringValues() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[\"abc\", \"ab\\\\c\"]"); + byte[] json = toUtf8("[\"abc\", \"ab\\\\c\"]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -185,7 +185,7 @@ public void testStringValues() { public void testStringValuesAsRoot(String jsonStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("\"" + jsonStr + "\""); + byte[] json = toUtf8("\"" + jsonStr + "\""); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -195,10 +195,10 @@ public void testStringValuesAsRoot(String jsonStr) { } @Test - public void testLongValues() { + public void testNumericValues() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[0, 1, -1]"); + byte[] json = toUtf8("[0, 1, -1, 1.1]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -210,58 +210,36 @@ public void testLongValues() { assertLong(it.next(), 0); assertLong(it.next(), 1); assertLong(it.next(), -1); + assertDouble(it.next(), "1.1"); assertThat(it.hasNext()).isFalse(); } @ParameterizedTest - @ValueSource(ints = {0, 1, -1}) - public void testLongValuesAsRoot(int intVal) { + @ValueSource(strings = {"0", "1", "-1"}) + public void testLongValuesAsRoot(String longStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(String.valueOf(intVal)); + byte[] json = toUtf8(longStr); // when JsonValue jsonValue = parser.parse(json, json.length); // then - assertLong(jsonValue, intVal); + assertLong(jsonValue, Long.parseLong(longStr)); } @ParameterizedTest - @ValueSource(longs = {Long.MAX_VALUE, Long.MIN_VALUE}) - public void testMinMaxLongValue(long longVal) { + @ValueSource(strings = {"1.1", "-1.1", "1e1", "1E1", "-1e1", "-1E1", "1e-1", "1E-1", "1.1e1", "1.1E1"}) + public void testDoubleValuesAsRoot(String doubleStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(String.valueOf(longVal)); + byte[] json = toUtf8(doubleStr); // when JsonValue jsonValue = parser.parse(json, json.length); // then - assertLong(jsonValue, longVal); - } - - @Test - public void testOutOfRangeLongValues() { - // given - SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[9223372036854775808, 99223372036854775808, -9223372036854775809, -99223372036854775809]"); - - // when - JsonValue jsonValue = parser.parse(json, json.length); - - // then - assertThat(jsonValue.isArray()).isTrue(); - Iterator it = jsonValue.arrayIterator(); - assertThat(it.hasNext()).isTrue(); - // Not sure if this is how out-of-ranges should be handled. - // The JSON specification doesn't say much about it: https://datatracker.ietf.org/doc/html/rfc8259#section-6. - // Jackson handles them in the same way. - assertLong(it.next(), -9223372036854775808L); - assertLong(it.next(), 6989651668307017728L); - assertLong(it.next(), 9223372036854775807L); - assertLong(it.next(), -6989651668307017729L); - assertThat(it.hasNext()).isFalse(); + assertDouble(jsonValue, doubleStr); } @ParameterizedTest @@ -269,7 +247,7 @@ public void testOutOfRangeLongValues() { public void testInvalidPrimitivesAsRoot(String jsonStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(jsonStr); + byte[] json = toUtf8(jsonStr); // when JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); @@ -279,38 +257,12 @@ public void testInvalidPrimitivesAsRoot(String jsonStr) { .isEqualTo("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); } - @ParameterizedTest - @ValueSource(strings = { - "1.1", - "9355950000000000000.00000000000000000000000000000000001844674407370955161600000184467440737095516161844674407370955161407370955161618446744073709551616000184467440737095516166000001844674407370955161618446744073709551614073709551616184467440737095516160001844674407370955161601844674407370955674451616184467440737095516140737095516161844674407370955161600018446744073709551616018446744073709551611616000184467440737095001844674407370955161600184467440737095516160018446744073709551168164467440737095516160001844073709551616018446744073709551616184467440737095516160001844674407536910751601611616000184467440737095001844674407370955161600184467440737095516160018446744073709551616184467440737095516160001844955161618446744073709551616000184467440753691075160018446744073709", - "2.2250738585072013e-308", - "-92666518056446206563E3", - "-42823146028335318693e-128", - "90054602635948575728E72", - "1.00000000000000188558920870223463870174566020691753515394643550663070558368373221972569761144603605635692374830246134201063722058e-309", - "0e9999999999999999999999999999", - "-2402844368454405395.2" - }) - @Disabled("https://github.com/simdjson/simdjson-java/issues/5") - public void testFloatValues(String floatStr) { - // given - SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(floatStr); - - // when - JsonValue jsonValue = parser.parse(json, json.length); - - // then - assertThat(jsonValue.isDouble()).isTrue(); - assertThat(jsonValue.asDouble()).isEqualTo(Double.valueOf(floatStr)); - } - @ParameterizedTest @ValueSource(strings = {"[n]", "{\"a\":n}"}) public void testInvalidNull(String jsonStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(jsonStr); + byte[] json = toUtf8(jsonStr); // when JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); @@ -324,7 +276,7 @@ public void testInvalidNull(String jsonStr) { public void testInvalidFalse(String jsonStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(jsonStr); + byte[] json = toUtf8(jsonStr); // when JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); @@ -338,7 +290,7 @@ public void testInvalidFalse(String jsonStr) { public void testInvalidTrue(String jsonStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(jsonStr); + byte[] json = toUtf8(jsonStr); // when JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); @@ -347,25 +299,11 @@ public void testInvalidTrue(String jsonStr) { assertThat(ex.getMessage()).isEqualTo("Invalid value starting at " + jsonStr.indexOf('t') + ". Expected 'true'."); } - @ParameterizedTest - @ValueSource(strings = {"[-]", "{\"a\":-}"}) - public void testInvalidNegativeNumber(String jsonStr) { - // given - SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes(jsonStr); - - // when - JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); - - // then - assertThat(ex.getMessage()).isEqualTo("Invalid number starting at " + jsonStr.indexOf('-')); - } - @Test public void testUnicodeString() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[\"\\u005C\"]"); + byte[] json = toUtf8("[\"\\u005C\"]"); // when UnsupportedOperationException ex = assertThrows(UnsupportedOperationException.class, () -> parser.parse(json, json.length)); @@ -379,7 +317,7 @@ public void testUnicodeString() { public void testInvalidEscape(String jsonStr) { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[\"" + jsonStr + "\"]"); + byte[] json = toUtf8("[\"" + jsonStr + "\"]"); // when JsonParsingException ex = assertThrows(JsonParsingException.class, () -> parser.parse(json, json.length)); @@ -392,7 +330,7 @@ public void testInvalidEscape(String jsonStr) { public void testLongString() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]"); + byte[] json = toUtf8("[\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -409,7 +347,7 @@ public void testLongString() { public void testArraySize() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("[1, 2, 3]"); + byte[] json = toUtf8("[1, 2, 3]"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -423,7 +361,7 @@ public void testArraySize() { public void testObjectSize() { // given SimdJsonParser parser = new SimdJsonParser(); - byte[] json = toBytes("{\"1\":1,\"2\":1,\"3\":1}"); + byte[] json = toUtf8("{\"1\":1,\"2\":1,\"3\":1}"); // when JsonValue jsonValue = parser.parse(json, json.length); @@ -480,7 +418,8 @@ private static void assertLong(JsonValue actual, long expected) { assertThat(actual.asLong()).isEqualTo(expected); } - private static byte[] toBytes(String str) { - return str.getBytes(UTF_8); + private static void assertDouble(JsonValue actual, String str) { + assertThat(actual.isDouble()).isTrue(); + assertThat(actual.asDouble()).isEqualTo(Double.valueOf(str)); } } diff --git a/src/test/java/org/simdjson/StringUtils.java b/src/test/java/org/simdjson/StringUtils.java index e4bd1bd..db1eefc 100644 --- a/src/test/java/org/simdjson/StringUtils.java +++ b/src/test/java/org/simdjson/StringUtils.java @@ -5,25 +5,26 @@ import java.util.Arrays; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; class StringUtils { static String padWithSpaces(String str) { - int targetLength = 64; - byte[] strBytes = str.getBytes(UTF_8); - assertThat(strBytes.length).isLessThanOrEqualTo(targetLength); - byte[] padded = new byte[targetLength]; + byte[] strBytes = toUtf8(str); + byte[] padded = new byte[strBytes.length + 64]; Arrays.fill(padded, (byte) ' '); System.arraycopy(strBytes, 0, padded, 0, strBytes.length); return new String(padded, UTF_8); } static ByteVector chunk0(String str) { - return ByteVector.fromArray(ByteVector.SPECIES_256, str.getBytes(UTF_8), 0); + return ByteVector.fromArray(ByteVector.SPECIES_256, toUtf8(str), 0); } static ByteVector chunk1(String str) { - return ByteVector.fromArray(ByteVector.SPECIES_256, str.getBytes(UTF_8), 32); + return ByteVector.fromArray(ByteVector.SPECIES_256, toUtf8(str), 32); + } + + static byte[] toUtf8(String str) { + return str.getBytes(UTF_8); } } From de2993fce6a3f64e9abd3773abab0ab4d4db439d Mon Sep 17 00:00:00 2001 From: Piotr Rzysko Date: Thu, 31 Aug 2023 05:29:38 +0200 Subject: [PATCH 2/2] more comments --- src/main/java/org/simdjson/NumberParser.java | 28 ++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/simdjson/NumberParser.java b/src/main/java/org/simdjson/NumberParser.java index b83c2ea..4b350aa 100644 --- a/src/main/java/org/simdjson/NumberParser.java +++ b/src/main/java/org/simdjson/NumberParser.java @@ -157,7 +157,7 @@ private static double computeDouble(boolean negative, long significand10, long e // Initially, the number we are parsing is in the form of w * 10^q = w * 5^q * 2^q, and our objective is to // convert it to m * 2^p. We can represent w * 10^q as w * 5^q * 2^r * 2^p, where w * 5^q * 2^r = m. - // As a result, the next step involves computing w * 5^q. The implementation of this multiplication is optimized + // Therefore, in the next step we compute w * 5^q. The implementation of this multiplication is optimized // to minimize necessary operations while ensuring precise results. For further insight, refer to the // aforementioned paper. int powersOfFiveTableIndex = 2 * (int) (exp10 - FAST_PATH_MIN_POWER_OF_TEN); @@ -179,8 +179,23 @@ private static double computeDouble(boolean negative, long significand10, long e // values stored in POWERS_OF_FIVE are normalized, ensuring that their most significant bits are set, the product // has either 0 or 1 leading zeros. As a result, we need to perform a right shift of either 9 or 10 bits. long upperBit = upper >>> 63; - long significand2 = upper >>> (upperBit + 9); - + long upperShift = upperBit + 9; + long significand2 = upper >>> upperShift; + + // Now, we have to determine the value of the binary exponent. Let's begin by calculating the contribution of + // 10^q. Our goal is to compute f0 and f1 such that: + // - when q >= 0: 10^q = (5^q / 2^(f0 - q)) * 2^f0 + // - when q < 0: 10^q = (2^(f1 - q) / 5^-q) * 2^f1 + // Both (5^q / 2^(f0 - q)) and (2^(f1 - q) / 5^-q) must fall within the range of [1, 2). + // It turns out that these conditions are met when: + // - 0 <= q <= FAST_PATH_MAX_POWER_OF_TEN, and f0 = floor(log2(5^q)) + q = floor(q * log(5) / log(2)) + q = (217706 * q) / 2^16. + // - FAST_PATH_MIN_POWER_OF_TEN <= q < 0, and f1 = -ceil(log2(5^-q)) + q = -ceil(-q * log(5) / log(2)) + q = (217706 * q) / 2^16. + // Thus, we can express the contribution of 10^q to the exponent as (217706 * exp10) >> 16. + // + // Furthermore, we need to factor in the following normalizations we've performed: + // - shifting the decimal significand left bitwise + // - shifting the binary significand right bitwise if the most significant bit of the product was 1 + // Therefore, we add (63 - lz + upperBit) to the exponent. long exp2 = ((217706 * exp10) >> 16) + 63 - lz + upperBit; if (exp2 < IEEE64_MIN_FINITE_NUMBER_EXPONENT) { // In the next step, we right-shift the binary significand by the difference between the minimum exponent @@ -205,8 +220,11 @@ private static double computeDouble(boolean negative, long significand10, long e return toDouble(negative, significand2, exp2); } + // Here, we are addressing a scenario of rounding the binary significand when it falls precisely halfway + // between two integers. To understand the rationale behind the condition used to identify this case, refer to + // sections 6, 8.1, and 9.1 of "Number Parsing at a Gigabyte per Second". if ((compareUnsigned(lower, 1) <= 0) && (exp10 >= -4) && (exp10 <= 23) && ((significand2 & 3) == 1)) { - if (significand2 << (upperBit + 64 - 53 - 2) == upper) { + if (significand2 << upperShift == upper) { significand2 &= ~1; } } @@ -490,7 +508,7 @@ long computeSignificand() { if (exp10 < digitCount) { roundUp = digits[exp10] >= 5; if ((digits[exp10] == 5) && (exp10 + 1 == digitCount)) { - // If the digits haven't been truncated, then we are exactly half-way between two integers. In such + // If the digits haven't been truncated, then we are exactly halfway between two integers. In such // cases, we round to even, otherwise we round up. roundUp = truncated || (significand & 1) == 1; }