Skip to content

[8.x] Remove legacy block loader test infrastructure (#127273) #127413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 25, 2025
Merged

[8.x] Remove legacy block loader test infrastructure (#127273) #127413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e663c0a
Remove legacy block loader test infrastructure (#127273)
lkts Apr 25, 2025
62cb142
fix
lkts Apr 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions ...as/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.apache.lucene.tests.analysis.CannedTokenStream;
import org.apache.lucene.tests.analysis.Token;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Strings;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.mapper.DocumentMapper;
Expand All @@ -44,7 +43,6 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.containsString;
Expand Down Expand Up @@ -257,9 +255,4 @@ public void testDocValuesLoadedFromSynthetic() throws IOException {
protected IngestScriptSupport ingestScriptSupport() {
throw new AssumptionViolatedException("not supported");
}

@Override
protected Function<Object, Object> loadBlockExpected() {
return v -> ((BytesRef) v).utf8ToString();
}
}
8 changes: 1 addition & 7 deletions ...tras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,6 @@ public SyntheticSourceExample example(int maxVals) {
return new SyntheticSourceExample(
example.expectedForSyntheticSource(),
example.expectedForSyntheticSource(),
example.expectedForBlockLoader(),
example.mapping()
);
}
Expand All @@ -400,7 +399,7 @@ public SyntheticSourceExample example(int maxValues) {
if (v.malformedOutput == null) {
return new SyntheticSourceExample(v.input, v.output, this::mapping);
}
return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping);
return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping);
}
List<Value> values = randomList(1, maxValues, this::generateValue);
List<Object> in = values.stream().map(Value::input).toList();
Expand Down Expand Up @@ -479,11 +478,6 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
}
}

protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
assumeTrue("Disabled, tested by ScaledFloatFieldBlockLoaderTests instead", false);
return null;
}

@Override
protected IngestScriptSupport ingestScriptSupport() {
throw new AssumptionViolatedException("not supported");
Expand Down
15 changes: 2 additions & 13 deletions ...xtras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -213,17 +211,13 @@ public boolean preservesExactSource() {
public SyntheticSourceExample example(int maxValues) {
if (randomBoolean()) {
var value = generateValue();
return new SyntheticSourceExample(value.text, value.text, value.tokenCount, this::mapping);
return new SyntheticSourceExample(value.text, value.text, this::mapping);
}

var values = randomList(1, 5, this::generateValue);

var textArray = values.stream().map(Value::text).toList();

var blockExpectedList = values.stream().map(Value::tokenCount).filter(Objects::nonNull).sorted().toList();
var blockExpected = blockExpectedList.size() == 1 ? blockExpectedList.get(0) : blockExpectedList;

return new SyntheticSourceExample(textArray, textArray, blockExpected, this::mapping);
return new SyntheticSourceExample(textArray, textArray, this::mapping);
}

private record Value(String text, Integer tokenCount) {}
Expand Down Expand Up @@ -258,11 +252,6 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
};
}

protected Function<Object, Object> loadBlockExpected() {
// we can get either a number from doc values or null
return v -> v != null ? (Number) v : null;
}

@Override
protected IngestScriptSupport ingestScriptSupport() {
throw new AssumptionViolatedException("not supported");
Expand Down
11 changes: 0 additions & 11 deletions ...test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -666,16 +665,6 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
return TextFieldFamilySyntheticSourceTestSetup.syntheticSourceSupport("annotated_text", false);
}

@Override
protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
return TextFieldFamilySyntheticSourceTestSetup.getSupportedReaders(mapper, loaderFieldName);
}

@Override
protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
return TextFieldFamilySyntheticSourceTestSetup.loadBlockExpected(blockReaderSupport, columnReader);
}

@Override
protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) {
TextFieldFamilySyntheticSourceTestSetup.validateRoundTripReader(syntheticSource, reader, roundTripReader);
Expand Down
10 changes: 7 additions & 3 deletions server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1020,9 +1020,13 @@ protected String delegatingTo() {
return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name());
}

// _ignored_source field will only be present if text field is not stored
// and there is no syntheticSourceDelegate
if (isSyntheticSource && syntheticSourceDelegate == null) {
// _ignored_source field will contain entries for this field if it is not stored
// and there is no syntheticSourceDelegate.
// See #syntheticSourceSupport().
// But if a text field is a multi field it won't have an entry in _ignored_source.
// The parent might, but we don't have enough context here to figure this out.
// So we bail.
if (isSyntheticSource && syntheticSourceDelegate == null && parentField == null) {
return fallbackSyntheticSourceBlockLoader();
}

Expand Down
8 changes: 0 additions & 8 deletions server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.function.Function;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -371,19 +370,12 @@ public SyntheticSourceExample example(int maxVals) throws IOException {
return new SyntheticSourceExample(
example.expectedForSyntheticSource(),
example.expectedForSyntheticSource(),
example.expectedForBlockLoader(),
example.mapping()
);
}
};
}

@Override
protected Function<Object, Object> loadBlockExpected() {
// Just assert that we expect a boolean. Otherwise no munging.
return v -> (Boolean) v;
}

protected IngestScriptSupport ingestScriptSupport() {
return new IngestScriptSupport() {
@Override
Expand Down
32 changes: 3 additions & 29 deletions server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import java.time.ZonedDateTime;
import java.util.Comparator;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Stream;

import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
Expand Down Expand Up @@ -595,15 +594,10 @@ public SyntheticSourceExample example(int maxValues) {
if (randomBoolean()) {
Value v = generateValue();
if (v.malformedOutput != null) {
return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping);
return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping);
}

return new SyntheticSourceExample(
v.input,
v.output,
resolution.convert(Instant.from(formatter.parse(v.output))),
this::mapping
);
return new SyntheticSourceExample(v.input, v.output, this::mapping);
}

List<Value> values = randomList(1, maxValues, this::generateValue);
Expand All @@ -625,11 +619,7 @@ public SyntheticSourceExample example(int maxValues) {
List<Object> outList = Stream.concat(outputFromDocValues.stream(), malformedOutput).toList();
Object out = outList.size() == 1 ? outList.get(0) : outList;

List<Long> outBlockList = outputFromDocValues.stream()
.map(v -> resolution.convert(Instant.from(formatter.parse(v))))
.toList();
Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
return new SyntheticSourceExample(in, out, outBlock, this::mapping);
return new SyntheticSourceExample(in, out, this::mapping);
}

private record Value(Object input, String output, Object malformedOutput) {}
Expand Down Expand Up @@ -727,22 +717,6 @@ public void execute() {
};
}

@Override
protected Function<Object, Object> loadBlockExpected() {
return v -> asJacksonNumberOutput(((Number) v).longValue());
}

protected static Object asJacksonNumberOutput(long l) {
// If a long value fits in int, Jackson will write it as int in NumberOutput.outputLong()
// and we hit this during serialization of expected values.
// Code below mimics that behaviour in order for matching to work.
if (l < 0 && l >= Integer.MIN_VALUE || l >= 0 && l <= Integer.MAX_VALUE) {
return (int) l;
} else {
return l;
}
}

public void testLegacyField() throws Exception {
// check that unknown date formats are treated leniently on old indices
MapperService service = createMapperService(IndexVersion.fromId(5000099), Settings.EMPTY, () -> false, mapping(b -> {
Expand Down
10 changes: 0 additions & 10 deletions server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

import java.io.IOException;
import java.util.List;
import java.util.function.Function;

public class FloatFieldMapperTests extends NumberFieldMapperTests {

Expand Down Expand Up @@ -61,15 +60,6 @@ protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean igno

}

@Override
protected Function<Object, Object> loadBlockExpected() {
return v -> {
// The test converts the float into a string so we do do
Number n = (Number) v;
return Double.parseDouble(Float.toString(n.floatValue()));
};
}

@Override
protected IngestScriptSupport ingestScriptSupport() {
throw new AssumptionViolatedException("not supported");
Expand Down
60 changes: 4 additions & 56 deletions server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,7 @@
import org.elasticsearch.common.geo.GeoJson;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.geo.GeometryTestUtils;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.geometry.utils.GeometryValidator;
import org.elasticsearch.geometry.utils.WellKnownBinary;
import org.elasticsearch.geometry.utils.WellKnownText;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSettings;
Expand All @@ -37,7 +34,6 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;

Expand Down Expand Up @@ -611,13 +607,13 @@ public SyntheticSourceExample example(int maxVals) {
if (randomBoolean()) {
Value v = generateValue();
if (v.malformedOutput != null) {
return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping);
return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping);
}

if (columnReader) {
return new SyntheticSourceExample(v.input, decode(encode(v.output)), encode(v.output), this::mapping);
return new SyntheticSourceExample(v.input, decode(encode(v.output)), this::mapping);
}
return new SyntheticSourceExample(v.input, v.output, v.output.toWKT(), this::mapping);
return new SyntheticSourceExample(v.input, v.output, this::mapping);

}
List<Value> values = randomList(1, maxVals, this::generateValue);
Expand All @@ -635,18 +631,7 @@ public SyntheticSourceExample example(int maxVals) {
List<Object> outList = Stream.concat(outputFromDocValues.stream(), malformedValues).toList();
Object out = outList.size() == 1 ? outList.get(0) : outList;

if (columnReader) {
// When reading doc-values, the block is a list of encoded longs
List<Long> outBlockList = outputFromDocValues.stream().map(this::encode).toList();
Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
return new SyntheticSourceExample(in, out, outBlock, this::mapping);
} else {
// When reading row-stride, the block is a list of WKT encoded BytesRefs.
// Values are ordered in order of input.
List<String> outBlockList = values.stream().filter(v -> v.malformedOutput == null).map(v -> v.output.toWKT()).toList();
Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList;
return new SyntheticSourceExample(in, out, outBlock, this::mapping);
}
return new SyntheticSourceExample(in, out, this::mapping);
}

private record Value(Object input, GeoPoint output, Object malformedOutput) {}
Expand Down Expand Up @@ -736,41 +721,4 @@ public void testSyntheticSourceKeepArrays() {
protected IngestScriptSupport ingestScriptSupport() {
throw new AssumptionViolatedException("not supported");
}

@Override
protected Function<Object, Object> loadBlockExpected() {
throw new IllegalStateException("Should never reach here, call loadBlockExpected(BlockReaderSupport, boolean) instead");
}

@Override
protected Function<Object, Object> loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) {
if (columnReader) {
// When using column reader, we expect the output to be doc-values (which means encoded longs)
return v -> asJacksonNumberOutput(((Number) v).longValue());
} else {
// When using row-stride reader, we expect the output to be WKT encoded BytesRef
return v -> asWKT((BytesRef) v);
}
}

protected static Object asJacksonNumberOutput(long l) {
// Cast to int to mimic jackson-core behaviour in NumberOutput.outputLong()
if (l < 0 && l >= Integer.MIN_VALUE || l >= 0 && l <= Integer.MAX_VALUE) {
return (int) l;
} else {
return l;
}
}

protected static Object asWKT(BytesRef value) {
// Internally we use WKB in BytesRef, but for test assertions we want to use WKT for readability
Geometry geometry = WellKnownBinary.fromWKB(GeometryValidator.NOOP, false, value.bytes);
return WellKnownText.toWKT(geometry);
}

@Override
protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
MappedFieldType ft = mapper.fieldType(loaderFieldName);
return new BlockReaderSupport(ft.hasDocValues(), false, mapper, loaderFieldName);
}
}
12 changes: 0 additions & 12 deletions server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

import java.io.IOException;
import java.util.List;
import java.util.function.Function;

public class HalfFloatFieldMapperTests extends NumberFieldMapperTests {

Expand Down Expand Up @@ -65,17 +64,6 @@ protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean igno

}

@Override
protected Function<Object, Object> loadBlockExpected() {
return v -> {
// The test converts the float into a string so we do do
Number n = (Number) v;
return Double.parseDouble(
Float.toString(HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(n.floatValue())))
);
};
}

@Override
protected IngestScriptSupport ingestScriptSupport() {
throw new AssumptionViolatedException("not supported");
Expand Down
6 changes: 0 additions & 6 deletions server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -435,11 +434,6 @@ public void execute() {
};
}

@Override
protected Function<Object, Object> loadBlockExpected() {
return v -> InetAddresses.toAddrString(InetAddressPoint.decode(BytesRef.deepCopyOf((BytesRef) v).bytes));
}

@Override
protected String randomSyntheticSourceKeep() {
return "all";
Expand Down
Loading