Skip to content

Fix reranker tests #126500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 9, 2025
Merged

Fix reranker tests #126500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a9e7731
Disable RERANK CSV tests on mixed clusters.
afoucret Apr 7, 2025
fae8c37
Remove score from CSV tests (rounding is causing flakiness).
afoucret Apr 8, 2025
705698b
More concise tests.
afoucret Apr 8, 2025
554ec4f
Merge branch 'main' into fix-reranker-tests
afoucret Apr 9, 2025
7d4936d
Code style.
afoucret Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions ...i-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.RERANK;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS;
import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
import static org.mockito.ArgumentMatchers.any;
Expand Down Expand Up @@ -131,8 +130,6 @@ protected void shouldSkipTest(String testName) throws IOException {
assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName()));
// Unmapped fields require a coorect capability response from every cluster, which isn't currently implemented.
assumeFalse("UNMAPPED FIELDS not yet supported in CCS", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()));
// Need to do additional developmnet to get CSS support for the rerank coammnd
assumeFalse("RERANK not yet supported in CCS", testCase.requiredCapabilities.contains(RERANK.capabilityName()));
}

@Override
Expand Down
12 changes: 10 additions & 2 deletions ...n/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.LongStream;
import java.util.stream.Stream;

import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
Expand All @@ -74,6 +75,8 @@
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METRICS_COMMAND;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.RERANK;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SEMANTIC_TEXT_FIELD_CAPS;
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SOURCE_FIELD_MAPPING;

// This test can run very long in serverless configurations
Expand Down Expand Up @@ -177,8 +180,8 @@ public final void test() throws Throwable {
}

protected void shouldSkipTest(String testName) throws IOException {
if (testCase.requiredCapabilities.contains("semantic_text_field_caps") || testCase.requiredCapabilities.contains("rerank")) {
assumeTrue("Inference test service needs to be supported for semantic_text", supportsInferenceTestService());
if (requiresInferenceEndpoint()) {
assumeTrue("Inference test service needs to be supported", supportsInferenceTestService());
}
checkCapabilities(adminClient(), testFeatureService, testName, testCase);
assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, instructions, Version.CURRENT));
Expand Down Expand Up @@ -248,6 +251,11 @@ protected boolean supportsInferenceTestService() {
return true;
}

protected boolean requiresInferenceEndpoint() {
return Stream.of(SEMANTIC_TEXT_FIELD_CAPS.capabilityName(), RERANK.capabilityName())
.anyMatch(testCase.requiredCapabilities::contains);
}

protected boolean supportsIndexModeLookup() throws IOException {
return true;
}
Expand Down
88 changes: 31 additions & 57 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/rerank.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,14 @@ required_capability: match_operator_colon
FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author, _score
| EVAL _score = ROUND(_score, 5)
| KEEP book_no, title, author
;

book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.03846
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.02222
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.02083
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.01515
book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy
;


Expand All @@ -29,15 +28,14 @@ required_capability: match_operator_colon
FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title, author WITH test_reranker
| KEEP book_no, title, author, _score
| EVAL _score = ROUND(_score, 5)
| KEEP book_no, title, author
;

book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.02083
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.01429
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.01136
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.00952
book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
;


Expand All @@ -50,14 +48,13 @@ FROM books METADATA _score
| SORT _score DESC
| LIMIT 3
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author, _score
| EVAL _score = ROUND(_score, 5)
| KEEP book_no, title, author
;

book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.03846
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.02222
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.02083
book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
;


Expand All @@ -68,35 +65,14 @@ required_capability: match_operator_colon
FROM books METADATA _score
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author, _score
| KEEP book_no, title, author
| LIMIT 3
| EVAL _score = ROUND(_score, 5)
;

book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.03846
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.02222
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.02083
;


reranker add the _score column when missing
required_capability: rerank
required_capability: match_operator_colon

FROM books
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author, _score
| EVAL _score = ROUND(_score, 5)
;


book_no:keyword | title:text | author:text | _score:double
5327 | War and Peace | Leo Tolstoy | 0.03846
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.02222
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo | 0.02083
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.01515
book_no:keyword | title:text | author:text
5327 | War and Peace | Leo Tolstoy
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
9032 | War and Peace: A Novel (6 Volumes) | Tolstoy Leo
;


Expand All @@ -107,16 +83,15 @@ required_capability: match_operator_colon
FROM books
| WHERE title:"war and peace" AND author:"Tolstoy"
| RERANK "war and peace" ON title WITH test_reranker
| KEEP book_no, title, author, _score
| KEEP book_no, title, author
| SORT author, title
| LIMIT 3
| EVAL _score = ROUND(_score, 5)
;

book_no:keyword | title:text | author:text | _score:double
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy] | 0.02222
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy | 0.01515
5327 | War and Peace | Leo Tolstoy | 0.03846
book_no:keyword | title:text | author:text
4536 | War and Peace (Signet Classics) | [John Hockenberry, Leo Tolstoy, Pat Conroy]
2776 | The Devil and Other Stories (Oxford World's Classics) | Leo Tolstoy
5327 | War and Peace | Leo Tolstoy
;


Expand All @@ -132,11 +107,10 @@ FROM books METADATA _id, _index, _score
| RRF
| RERANK "Tolkien" ON title WITH test_reranker
| LIMIT 2
| KEEP book_no, title, author, _score
| EVAL _score = ROUND(_score, 5)
| KEEP book_no, title, author
;

book_no:keyword | title:keyword | author:keyword | _score:double
5335 | Letters of J R R Tolkien | J.R.R. Tolkien | 0.02632
2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien] | 0.01961
book_no:keyword | title:keyword | author:keyword
5335 | Letters of J R R Tolkien | J.R.R. Tolkien
2130 | The J. R. R. Tolkien Audio Collection | [Christopher Tolkien, John Ronald Reuel Tolkien]
;