From d95121e860df03499a8aa5868e042e702a24de2f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 14 Mar 2025 10:59:22 -0400 Subject: [PATCH 1/7] ESQL: Report original_types Adds the `original_types` to the description of ESQL's `unsupported` fields. This looks like: ``` { "name" : "a", "type" : "unsupported", "original_types" : [ "long", "text" ] } ``` for union types. And like: ``` { "name" : "a", "type" : "unsupported", "original_types" : [ "date_range" ] } ``` for truly unsupported types. This information is useful for the UI. For union types it can suggest that users append a cast. --- .../org/elasticsearch/TransportVersions.java | 1 + .../xpack/esql/core/expression/Attribute.java | 10 ++ .../esql/core/type/UnsupportedEsField.java | 43 ++++--- .../esql/qa/rest/FieldExtractorTestCase.java | 69 ++++++----- .../elasticsearch/xpack/esql/LoadMapping.java | 7 +- .../esql/action/AsyncEsqlQueryActionIT.java | 8 +- .../esql/action/CrossClusterAsyncQueryIT.java | 9 +- .../xpack/esql/action/EsqlActionIT.java | 103 ++++++++++------- .../xpack/esql/action/TimeSeriesIT.java | 108 ++++++++++-------- .../xpack/esql/action/ColumnInfoImpl.java | 48 ++++++-- .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../xpack/esql/analysis/Analyzer.java | 2 +- .../function/UnsupportedAttribute.java | 8 +- .../esql/plugin/TransportEsqlQueryAction.java | 17 ++- .../xpack/esql/session/IndexResolver.java | 4 +- .../esql/action/EsqlQueryResponseTests.java | 49 ++++++-- .../xpack/esql/analysis/VerifierTests.java | 2 +- .../xpack/esql/formatter/TextFormatTests.java | 17 +-- .../esql/formatter/TextFormatterTests.java | 22 ++-- .../esql/type/UnsupportedEsFieldTests.java | 17 ++- .../test/esql/160_union_types.yml | 4 + .../test/esql/40_unsupported_types.yml | 18 +-- 22 files changed, 367 insertions(+), 206 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 3ace93ece62f0..4be6074919e19 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -185,6 +185,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_THREAD_NAME_IN_DRIVER_PROFILE = def(9_027_0_00); public static final TransportVersion INFERENCE_CONTEXT = def(9_028_0_00); public static final TransportVersion ML_INFERENCE_DEEPSEEK = def(9_029_00_0); + public static final TransportVersion ESQL_REPORT_ORIGINAL_TYPES = def(9_030_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java index 829943d245149..d419f7368a72d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java @@ -137,4 +137,14 @@ public String nodeString() { } protected abstract String label(); + + /** + * If this field is unsupported this contains the underlying ES types. If there + * is a type conflict this will have many elements, some or all of which may + * be actually supported types. + */ + @Nullable + public List originalTypes() { + return null; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java index 02ce741243c20..ba2e138461a01 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java @@ -6,10 +6,14 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; @@ -23,32 +27,39 @@ */ public class UnsupportedEsField extends EsField { - private final String originalType; + private final List originalTypes; private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type - public UnsupportedEsField(String name, String originalType) { - this(name, originalType, null, new TreeMap<>()); + public UnsupportedEsField(String name, List originalTypes) { + this(name, originalTypes, null, new TreeMap<>()); } - public UnsupportedEsField(String name, String originalType, String inherited, Map properties) { + public UnsupportedEsField(String name, List originalTypes, String inherited, Map properties) { super(name, DataType.UNSUPPORTED, properties, false); - this.originalType = originalType; + this.originalTypes = originalTypes; this.inherited = inherited; } public UnsupportedEsField(StreamInput in) throws IOException { - this( - readCachedStringWithVersionCheck(in), - readCachedStringWithVersionCheck(in), - in.readOptionalString(), - in.readImmutableMap(EsField::readFrom) - ); + this(readCachedStringWithVersionCheck(in), readOriginalTypes(in), in.readOptionalString(), in.readImmutableMap(EsField::readFrom)); + } + + private static List readOriginalTypes(StreamInput in) throws IOException { + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REPORT_ORIGINAL_TYPES)) { + return in.readCollectionAsList(i -> ((PlanStreamInput) i).readCachedString()); + } else { + return List.of(readCachedStringWithVersionCheck(in).split(",")); + } } @Override public void writeContent(StreamOutput out) throws IOException { writeCachedStringWithVersionCheck(out, getName()); - writeCachedStringWithVersionCheck(out, getOriginalType()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REPORT_ORIGINAL_TYPES)) { + out.writeCollection(getOriginalTypes(), (o, s) -> ((PlanStreamOutput) o).writeCachedString(s)); + } else { + writeCachedStringWithVersionCheck(out, String.join(",", getOriginalTypes())); + } out.writeOptionalString(getInherited()); out.writeMap(getProperties(), (o, x) -> x.writeTo(out)); } @@ -57,8 +68,8 @@ public String getWriteableName() { return "UnsupportedEsField"; } - public String getOriginalType() { - return originalType; + public List getOriginalTypes() { + return originalTypes; } public String getInherited() { @@ -81,11 +92,11 @@ public boolean equals(Object o) { return false; } UnsupportedEsField that = (UnsupportedEsField) o; - return Objects.equals(originalType, that.originalType) && Objects.equals(inherited, that.inherited); + return Objects.equals(originalTypes, that.originalTypes) && Objects.equals(inherited, that.inherited); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), originalType, inherited); + return Objects.hash(super.hashCode(), originalTypes, inherited); } } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java index 350260e17640a..9a537155b9426 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java @@ -323,12 +323,13 @@ public void testAliasToInt() throws IOException { } public void testFlattenedUnsupported() throws IOException { + assumeOriginalTypesReported(); new Test("flattened").createIndex("test", "flattened"); index("test", """ {"flattened": {"a": "foo"}}"""); Map result = runEsql("FROM test* | LIMIT 2"); - assertResultMap(result, List.of(columnInfo("flattened", "unsupported")), List.of(matchesList().item(null))); + assertResultMap(result, List.of(unsupportedColumnInfo("flattened", "flattened")), List.of(matchesList().item(null))); } public void testEmptyMapping() throws IOException { @@ -685,6 +686,7 @@ public void testByteFieldWithIntSubfieldTooBig() throws IOException { * . */ public void testIncompatibleTypes() throws IOException { + assumeOriginalTypesReported(); keywordTest().createIndex("test1", "f"); index("test1", """ {"f": "f1"}"""); @@ -693,7 +695,11 @@ public void testIncompatibleTypes() throws IOException { {"f": 1}"""); Map result = runEsql("FROM test*"); - assertResultMap(result, List.of(columnInfo("f", "unsupported")), List.of(matchesList().item(null), matchesList().item(null))); + assertResultMap( + result, + List.of(unsupportedColumnInfo("f", "keyword", "long")), + List.of(matchesList().item(null), matchesList().item(null)) + ); ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT f | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat( @@ -754,10 +760,7 @@ public void testDistinctInEachIndex() throws IOException { * . */ public void testMergeKeywordAndObject() throws IOException { - assumeTrue( - "order of fields in error message inconsistent before 8.14", - getCachedNodesVersions().stream().allMatch(v -> Version.fromString(v).onOrAfter(Version.V_8_14_0)) - ); + assumeOriginalTypesReported(); keywordTest().createIndex("test1", "file"); index("test1", """ {"file": "f1"}"""); @@ -793,7 +796,7 @@ public void testMergeKeywordAndObject() throws IOException { Map result = runEsql("FROM test* | SORT file.raw | LIMIT 2"); assertResultMap( result, - List.of(columnInfo("file", "unsupported"), columnInfo("file.raw", "keyword")), + List.of(unsupportedColumnInfo("file", "keyword", "object"), columnInfo("file.raw", "keyword")), List.of(matchesList().item(null).item("o2"), matchesList().item(null).item(null)) ); } @@ -813,6 +816,7 @@ public void testMergeKeywordAndObject() throws IOException { * . */ public void testPropagateUnsupportedToSubFields() throws IOException { + assumeOriginalTypesReported(); createIndex("test", index -> { index.startObject("properties"); index.startObject("f"); @@ -838,7 +842,7 @@ public void testPropagateUnsupportedToSubFields() throws IOException { Map result = runEsql("FROM test* | LIMIT 2"); assertResultMap( result, - List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported")), + List.of(unsupportedColumnInfo("f", "ip_range"), unsupportedColumnInfo("f.raw", "ip_range")), List.of(matchesList().item(null).item(null)) ); } @@ -863,10 +867,7 @@ public void testPropagateUnsupportedToSubFields() throws IOException { * . */ public void testMergeUnsupportedAndObject() throws IOException { - assumeTrue( - "order of fields in error message inconsistent before 8.14", - getCachedNodesVersions().stream().allMatch(v -> Version.fromString(v).onOrAfter(Version.V_8_14_0)) - ); + assumeOriginalTypesReported(); createIndex("test1", index -> { index.startObject("properties"); index.startObject("f").field("type", "ip_range").endObject(); @@ -901,7 +902,7 @@ public void testMergeUnsupportedAndObject() throws IOException { Map result = runEsql("FROM test* | LIMIT 2"); assertResultMap( result, - List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported")), + List.of(unsupportedColumnInfo("f", "ip_range"), unsupportedColumnInfo("f.raw", "ip_range")), List.of(matchesList().item(null).item(null), matchesList().item(null).item(null)) ); } @@ -954,10 +955,7 @@ public void testIntegerDocValuesConflict() throws IOException { * In an ideal world we'd promote the {@code integer} to an {@code long} and just go. */ public void testLongIntegerConflict() throws IOException { - assumeTrue( - "order of fields in error message inconsistent before 8.14", - getCachedNodesVersions().stream().allMatch(v -> Version.fromString(v).onOrAfter(Version.V_8_14_0)) - ); + assumeOriginalTypesReported(); longTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ {"emp_no": 1}"""); @@ -976,7 +974,11 @@ public void testLongIntegerConflict() throws IOException { ); Map result = runEsql("FROM test* | LIMIT 2"); - assertResultMap(result, List.of(columnInfo("emp_no", "unsupported")), List.of(matchesList().item(null), matchesList().item(null))); + assertResultMap( + result, + List.of(unsupportedColumnInfo("emp_no", "integer", "long")), + List.of(matchesList().item(null), matchesList().item(null)) + ); } /** @@ -996,10 +998,7 @@ public void testLongIntegerConflict() throws IOException { * In an ideal world we'd promote the {@code short} to an {@code integer} and just go. */ public void testIntegerShortConflict() throws IOException { - assumeTrue( - "order of fields in error message inconsistent before 8.14", - getCachedNodesVersions().stream().allMatch(v -> Version.fromString(v).onOrAfter(Version.V_8_14_0)) - ); + assumeOriginalTypesReported(); intTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ {"emp_no": 1}"""); @@ -1018,7 +1017,11 @@ public void testIntegerShortConflict() throws IOException { ); Map result = runEsql("FROM test* | LIMIT 2"); - assertResultMap(result, List.of(columnInfo("emp_no", "unsupported")), List.of(matchesList().item(null), matchesList().item(null))); + assertResultMap( + result, + List.of(unsupportedColumnInfo("emp_no", "integer", "short")), + List.of(matchesList().item(null), matchesList().item(null)) + ); } /** @@ -1044,10 +1047,7 @@ public void testIntegerShortConflict() throws IOException { * . */ public void testTypeConflictInObject() throws IOException { - assumeTrue( - "order of fields in error message inconsistent before 8.14", - getCachedNodesVersions().stream().allMatch(v -> Version.fromString(v).onOrAfter(Version.V_8_14_0)) - ); + assumeOriginalTypesReported(); createIndex("test1", empNoInObject("integer")); index("test1", """ {"foo": {"emp_no": 1}}"""); @@ -1056,7 +1056,10 @@ public void testTypeConflictInObject() throws IOException { {"foo": {"emp_no": "cat"}}"""); Map result = runEsql("FROM test* | LIMIT 3"); - assertMap(result, getResultMatcher(result).entry("columns", List.of(columnInfo("foo.emp_no", "unsupported"))).extraOk()); + assertMap( + result, + getResultMatcher(result).entry("columns", List.of(unsupportedColumnInfo("foo.emp_no", "integer", "keyword"))).extraOk() + ); ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT foo.emp_no | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); @@ -1366,6 +1369,12 @@ private void assumeIndexResolverNestedFieldsNameClashFixed() throws IOException ); } + private void assumeOriginalTypesReported() throws IOException { + var capsName = EsqlCapabilities.Cap.REPORT_ORIGINAL_TYPES.name().toLowerCase(Locale.ROOT); + boolean requiredClusterCapability = clusterHasCapability("POST", "/_query", List.of(), List.of(capsName)).orElse(false); + assumeTrue("This test makes sense for versions that report original types", requiredClusterCapability); + } + private CheckedConsumer empNoInObject(String empNoType) { return index -> { index.startObject("properties"); @@ -1701,6 +1710,10 @@ private static Map columnInfo(String name, String type) { return Map.of("name", name, "type", type); } + private static Map unsupportedColumnInfo(String name, String... originalTypes) { + return Map.of("name", name, "type", "unsupported", "original_types", List.of(originalTypes)); + } + private static void index(String name, String... docs) throws IOException { Request request = new Request("POST", "/" + name + "/_bulk"); request.addParameter("refresh", "true"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java index e750502cad198..1146d7729a8d4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; @@ -110,7 +111,7 @@ private static void walkMapping(String name, Object value, Map field = DateEsField.dateEsField(name, properties, docValues); } else if (esDataType == UNSUPPORTED) { String type = content.get("type").toString(); - field = new UnsupportedEsField(name, type, null, properties); + field = new UnsupportedEsField(name, List.of(type), null, properties); propagateUnsupportedType(name, type, properties); } else { field = new EsField(name, esDataType, properties, docValues); @@ -165,9 +166,9 @@ public static void propagateUnsupportedType(String inherited, String originalTyp UnsupportedEsField u; if (field instanceof UnsupportedEsField) { u = (UnsupportedEsField) field; - u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties()); + u = new UnsupportedEsField(u.getName(), List.of(originalType), inherited, u.getProperties()); } else { - u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties()); + u = new UnsupportedEsField(field.getName(), List.of(originalType), inherited, field.getProperties()); } entry.setValue(u); propagateUnsupportedType(inherited, originalType, u.getProperties()); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java index 163cbc8491298..41bac7f09df77 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java @@ -94,7 +94,7 @@ public void testBasicAsyncExecution() throws Exception { try (var finalResponse = future.get()) { assertThat(finalResponse, notNullValue()); assertThat(finalResponse.isRunning(), is(false)); - assertThat(finalResponse.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long")))); + assertThat(finalResponse.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long", null)))); assertThat(getValuesList(finalResponse).size(), equalTo(1)); } @@ -103,7 +103,7 @@ public void testBasicAsyncExecution() throws Exception { try (var finalResponse = again.get()) { assertThat(finalResponse, notNullValue()); assertThat(finalResponse.isRunning(), is(false)); - assertThat(finalResponse.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long")))); + assertThat(finalResponse.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long", null)))); assertThat(getValuesList(finalResponse).size(), equalTo(1)); } @@ -231,7 +231,7 @@ private void testFinishingBeforeTimeout(boolean keepOnCompletion) { try (var response = request.execute().actionGet(60, TimeUnit.SECONDS)) { assertThat(response.isRunning(), is(false)); - assertThat(response.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long")))); + assertThat(response.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long", null)))); assertThat(getValuesList(response).size(), equalTo(1)); if (keepOnCompletion) { @@ -244,7 +244,7 @@ private void testFinishingBeforeTimeout(boolean keepOnCompletion) { try (var resp = future.actionGet(60, TimeUnit.SECONDS)) { assertThat(resp.asyncExecutionId().get(), equalTo(id)); assertThat(resp.isRunning(), is(false)); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(pause_me)", "long", null)))); assertThat(getValuesList(resp).size(), equalTo(1)); } } else { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java index d5c53e94f8687..cb404c0c93006 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java @@ -31,6 +31,7 @@ import static org.elasticsearch.xpack.esql.action.EsqlAsyncTestUtils.waitForCluster; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; @@ -151,10 +152,10 @@ public void testAsyncQueriesWithLimit0() throws IOException { } else { assertThat(resp.columns().size(), equalTo(4)); - assertThat(resp.columns().contains(new ColumnInfoImpl("const", "long")), is(true)); - assertThat(resp.columns().contains(new ColumnInfoImpl("id", "keyword")), is(true)); - assertThat(resp.columns().contains(new ColumnInfoImpl("tag", "keyword")), is(true)); - assertThat(resp.columns().contains(new ColumnInfoImpl("v", "long")), is(true)); + assertThat(resp.columns(), hasItem(new ColumnInfoImpl("const", "long", null))); + assertThat(resp.columns(), hasItem(new ColumnInfoImpl("id", "keyword", null))); + assertThat(resp.columns(), hasItem(new ColumnInfoImpl("tag", "keyword", null))); + assertThat(resp.columns(), hasItem(new ColumnInfoImpl("v", "long", null))); assertThat(resp.values().hasNext(), is(false)); // values should be empty list assertNotNull(executionInfo); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index 9775b02bdf217..db11757097507 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -102,7 +102,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testProjectConstant() { try (EsqlQueryResponse results = run("from test | eval x = 1 | keep x")) { - assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "integer")))); + assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "integer", null)))); assertThat(getValuesList(results).size(), equalTo(40)); assertThat(getValuesList(results).get(0).get(0), equalTo(1)); } @@ -110,7 +110,7 @@ public void testProjectConstant() { public void testStatsOverConstant() { try (EsqlQueryResponse results = run("from test | eval x = 1 | stats x = count(x)")) { - assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "long")))); + assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "long", null)))); assertThat(getValuesList(results).size(), equalTo(1)); assertThat(getValuesList(results).get(0).get(0), equalTo(40L)); } @@ -538,10 +538,14 @@ public void testFromStatsEvalWithPragma() { logger.info(results); assertEquals(1, getValuesList(results).size()); assertEquals(2, getValuesList(results).get(0).size()); - assertEquals(50, (double) getValuesList(results).get(0).get(results.columns().indexOf(new ColumnInfoImpl("x", "double"))), 1d); + assertEquals( + 50, + (double) getValuesList(results).get(0).get(results.columns().indexOf(new ColumnInfoImpl("x", "double", null))), + 1d + ); assertEquals( 43, - (double) getValuesList(results).get(0).get(results.columns().indexOf(new ColumnInfoImpl("avg_count", "double"))), + (double) getValuesList(results).get(0).get(results.columns().indexOf(new ColumnInfoImpl("avg_count", "double", null))), 1d ); } @@ -551,7 +555,7 @@ public void testWhere() { try (EsqlQueryResponse results = run("from test | where count > 40")) { logger.info(results); assertEquals(30, getValuesList(results).size()); - var countIndex = results.columns().indexOf(new ColumnInfoImpl("count", "long")); + var countIndex = results.columns().indexOf(new ColumnInfoImpl("count", "long", null)); for (List values : getValuesList(results)) { assertThat((Long) values.get(countIndex), greaterThan(40L)); } @@ -562,7 +566,7 @@ public void testProjectWhere() { try (EsqlQueryResponse results = run("from test | keep count | where count > 40")) { logger.info(results); assertEquals(30, getValuesList(results).size()); - int countIndex = results.columns().indexOf(new ColumnInfoImpl("count", "long")); + int countIndex = results.columns().indexOf(new ColumnInfoImpl("count", "long", null)); for (List values : getValuesList(results)) { assertThat((Long) values.get(countIndex), greaterThan(40L)); } @@ -573,7 +577,7 @@ public void testEvalWhere() { try (EsqlQueryResponse results = run("from test | eval x = count / 2 | where x > 20")) { logger.info(results); assertEquals(30, getValuesList(results).size()); - int countIndex = results.columns().indexOf(new ColumnInfoImpl("x", "long")); + int countIndex = results.columns().indexOf(new ColumnInfoImpl("x", "long", null)); for (List values : getValuesList(results)) { assertThat((Long) values.get(countIndex), greaterThan(20L)); } @@ -591,7 +595,7 @@ public void testSortWithNull() { try (EsqlQueryResponse results = run("row a = null | sort a")) { logger.info(results); assertEquals(1, getValuesList(results).size()); - int countIndex = results.columns().indexOf(new ColumnInfoImpl("a", "null")); + int countIndex = results.columns().indexOf(new ColumnInfoImpl("a", "null", null)); assertThat(results.columns().stream().map(ColumnInfo::name).toList(), contains("a")); assertThat(results.columns().stream().map(ColumnInfoImpl::type).toList(), contains(DataType.NULL)); assertNull(getValuesList(results).getFirst().get(countIndex)); @@ -602,7 +606,7 @@ public void testStatsByNull() { try (EsqlQueryResponse results = run("row a = null | stats by a")) { logger.info(results); assertEquals(1, getValuesList(results).size()); - int countIndex = results.columns().indexOf(new ColumnInfoImpl("a", "null")); + int countIndex = results.columns().indexOf(new ColumnInfoImpl("a", "null", null)); assertThat(results.columns().stream().map(ColumnInfo::name).toList(), contains("a")); assertThat(results.columns().stream().map(ColumnInfoImpl::type).toList(), contains(DataType.NULL)); assertNull(getValuesList(results).getFirst().get(countIndex)); @@ -613,7 +617,7 @@ public void testStringLength() { try (EsqlQueryResponse results = run("from test | eval l = length(color)")) { logger.info(results); assertThat(getValuesList(results), hasSize(40)); - int countIndex = results.columns().indexOf(new ColumnInfoImpl("l", "integer")); + int countIndex = results.columns().indexOf(new ColumnInfoImpl("l", "integer", null)); for (List values : getValuesList(results)) { assertThat((Integer) values.get(countIndex), greaterThanOrEqualTo(3)); } @@ -632,11 +636,11 @@ public void testFilterWithNullAndEvalFromIndex() { try (EsqlQueryResponse results = run("from test | eval newCount = count + 1 | where newCount > 1")) { logger.info(results); assertEquals(40, getValuesList(results).size()); - assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("count", "long")))); - assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("count_d", "double")))); - assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("data", "long")))); - assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("data_d", "double")))); - assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("time", "long")))); + assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("count", "long", null)))); + assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("count_d", "double", null)))); + assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("data", "long", null)))); + assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("data_d", "double", null)))); + assertThat(results.columns(), hasItem(equalTo(new ColumnInfoImpl("time", "long", null)))); } } @@ -670,7 +674,7 @@ public void testEvalOverride() { assertEquals(40, getValuesList(results).size()); assertEquals(1, results.columns().stream().filter(c -> c.name().equals("count")).count()); int countIndex = results.columns().size() - 1; - assertEquals(new ColumnInfoImpl("count", "long"), results.columns().get(countIndex)); + assertEquals(new ColumnInfoImpl("count", "long", null), results.columns().get(countIndex)); for (List values : getValuesList(results)) { assertThat((Long) values.get(countIndex), greaterThanOrEqualTo(42L)); } @@ -681,7 +685,7 @@ public void testProjectRename() { try (var results = run("from test | eval y = count | rename count as x | keep x, y")) { logger.info(results); assertEquals(40, getValuesList(results).size()); - assertThat(results.columns(), contains(new ColumnInfoImpl("x", "long"), new ColumnInfoImpl("y", "long"))); + assertThat(results.columns(), contains(new ColumnInfoImpl("x", "long", null), new ColumnInfoImpl("y", "long", null))); for (List values : getValuesList(results)) { assertThat((Long) values.get(0), greaterThanOrEqualTo(40L)); assertThat(values.get(1), is(values.get(0))); @@ -696,10 +700,10 @@ public void testProjectRenameEval() { assertThat( results.columns(), contains( - new ColumnInfoImpl("x", "long"), - new ColumnInfoImpl("y", "long"), - new ColumnInfoImpl("x2", "long"), - new ColumnInfoImpl("y2", "long") + new ColumnInfoImpl("x", "long", null), + new ColumnInfoImpl("y", "long", null), + new ColumnInfoImpl("x2", "long", null), + new ColumnInfoImpl("y2", "long", null) ) ); for (List values : getValuesList(results)) { @@ -717,7 +721,11 @@ public void testProjectRenameEvalProject() { assertEquals(40, getValuesList(results).size()); assertThat( results.columns(), - contains(new ColumnInfoImpl("x", "long"), new ColumnInfoImpl("y", "long"), new ColumnInfoImpl("z", "long")) + contains( + new ColumnInfoImpl("x", "long", null), + new ColumnInfoImpl("y", "long", null), + new ColumnInfoImpl("z", "long", null) + ) ); for (List values : getValuesList(results)) { assertThat((Long) values.get(0), greaterThanOrEqualTo(40L)); @@ -731,7 +739,7 @@ public void testProjectOverride() { try (var results = run("from test | eval cnt = count | rename count as data | keep cnt, data")) { logger.info(results); assertEquals(40, getValuesList(results).size()); - assertThat(results.columns(), contains(new ColumnInfoImpl("cnt", "long"), new ColumnInfoImpl("data", "long"))); + assertThat(results.columns(), contains(new ColumnInfoImpl("cnt", "long", null), new ColumnInfoImpl("data", "long", null))); for (List values : getValuesList(results)) { assertThat(values.get(1), is(values.get(0))); } @@ -901,7 +909,7 @@ public void testEvalWithNullAndAvg() { public void testFromStatsLimit() { try (EsqlQueryResponse results = run("from test | stats ac = avg(count) by data | limit 1")) { logger.info(results); - assertThat(results.columns(), contains(new ColumnInfoImpl("ac", "double"), new ColumnInfoImpl("data", "long"))); + assertThat(results.columns(), contains(new ColumnInfoImpl("ac", "double", null), new ColumnInfoImpl("data", "long", null))); assertThat(getValuesList(results), contains(anyOf(contains(42.0, 1L), contains(44.0, 2L)))); } } @@ -909,7 +917,7 @@ public void testFromStatsLimit() { public void testFromLimit() { try (EsqlQueryResponse results = run("from test | keep data | limit 2")) { logger.info(results); - assertThat(results.columns(), contains(new ColumnInfoImpl("data", "long"))); + assertThat(results.columns(), contains(new ColumnInfoImpl("data", "long", null))); assertThat(getValuesList(results), contains(anyOf(contains(1L), contains(2L)), anyOf(contains(1L), contains(2L)))); } } @@ -918,7 +926,7 @@ public void testDropAllColumns() { try (EsqlQueryResponse results = run("from test | keep data | drop data | eval a = 1")) { logger.info(results); assertThat(results.columns(), hasSize(1)); - assertThat(results.columns(), contains(new ColumnInfoImpl("a", "integer"))); + assertThat(results.columns(), contains(new ColumnInfoImpl("a", "integer", null))); assertThat(getValuesList(results), is(empty())); } } @@ -1037,7 +1045,10 @@ public void testErrorMessageForEmptyParams() { public void testEmptyIndex() { assertAcked(client().admin().indices().prepareCreate("test_empty").setMapping("k", "type=keyword", "v", "type=long").get()); try (EsqlQueryResponse results = run("from test_empty")) { - assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("k", "keyword"), new ColumnInfoImpl("v", "long")))); + assertThat( + results.columns(), + equalTo(List.of(new ColumnInfoImpl("k", "keyword", null), new ColumnInfoImpl("v", "long", null))) + ); assertThat(getValuesList(results), empty()); } } @@ -1048,9 +1059,9 @@ public void testShowInfo() { results.columns(), equalTo( List.of( - new ColumnInfoImpl("version", "keyword"), - new ColumnInfoImpl("date", "keyword"), - new ColumnInfoImpl("hash", "keyword") + new ColumnInfoImpl("version", "keyword", null), + new ColumnInfoImpl("date", "keyword", null), + new ColumnInfoImpl("hash", "keyword", null) ) ) ); @@ -1063,7 +1074,7 @@ public void testShowInfo() { public void testInWithNullValue() { try (EsqlQueryResponse results = run("from test | where null in (data, 2) | keep data")) { - assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("data", "long")))); + assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("data", "long", null)))); assertThat(getValuesList(results).size(), equalTo(0)); } } @@ -1224,7 +1235,7 @@ public void testGroupingMultiValueByOrdinals() { public void testLoadId() { try (EsqlQueryResponse results = run("from test metadata _id | keep _id | sort _id ")) { - assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("_id", "keyword")))); + assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("_id", "keyword", null)))); ListMatcher values = matchesList(); for (int i = 10; i < 50; i++) { values = values.item(List.of(Integer.toString(i))); @@ -1433,12 +1444,15 @@ public void testQueryOnEmptyMappingIndex() { try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) { assertFalse(resp.values().hasNext()); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("_source", "_source"), new ColumnInfoImpl("x", "integer")))); + assertThat( + resp.columns(), + equalTo(List.of(new ColumnInfoImpl("_source", "_source", null), new ColumnInfoImpl("x", "integer", null))) + ); } try (EsqlQueryResponse resp = run(from)) { assertFalse(resp.values().hasNext()); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("", "null")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("", "null", null)))); } } @@ -1468,9 +1482,9 @@ public void testQueryOnEmptyDataIndex() { resp.columns(), equalTo( List.of( - new ColumnInfoImpl("name", "text"), - new ColumnInfoImpl("_source", "_source"), - new ColumnInfoImpl("x", "integer") + new ColumnInfoImpl("name", "text", null), + new ColumnInfoImpl("_source", "_source", null), + new ColumnInfoImpl("x", "integer", null) ) ) ); @@ -1478,26 +1492,29 @@ public void testQueryOnEmptyDataIndex() { try (EsqlQueryResponse resp = run(from)) { assertFalse(resp.values().hasNext()); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("name", "text")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("name", "text", null)))); } } private void assertEmptyIndexQueries(String from) { try (EsqlQueryResponse resp = run(from + "METADATA _source | KEEP _source | LIMIT 1")) { assertFalse(resp.values().hasNext()); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("_source", "_source")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("_source", "_source", null)))); } try (EsqlQueryResponse resp = run(from + "| EVAL y = 1 | KEEP y | LIMIT 1 | EVAL x = 1")) { assertFalse(resp.values().hasNext()); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("y", "integer"), new ColumnInfoImpl("x", "integer")))); + assertThat( + resp.columns(), + equalTo(List.of(new ColumnInfoImpl("y", "integer", null), new ColumnInfoImpl("x", "integer", null))) + ); } try (EsqlQueryResponse resp = run(from + "| STATS c = count()")) { assertTrue(resp.values().hasNext()); Iterator row = resp.values().next(); assertThat(row.next(), equalTo((long) 0)); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("c", "long")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("c", "long", null)))); } try (EsqlQueryResponse resp = run(from + "| STATS c = count() | EVAL x = 123")) { @@ -1506,7 +1523,7 @@ private void assertEmptyIndexQueries(String from) { assertThat(row.next(), equalTo((long) 0)); assertThat(row.next(), equalTo(123)); assertFalse(row.hasNext()); - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("c", "long"), new ColumnInfoImpl("x", "integer")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("c", "long", null), new ColumnInfoImpl("x", "integer", null)))); } } @@ -1583,7 +1600,7 @@ private void createAlias(List indices, String alias) throws InterruptedE private void assertNoNestedDocuments(String query, int docsCount, long minValue, long maxValue) { try (EsqlQueryResponse results = run(query)) { - assertThat(results.columns(), contains(new ColumnInfoImpl("data", "long"))); + assertThat(results.columns(), contains(new ColumnInfoImpl("data", "long", null))); assertThat(results.columns().size(), is(1)); assertThat(getValuesList(results).size(), is(docsCount)); for (List row : getValuesList(results)) { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 77726ca9fdcce..bf89ac8710fd6 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -196,7 +196,7 @@ record RateKey(String cluster, String host) { } } try (var resp = run("METRICS hosts sum(rate(request_count, 1second))")) { - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count, 1second))", "double")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count, 1second))", "double", null)))); List> values = EsqlTestUtils.getValuesList(resp); assertThat(values, hasSize(1)); assertThat(values.get(0), hasSize(1)); @@ -207,8 +207,8 @@ record RateKey(String cluster, String host) { resp.columns(), equalTo( List.of( - new ColumnInfoImpl("max(rate(request_count))", "double"), - new ColumnInfoImpl("min(rate(request_count))", "double") + new ColumnInfoImpl("max(rate(request_count))", "double", null), + new ColumnInfoImpl("min(rate(request_count))", "double", null) ) ) ); @@ -223,9 +223,9 @@ record RateKey(String cluster, String host) { resp.columns(), equalTo( List.of( - new ColumnInfoImpl("max(rate(request_count))", "double"), - new ColumnInfoImpl("avg(rate(request_count))", "double"), - new ColumnInfoImpl("max(rate(request_count, 1minute))", "double") + new ColumnInfoImpl("max(rate(request_count))", "double", null), + new ColumnInfoImpl("avg(rate(request_count))", "double", null), + new ColumnInfoImpl("max(rate(request_count, 1minute))", "double", null) ) ) ); @@ -242,8 +242,8 @@ record RateKey(String cluster, String host) { resp.columns(), equalTo( List.of( - new ColumnInfoImpl("avg(rate(request_count))", "double"), - new ColumnInfoImpl("avg(rate(request_count, 1second))", "double") + new ColumnInfoImpl("avg(rate(request_count))", "double", null), + new ColumnInfoImpl("avg(rate(request_count, 1second))", "double", null) ) ) ); @@ -259,10 +259,10 @@ record RateKey(String cluster, String host) { resp.columns(), equalTo( List.of( - new ColumnInfoImpl("max(rate(request_count))", "double"), - new ColumnInfoImpl("min(rate(request_count))", "double"), - new ColumnInfoImpl("min(cpu)", "double"), - new ColumnInfoImpl("max(cpu)", "double") + new ColumnInfoImpl("max(rate(request_count))", "double", null), + new ColumnInfoImpl("min(rate(request_count))", "double", null), + new ColumnInfoImpl("min(cpu)", "double", null), + new ColumnInfoImpl("max(cpu)", "double", null) ) ) ); @@ -297,7 +297,9 @@ record RateKey(String cluster, String host) { try (var resp = run("METRICS hosts sum(rate(request_count)) BY cluster | SORT cluster")) { assertThat( resp.columns(), - equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double"), new ColumnInfoImpl("cluster", "keyword"))) + equalTo( + List.of(new ColumnInfoImpl("sum(rate(request_count))", "double", null), new ColumnInfoImpl("cluster", "keyword", null)) + ) ); List> values = EsqlTestUtils.getValuesList(resp); assertThat(values, hasSize(bucketToRates.size())); @@ -312,7 +314,9 @@ record RateKey(String cluster, String host) { try (var resp = run("METRICS hosts avg(rate(request_count)) BY cluster | SORT cluster")) { assertThat( resp.columns(), - equalTo(List.of(new ColumnInfoImpl("avg(rate(request_count))", "double"), new ColumnInfoImpl("cluster", "keyword"))) + equalTo( + List.of(new ColumnInfoImpl("avg(rate(request_count))", "double", null), new ColumnInfoImpl("cluster", "keyword", null)) + ) ); List> values = EsqlTestUtils.getValuesList(resp); assertThat(values, hasSize(bucketToRates.size())); @@ -335,9 +339,9 @@ record RateKey(String cluster, String host) { resp.columns(), equalTo( List.of( - new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double"), - new ColumnInfoImpl("avg(rate(request_count))", "double"), - new ColumnInfoImpl("cluster", "keyword") + new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double", null), + new ColumnInfoImpl("avg(rate(request_count))", "double", null), + new ColumnInfoImpl("cluster", "keyword", null) ) ) ); @@ -381,7 +385,7 @@ record RateKey(String host, String cluster, long interval) {} try (var resp = run("METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute) | SORT ts | LIMIT 5")) { assertThat( resp.columns(), - equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double"), new ColumnInfoImpl("ts", "date"))) + equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double", null), new ColumnInfoImpl("ts", "date", null))) ); List> values = EsqlTestUtils.getValuesList(resp); assertThat(values, hasSize(sortedKeys.size())); @@ -401,7 +405,7 @@ record RateKey(String host, String cluster, long interval) {} try (var resp = run("METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute) | SORT ts | LIMIT 5")) { assertThat( resp.columns(), - equalTo(List.of(new ColumnInfoImpl("avg(rate(request_count))", "double"), new ColumnInfoImpl("ts", "date"))) + equalTo(List.of(new ColumnInfoImpl("avg(rate(request_count))", "double", null), new ColumnInfoImpl("ts", "date", null))) ); List> values = EsqlTestUtils.getValuesList(resp); assertThat(values, hasSize(sortedKeys.size())); @@ -428,9 +432,9 @@ METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts= resp.columns(), equalTo( List.of( - new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double"), - new ColumnInfoImpl("avg(rate(request_count))", "double"), - new ColumnInfoImpl("ts", "date") + new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double", null), + new ColumnInfoImpl("avg(rate(request_count))", "double", null), + new ColumnInfoImpl("ts", "date", null) ) ) ); @@ -490,9 +494,9 @@ METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute), clust resp.columns(), equalTo( List.of( - new ColumnInfoImpl("sum(rate(request_count))", "double"), - new ColumnInfoImpl("ts", "date"), - new ColumnInfoImpl("cluster", "keyword") + new ColumnInfoImpl("sum(rate(request_count))", "double", null), + new ColumnInfoImpl("ts", "date", null), + new ColumnInfoImpl("cluster", "keyword", null) ) ) ); @@ -520,9 +524,9 @@ METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluste resp.columns(), equalTo( List.of( - new ColumnInfoImpl("avg(rate(request_count))", "double"), - new ColumnInfoImpl("ts", "date"), - new ColumnInfoImpl("cluster", "keyword") + new ColumnInfoImpl("avg(rate(request_count))", "double", null), + new ColumnInfoImpl("ts", "date", null), + new ColumnInfoImpl("cluster", "keyword", null) ) ) ); @@ -551,10 +555,10 @@ METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts= resp.columns(), equalTo( List.of( - new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double"), - new ColumnInfoImpl("avg(rate(request_count))", "double"), - new ColumnInfoImpl("ts", "date"), - new ColumnInfoImpl("cluster", "keyword") + new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double", null), + new ColumnInfoImpl("avg(rate(request_count))", "double", null), + new ColumnInfoImpl("ts", "date", null), + new ColumnInfoImpl("cluster", "keyword", null) ) ) ); @@ -593,11 +597,11 @@ METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts= resp.columns(), equalTo( List.of( - new ColumnInfoImpl("avg_rate", "double"), - new ColumnInfoImpl("max(rate(request_count))", "double"), - new ColumnInfoImpl("avg(rate(request_count))", "double"), - new ColumnInfoImpl("ts", "date"), - new ColumnInfoImpl("cluster", "keyword") + new ColumnInfoImpl("avg_rate", "double", null), + new ColumnInfoImpl("max(rate(request_count))", "double", null), + new ColumnInfoImpl("avg(rate(request_count))", "double", null), + new ColumnInfoImpl("ts", "date", null), + new ColumnInfoImpl("cluster", "keyword", null) ) ) ); @@ -630,10 +634,10 @@ METRICS hosts sum(rate(request_count)), max(cpu) BY ts=bucket(@timestamp, 1 minu resp.columns(), equalTo( List.of( - new ColumnInfoImpl("sum(rate(request_count))", "double"), - new ColumnInfoImpl("max(cpu)", "double"), - new ColumnInfoImpl("ts", "date"), - new ColumnInfoImpl("cluster", "keyword") + new ColumnInfoImpl("sum(rate(request_count))", "double", null), + new ColumnInfoImpl("max(cpu)", "double", null), + new ColumnInfoImpl("ts", "date", null), + new ColumnInfoImpl("cluster", "keyword", null) ) ) ); @@ -667,10 +671,10 @@ METRICS hosts sum(rate(request_count)), avg(cpu) BY ts=bucket(@timestamp, 1 minu resp.columns(), equalTo( List.of( - new ColumnInfoImpl("sum(rate(request_count))", "double"), - new ColumnInfoImpl("avg(cpu)", "double"), - new ColumnInfoImpl("ts", "date"), - new ColumnInfoImpl("cluster", "keyword") + new ColumnInfoImpl("sum(rate(request_count))", "double", null), + new ColumnInfoImpl("avg(cpu)", "double", null), + new ColumnInfoImpl("ts", "date", null), + new ColumnInfoImpl("cluster", "keyword", null) ) ) ); @@ -716,14 +720,14 @@ record RateKey(String cluster, String host) { } } try (var resp = run("METRICS hosts sum(abs(rate(request_count, 1second)))")) { - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(abs(rate(request_count, 1second)))", "double")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(abs(rate(request_count, 1second)))", "double", null)))); List> values = EsqlTestUtils.getValuesList(resp); assertThat(values, hasSize(1)); assertThat(values.get(0), hasSize(1)); assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).sum(), 0.1)); } try (var resp = run("METRICS hosts sum(10.0 * rate(request_count, 1second))")) { - assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(10.0 * rate(request_count, 1second))", "double")))); + assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(10.0 * rate(request_count, 1second))", "double", null)))); List> values = EsqlTestUtils.getValuesList(resp); assertThat(values, hasSize(1)); assertThat(values.get(0), hasSize(1)); @@ -734,7 +738,11 @@ record RateKey(String cluster, String host) { resp.columns(), equalTo( List.of( - new ColumnInfoImpl("sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))", "double") + new ColumnInfoImpl( + "sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))", + "double", + null + ) ) ) ); @@ -753,8 +761,8 @@ public void testIndexMode() { } refresh("events"); List columns = List.of( - new ColumnInfoImpl("_index", DataType.KEYWORD), - new ColumnInfoImpl("_index_mode", DataType.KEYWORD) + new ColumnInfoImpl("_index", DataType.KEYWORD, null), + new ColumnInfoImpl("_index_mode", DataType.KEYWORD, null) ); try (EsqlQueryResponse resp = run(""" FROM events,hosts METADATA _index_mode, _index diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java index 94da383b40957..33b87b7de1531 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java @@ -7,8 +7,10 @@ package org.elasticsearch.xpack.esql.action; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.InstantiatingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ParserConstructor; @@ -19,9 +21,13 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public class ColumnInfoImpl implements ColumnInfo { @@ -34,6 +40,7 @@ public class ColumnInfoImpl implements ColumnInfo { ); parser.declareString(constructorArg(), new ParseField("name")); parser.declareString(constructorArg(), new ParseField("type")); + parser.declareStringArray(optionalConstructorArg(), new ParseField("original_types")); PARSER = parser.build(); } @@ -43,41 +50,58 @@ public boolean equals(Object o) { return true; } if ((o instanceof ColumnInfoImpl that)) { - return Objects.equals(name, that.name) && Objects.equals(type, that.type); + return Objects.equals(name, that.name) && Objects.equals(type, that.type) && Objects.equals(originalTypes, that.originalTypes); } return false; } @Override public int hashCode() { - return Objects.hash(name, type); + return Objects.hash(name, type, originalTypes); } public static ColumnInfo fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } - private String name; - private DataType type; + private final String name; + private final DataType type; + /** + * If this field is unsupported this contains the underlying ES types. If there + * is a type conflict this will have many elements, some or all of which may + * be actually supported types. + */ + @Nullable + private final List originalTypes; @ParserConstructor - public ColumnInfoImpl(String name, String type) { - this(name, DataType.fromEs(type)); + public ColumnInfoImpl(String name, String type, @Nullable List originalTypes) { + this(name, DataType.fromEs(type), originalTypes); } - public ColumnInfoImpl(String name, DataType type) { + public ColumnInfoImpl(String name, DataType type, @Nullable List originalTypes) { this.name = name; this.type = type; + this.originalTypes = originalTypes; } public ColumnInfoImpl(StreamInput in) throws IOException { - this(in.readString(), in.readString()); + this.name = in.readString(); + this.type = DataType.fromEs(in.readString()); + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REPORT_ORIGINAL_TYPES)) { + this.originalTypes = in.readOptionalStringCollectionAsList(); + } else { + this.originalTypes = null; + } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeString(type.outputType()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_REPORT_ORIGINAL_TYPES)) { + out.writeOptionalStringCollection(originalTypes); + } } @Override @@ -85,6 +109,9 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.startObject(); builder.field("name", name); builder.field("type", type.outputType()); + if (originalTypes != null) { + builder.field("original_types", originalTypes); + } builder.endObject(); return builder; } @@ -102,4 +129,9 @@ public String outputType() { public DataType type() { return type; } + + @Nullable + public List originalTypes() { + return originalTypes; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 276e9161d6482..873f812b5fcc2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -886,7 +886,12 @@ public enum Cap { /** * Do {@code TO_LOWER} and {@code TO_UPPER} process all field values? */ - TO_LOWER_MV; + TO_LOWER_MV, + + /** + * The {@code _query} API now reports the original types. + */ + REPORT_ORIGINAL_TYPES; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 93e9ede80b420..bc3b6a0f9c32e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1653,7 +1653,7 @@ public LogicalPlan apply(LogicalPlan plan) { static Attribute checkUnresolved(FieldAttribute fa) { if (fa.field() instanceof InvalidMappedField imf) { String unresolvedMessage = "Cannot use field [" + fa.name() + "] due to ambiguities being " + imf.errorMessage(); - String types = imf.getTypesToIndices().keySet().stream().collect(Collectors.joining(",")); + List types = imf.getTypesToIndices().keySet().stream().toList(); return new UnsupportedAttribute( fa.source(), fa.name(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java index 089f6db373c54..850b909872635 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; +import java.util.List; import java.util.Objects; import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck; @@ -60,7 +61,7 @@ public final class UnsupportedAttribute extends FieldAttribute implements Unreso private final boolean hasCustomMessage; // TODO remove me and just use message != null? private static String errorMessage(String name, UnsupportedEsField field) { - return "Cannot use field [" + name + "] with unsupported type [" + field.getOriginalType() + "]"; + return "Cannot use field [" + name + "] with unsupported type [" + String.join(",", field.getOriginalTypes()) + "]"; } public UnsupportedAttribute(Source source, String name, UnsupportedEsField field) { @@ -174,4 +175,9 @@ public boolean equals(Object obj) { } return false; } + + @Override + public List originalTypes() { + return field().getOriginalTypes(); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 65562b9e65c27..135d6f342039c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -55,6 +55,9 @@ import java.io.IOException; import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -314,7 +317,19 @@ private EsqlExecutionInfo createEsqlExecutionInfo(EsqlQueryRequest request) { } private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Configuration configuration, Result result) { - List columns = result.schema().stream().map(c -> new ColumnInfoImpl(c.name(), c.dataType().outputType())).toList(); + List columns = result.schema() + .stream() + .map(c -> { + List originalTypes; + if (c.originalTypes() == null) { + originalTypes = null; + } else { + originalTypes = new ArrayList<>(c.originalTypes()); + Collections.sort(originalTypes); + } + return new ColumnInfoImpl(c.name(), c.dataType().outputType(), originalTypes); + }) + .toList(); EsqlQueryResponse.Profile profile = configuration.profile() ? new EsqlQueryResponse.Profile(result.profiles()) : null; threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_IS_RUNNING_HEADER, "?0"); if (task instanceof EsqlQueryTask asyncTask && request.keepOnCompletion()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index d0d35374242e7..2b937a80214ed 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -136,7 +136,7 @@ public static IndexResolution mergedMappings(String indexPattern, FieldCapabilit ? createField(fieldCapsResponse, name, fullName, fcs, isAlias) : new UnsupportedEsField( fullName, - firstUnsupportedParent.getOriginalType(), + firstUnsupportedParent.getOriginalTypes(), firstUnsupportedParent.getName(), new HashMap<>() ); @@ -235,7 +235,7 @@ private static EsField createField( private static UnsupportedEsField unsupported(String name, IndexFieldCapabilities fc) { String originalType = fc.metricType() == TimeSeriesParams.MetricType.COUNTER ? "counter" : fc.type(); - return new UnsupportedEsField(name, originalType); + return new UnsupportedEsField(name, List.of(originalType)); } private static EsField conflictingTypes(String name, String fullName, FieldCapabilitiesResponse fieldCapsResponse) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 95b0abc31062f..5dc913af74165 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import org.elasticsearch.xpack.esql.type.UnsupportedEsFieldTests; import org.elasticsearch.xpack.versionfield.Version; import org.junit.After; import org.junit.Before; @@ -176,7 +177,12 @@ private ColumnInfoImpl randomColumnInfo() { || t == DataType.AGGREGATE_METRIC_DOUBLE, () -> randomFrom(DataType.types()) ).widenSmallNumeric(); - return new ColumnInfoImpl(randomAlphaOfLength(10), type.esType()); + return new ColumnInfoImpl(randomAlphaOfLength(10), type.esType(), randomOriginalTypes()); + } + + @Nullable + public static List randomOriginalTypes() { + return randomBoolean() ? null : UnsupportedEsFieldTests.randomOriginalTypes(); } private EsqlQueryResponse.Profile randomProfile() { @@ -254,7 +260,10 @@ protected EsqlQueryResponse mutateInstance(EsqlQueryResponse instance) { int mutCol = between(0, instance.columns().size() - 1); List cols = new ArrayList<>(instance.columns()); // keep the type the same so the values are still valid but change the name - cols.set(mutCol, new ColumnInfoImpl(cols.get(mutCol).name() + "mut", cols.get(mutCol).type())); + cols.set( + mutCol, + new ColumnInfoImpl(cols.get(mutCol).name() + "mut", cols.get(mutCol).type(), cols.get(mutCol).originalTypes()) + ); yield new EsqlQueryResponse( cols, deepCopyOfPages(instance), @@ -627,7 +636,7 @@ public void testSimpleXContentRowsAsync() { public void testBasicXContentIdAndRunning() { try ( EsqlQueryResponse response = new EsqlQueryResponse( - List.of(new ColumnInfoImpl("foo", "integer")), + List.of(new ColumnInfoImpl("foo", "integer", null)), List.of(new Page(blockFactory.newIntArrayVector(new int[] { 40, 80 }, 2).asBlock())), null, false, @@ -642,10 +651,28 @@ public void testBasicXContentIdAndRunning() { } } + public void testXContentOriginalTypes() { + try ( + EsqlQueryResponse response = new EsqlQueryResponse( + List.of(new ColumnInfoImpl("foo", "unsupported", List.of("foo", "bar"))), + List.of(new Page(blockFactory.newConstantNullBlock(2))), + null, + false, + null, + false, + false, + null + ) + ) { + assertThat(Strings.toString(response), equalTo(""" + {"columns":[{"name":"foo","type":"unsupported","original_types":["foo","bar"]}],"values":[[null],[null]]}""")); + } + } + public void testNullColumnsXContentDropNulls() { try ( EsqlQueryResponse response = new EsqlQueryResponse( - List.of(new ColumnInfoImpl("foo", "integer"), new ColumnInfoImpl("all_null", "integer")), + List.of(new ColumnInfoImpl("foo", "integer", null), new ColumnInfoImpl("all_null", "integer", null)), List.of(new Page(blockFactory.newIntArrayVector(new int[] { 40, 80 }, 2).asBlock(), blockFactory.newConstantNullBlock(2))), null, false, @@ -675,7 +702,7 @@ public void testNullColumnsFromBuilderXContentDropNulls() { b.appendNull(); try ( EsqlQueryResponse response = new EsqlQueryResponse( - List.of(new ColumnInfoImpl("foo", "integer"), new ColumnInfoImpl("all_null", "integer")), + List.of(new ColumnInfoImpl("foo", "integer", null), new ColumnInfoImpl("all_null", "integer", null)), List.of(new Page(blockFactory.newIntArrayVector(new int[] { 40, 80 }, 2).asBlock(), b.build())), null, false, @@ -702,7 +729,7 @@ private EsqlQueryResponse simple(boolean columnar) { private EsqlQueryResponse simple(boolean columnar, boolean async) { return new EsqlQueryResponse( - List.of(new ColumnInfoImpl("foo", "integer")), + List.of(new ColumnInfoImpl("foo", "integer", null)), List.of(new Page(blockFactory.newIntArrayVector(new int[] { 40, 80 }, 2).asBlock())), null, columnar, @@ -714,7 +741,7 @@ private EsqlQueryResponse simple(boolean columnar, boolean async) { public void testProfileXContent() { try ( EsqlQueryResponse response = new EsqlQueryResponse( - List.of(new ColumnInfoImpl("foo", "integer")), + List.of(new ColumnInfoImpl("foo", "integer", null)), List.of(new Page(blockFactory.newIntArrayVector(new int[] { 40, 80 }, 2).asBlock())), new EsqlQueryResponse.Profile( List.of( @@ -798,7 +825,7 @@ public void testColumns() { var intBlk2 = blockFactory.newIntArrayVector(new int[] { 30, 40, 50 }, 3).asBlock(); var longBlk1 = blockFactory.newLongArrayVector(new long[] { 100L, 200L }, 2).asBlock(); var longBlk2 = blockFactory.newLongArrayVector(new long[] { 300L, 400L, 500L }, 3).asBlock(); - var columnInfo = List.of(new ColumnInfoImpl("foo", "integer"), new ColumnInfoImpl("bar", "long")); + var columnInfo = List.of(new ColumnInfoImpl("foo", "integer", null), new ColumnInfoImpl("bar", "long", null)); var pages = List.of(new Page(intBlk1, longBlk1), new Page(intBlk2, longBlk2)); try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) { assertThat(columnValues(response.column(0)), contains(10, 20, 30, 40, 50)); @@ -810,7 +837,7 @@ public void testColumns() { public void testColumnsIllegalArg() { var intBlk1 = blockFactory.newIntArrayVector(new int[] { 10 }, 1).asBlock(); - var columnInfo = List.of(new ColumnInfoImpl("foo", "integer")); + var columnInfo = List.of(new ColumnInfoImpl("foo", "integer", null)); var pages = List.of(new Page(intBlk1)); try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) { expectThrows(IllegalArgumentException.class, () -> response.column(-1)); @@ -829,7 +856,7 @@ public void testColumnsWithNull() { blk2 = bb2.appendInt(30).appendNull().appendNull().appendInt(60).build(); blk3 = bb3.appendNull().appendInt(80).appendInt(90).appendNull().build(); } - var columnInfo = List.of(new ColumnInfoImpl("foo", "integer")); + var columnInfo = List.of(new ColumnInfoImpl("foo", "integer", null)); var pages = List.of(new Page(blk1), new Page(blk2), new Page(blk3)); try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) { assertThat(columnValues(response.column(0)), contains(10, null, 30, null, null, 60, null, 80, 90, null)); @@ -849,7 +876,7 @@ public void testColumnsWithMultiValue() { blk2 = bb2.beginPositionEntry().appendInt(40).appendInt(50).endPositionEntry().build(); blk3 = bb3.appendNull().appendInt(70).appendInt(80).appendNull().build(); } - var columnInfo = List.of(new ColumnInfoImpl("foo", "integer")); + var columnInfo = List.of(new ColumnInfoImpl("foo", "integer", null)); var pages = List.of(new Page(blk1), new Page(blk2), new Page(blk3)); try (var response = new EsqlQueryResponse(columnInfo, pages, null, false, null, false, false, null)) { assertThat(columnValues(response.column(0)), contains(List.of(10, 20), null, List.of(40, 50), null, 70, 80, null)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index db7f7b4adfa8b..00aa2773c1684 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -85,7 +85,7 @@ public void testUnsupportedAndMultiTypedFields() { final String unsupported = "unsupported"; final String multiTyped = "multi_typed"; - EsField unsupportedField = new UnsupportedEsField(unsupported, "flattened"); + EsField unsupportedField = new UnsupportedEsField(unsupported, List.of("flattened")); // Use linked maps/sets to fix the order in the error message. LinkedHashSet ipIndices = new LinkedHashSet<>(); ipIndices.add("test1"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java index f3746db2b38a6..723f6c89f62cd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java @@ -269,18 +269,18 @@ public void testTsvFormatWithDropNullColumns() { } private static EsqlQueryResponse emptyData() { - return new EsqlQueryResponse(singletonList(new ColumnInfoImpl("name", "keyword")), emptyList(), null, false, false, null); + return new EsqlQueryResponse(singletonList(new ColumnInfoImpl("name", "keyword", null)), emptyList(), null, false, false, null); } private static EsqlQueryResponse regularData() { BlockFactory blockFactory = TestBlockFactory.getNonBreakingInstance(); // headers List headers = asList( - new ColumnInfoImpl("string", "keyword"), - new ColumnInfoImpl("number", "integer"), - new ColumnInfoImpl("location", "geo_point"), - new ColumnInfoImpl("location2", "cartesian_point"), - new ColumnInfoImpl("null_field", "keyword") + new ColumnInfoImpl("string", "keyword", null), + new ColumnInfoImpl("number", "integer", null), + new ColumnInfoImpl("location", "geo_point", null), + new ColumnInfoImpl("location2", "cartesian_point", null), + new ColumnInfoImpl("null_field", "keyword", null) ); BytesRefArray geoPoints = new BytesRefArray(2, BigArrays.NON_RECYCLING_INSTANCE); @@ -308,7 +308,10 @@ private static EsqlQueryResponse regularData() { private static EsqlQueryResponse escapedData() { // headers - List headers = asList(new ColumnInfoImpl("first", "keyword"), new ColumnInfoImpl("\"special\"", "keyword")); + List headers = asList( + new ColumnInfoImpl("first", "keyword", null), + new ColumnInfoImpl("\"special\"", "keyword", null) + ); // values List values = List.of( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java index 6ca63dfb84f37..ec9bb14d2a265 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java @@ -34,16 +34,16 @@ public class TextFormatterTests extends ESTestCase { static BlockFactory blockFactory = TestBlockFactory.getNonBreakingInstance(); private final List columns = Arrays.asList( - new ColumnInfoImpl("foo", "keyword"), - new ColumnInfoImpl("bar", "long"), - new ColumnInfoImpl("15charwidename!", "double"), - new ColumnInfoImpl("null_field1", "integer"), - new ColumnInfoImpl("superduperwidename!!!", "double"), - new ColumnInfoImpl("baz", "keyword"), - new ColumnInfoImpl("date", "date"), - new ColumnInfoImpl("location", "geo_point"), - new ColumnInfoImpl("location2", "cartesian_point"), - new ColumnInfoImpl("null_field2", "keyword") + new ColumnInfoImpl("foo", "keyword", null), + new ColumnInfoImpl("bar", "long", null), + new ColumnInfoImpl("15charwidename!", "double", null), + new ColumnInfoImpl("null_field1", "integer", null), + new ColumnInfoImpl("superduperwidename!!!", "double", null), + new ColumnInfoImpl("baz", "keyword", null), + new ColumnInfoImpl("date", "date", null), + new ColumnInfoImpl("location", "geo_point", null), + new ColumnInfoImpl("location2", "cartesian_point", null), + new ColumnInfoImpl("null_field2", "keyword", null) ); private static final BytesRefArray geoPoints = new BytesRefArray(2, BigArrays.NON_RECYCLING_INSTANCE); @@ -213,7 +213,7 @@ public void testVeryLongPadding() { getTextBodyContent( new TextFormatter( new EsqlQueryResponse( - List.of(new ColumnInfoImpl("foo", "keyword")), + List.of(new ColumnInfoImpl("foo", "keyword", null)), List.of( new Page( blockFactory.newBytesRefBlockBuilder(2) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/UnsupportedEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/UnsupportedEsFieldTests.java index a89ca9481b7e1..908cdb279613c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/UnsupportedEsFieldTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/UnsupportedEsFieldTests.java @@ -10,15 +10,20 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; +import java.util.List; import java.util.Map; public class UnsupportedEsFieldTests extends AbstractEsFieldTypeTests { public static UnsupportedEsField randomUnsupportedEsField(int maxPropertiesDepth) { String name = randomAlphaOfLength(4); - String originalType = randomAlphaOfLength(5); + List originalTypes = randomOriginalTypes(); String inherited = randomBoolean() ? null : randomAlphaOfLength(5); Map properties = randomProperties(maxPropertiesDepth); - return new UnsupportedEsField(name, originalType, inherited, properties); + return new UnsupportedEsField(name, originalTypes, inherited, properties); + } + + public static List randomOriginalTypes() { + return randomBoolean() ? List.of(randomAlphaOfLength(5)) : randomList(4, 4, () -> randomAlphaOfLength(5)); } @Override @@ -29,16 +34,16 @@ protected UnsupportedEsField createTestInstance() { @Override protected UnsupportedEsField mutate(UnsupportedEsField instance) { String name = instance.getName(); - String originalType = randomAlphaOfLength(5); - String inherited = randomBoolean() ? null : randomAlphaOfLength(5); + List originalTypes = instance.getOriginalTypes(); + String inherited = instance.getInherited(); Map properties = instance.getProperties(); switch (between(0, 3)) { case 0 -> name = randomAlphaOfLength(name.length() + 1); - case 1 -> originalType = randomValueOtherThan(originalType, () -> randomAlphaOfLength(4)); + case 1 -> originalTypes = randomValueOtherThan(originalTypes, UnsupportedEsFieldTests::randomOriginalTypes); case 2 -> inherited = randomValueOtherThan(inherited, () -> randomBoolean() ? null : randomAlphaOfLength(4)); case 3 -> properties = randomValueOtherThan(properties, () -> randomProperties(4)); default -> throw new IllegalArgumentException(); } - return new UnsupportedEsField(name, originalType, inherited, properties); + return new UnsupportedEsField(name, originalTypes, inherited, properties); } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml index 359ac40bc3672..c477a8b171074 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml @@ -276,8 +276,10 @@ load two indices, showing unsupported type and null value for event_duration: - match: { columns.0.type: "date" } - match: { columns.1.name: "client_ip" } - match: { columns.1.type: "ip" } + - is_false: columns.1.original_types - match: { columns.2.name: "event_duration" } - match: { columns.2.type: "unsupported" } + - match: { columns.2.original_types: ["keyword", "long"] } - match: { columns.3.name: "message" } - match: { columns.3.type: "keyword" } - match: { columns.4.name: "_index" } @@ -318,7 +320,9 @@ load two indices with no conversion function, but needs TO_LONG conversion: - match: { columns.1.type: "date" } - match: { columns.2.name: "client_ip" } - match: { columns.2.type: "ip" } + - is_false: columns.2.original_types - match: { columns.3.name: "event_duration" } + - match: { columns.3.original_types: ["keyword", "long"] } - match: { columns.3.type: "unsupported" } - match: { columns.4.name: "message" } - match: { columns.4.type: "keyword" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index 2a73a0efb588a..ed0f1a1b63230 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -110,17 +110,13 @@ setup: --- unsupported: - requires: - cluster_features: ["gte_v8.13.0"] - reason: "Latest types supported in ESQL as of 8.13.0" - - - requires: - test_runner_features: [ capabilities ] + test_runner_features: [capabilities, contains] capabilities: - method: POST path: /_query - parameters: [ ] - capabilities: [ aggregate_metric_double_rendering ] - reason: "support for rendering aggregate_metric_double type" + parameters: [] + capabilities: [REPORT_ORIGINAL_TYPES] + reason: "uses original_type" - do: allowed_warnings_regex: @@ -132,10 +128,13 @@ unsupported: - match: { columns.0.name: aggregate_metric_double } - match: { columns.0.type: aggregate_metric_double } + - is_false: columns.0.original_types - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } + - match: { columns.1.original_types: [binary] } - match: { columns.2.name: completion } - match: { columns.2.type: unsupported } + - match: { columns.2.original_types: [completion] } - match: { columns.3.name: date_nanos } - match: { columns.3.type: date_nanos } - match: { columns.4.name: date_range } @@ -228,10 +227,13 @@ unsupported: query: 'from test | limit 0' - match: { columns.0.name: aggregate_metric_double } - match: { columns.0.type: aggregate_metric_double } + - is_false: columns.0.original_types - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } + - match: { columns.1.original_types: [binary] } - match: { columns.2.name: completion } - match: { columns.2.type: unsupported } + - match: { columns.2.original_types: [completion] } - match: { columns.3.name: date_nanos } - match: { columns.3.type: date_nanos } - match: { columns.4.name: date_range } From 2f35b4353493a27e6c640aeda6870e418401d17c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 14 Mar 2025 14:57:03 -0400 Subject: [PATCH 2/7] Update docs/changelog/124913.yaml --- docs/changelog/124913.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/124913.yaml diff --git a/docs/changelog/124913.yaml b/docs/changelog/124913.yaml new file mode 100644 index 0000000000000..3b8f6c24b096d --- /dev/null +++ b/docs/changelog/124913.yaml @@ -0,0 +1,5 @@ +pr: 124913 +summary: Report `original_types` +area: ES|QL +type: enhancement +issues: [] From 351e15b1b6150e2f5bab2dad4f600de98b7b5d4f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 14 Mar 2025 19:04:26 +0000 Subject: [PATCH 3/7] [CI] Auto commit changes from spotless --- .../xpack/esql/action/ColumnInfoImpl.java | 2 -- .../esql/plugin/TransportEsqlQueryAction.java | 24 ++++++++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java index 33b87b7de1531..b37c1a30ae0a3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java @@ -21,8 +21,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 135d6f342039c..1e7b150462902 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -56,7 +56,6 @@ import java.io.IOException; import java.time.ZoneOffset; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -317,19 +316,16 @@ private EsqlExecutionInfo createEsqlExecutionInfo(EsqlQueryRequest request) { } private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Configuration configuration, Result result) { - List columns = result.schema() - .stream() - .map(c -> { - List originalTypes; - if (c.originalTypes() == null) { - originalTypes = null; - } else { - originalTypes = new ArrayList<>(c.originalTypes()); - Collections.sort(originalTypes); - } - return new ColumnInfoImpl(c.name(), c.dataType().outputType(), originalTypes); - }) - .toList(); + List columns = result.schema().stream().map(c -> { + List originalTypes; + if (c.originalTypes() == null) { + originalTypes = null; + } else { + originalTypes = new ArrayList<>(c.originalTypes()); + Collections.sort(originalTypes); + } + return new ColumnInfoImpl(c.name(), c.dataType().outputType(), originalTypes); + }).toList(); EsqlQueryResponse.Profile profile = configuration.profile() ? new EsqlQueryResponse.Profile(result.profiles()) : null; threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_IS_RUNNING_HEADER, "?0"); if (task instanceof EsqlQueryTask asyncTask && request.keepOnCompletion()) { From d3f343083f825df76400bb136e61f8c77d2c0c43 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 14 Mar 2025 15:05:18 -0400 Subject: [PATCH 4/7] Explain --- .../xpack/esql/plugin/TransportEsqlQueryAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 135d6f342039c..e313afc7de43d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -324,6 +324,7 @@ private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Config if (c.originalTypes() == null) { originalTypes = null; } else { + // Sort the original types so they are easier to test against and prettier. originalTypes = new ArrayList<>(c.originalTypes()); Collections.sort(originalTypes); } From 46c9f72af138a5cbaf1daa35d0f730a27f8e429f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 19 Mar 2025 12:52:20 -0400 Subject: [PATCH 5/7] Fix test --- .../rest-api-spec/test/esql/160_union_types.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml index c477a8b171074..68d14e12e74ac 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml @@ -263,6 +263,15 @@ load single index ip_long stats invalid grouping: --- load two indices, showing unsupported type and null value for event_duration: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [REPORT_ORIGINAL_TYPES] + reason: "Uses original_types" + - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -304,8 +313,8 @@ load two indices with no conversion function, but needs TO_LONG conversion: - method: POST path: /_query parameters: [] - capabilities: [union_types_fix_rename_resolution] - reason: "Union type resolution fix for rename also allows direct usage of unsupported fields in KEEP" + capabilities: [REPORT_ORIGINAL_TYPES] + reason: "Uses original_types" - do: allowed_warnings_regex: From 09c515e7c6389eb02f8fb7f859af3d9e5a149bb4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 21 Mar 2025 12:37:33 -0400 Subject: [PATCH 6/7] Move --- .../xpack/esql/core}/ColumnInfoImpl.java | 2 +- .../xpack/esql/core/expression/Attribute.java | 10 ++++------ .../test/esql/qa/action/CoreEsqlActionIT.java | 2 +- .../esql/action/AsyncEsqlQueryActionIT.java | 1 + .../esql/action/CrossClusterAsyncQueryIT.java | 1 + ...CrossClusterEnrichUnavailableClustersIT.java | 1 + .../CrossClusterQueryUnavailableRemotesIT.java | 1 + .../xpack/esql/action/EnrichIT.java | 1 + .../xpack/esql/action/EsqlActionIT.java | 1 + .../xpack/esql/action/TimeSeriesIT.java | 1 + .../xpack/esql/action/EsqlQueryResponse.java | 1 + .../xpack/esql/action/PositionToXContent.java | 1 + .../esql/action/ResponseXContentUtils.java | 1 + .../function/UnsupportedAttribute.java | 13 +++++++++++-- .../esql/plugin/TransportEsqlQueryAction.java | 17 +++-------------- .../esql/action/EsqlQueryResponseTests.java | 1 + .../xpack/esql/formatter/TextFormatTests.java | 2 +- .../esql/formatter/TextFormatterTests.java | 2 +- 18 files changed, 33 insertions(+), 26 deletions(-) rename x-pack/plugin/{esql/src/main/java/org/elasticsearch/xpack/esql/action => esql-core/src/main/java/org/elasticsearch/xpack/esql/core}/ColumnInfoImpl.java (99%) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ColumnInfoImpl.java similarity index 99% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java rename to x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ColumnInfoImpl.java index b37c1a30ae0a3..4dc43223a4cff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ColumnInfoImpl.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.action; +package org.elasticsearch.xpack.esql.core; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java index d419f7368a72d..7f0800644e13a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.core.expression; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -139,12 +140,9 @@ public String nodeString() { protected abstract String label(); /** - * If this field is unsupported this contains the underlying ES types. If there - * is a type conflict this will have many elements, some or all of which may - * be actually supported types. + * The description of this column in the HTTP response. */ - @Nullable - public List originalTypes() { - return null; + public ColumnInfoImpl columnInfo() { + return new ColumnInfoImpl(name(), dataType().outputType(), null); } } diff --git a/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java b/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java index 46fff385b5398..7b6543427a981 100644 --- a/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java +++ b/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.esql.action.EsqlQueryRequestBuilder; import org.elasticsearch.xpack.core.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.core.esql.action.EsqlResponse; -import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java index 41bac7f09df77..39a8b8c66d656 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import org.elasticsearch.xpack.core.async.TransportDeleteAsyncResultAction; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.hamcrest.core.IsEqual; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java index cb404c0c93006..f9a555e47096a 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.elasticsearch.xpack.core.async.AsyncStopRequest; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.io.IOException; import java.util.Map; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java index 40ea21371e513..3c1b113c1e8d7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.plan.logical.Enrich; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java index 667ac23461000..521a64132efa6 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.core.Tuple; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import java.util.List; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index 19157b636dffc..0a36c686ee8e1 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -47,6 +47,7 @@ import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction; import org.elasticsearch.xpack.enrich.EnrichPlugin; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.plan.logical.Enrich; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index db11757097507..604f1cca352de 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.esql.action.ColumnInfo; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 0e692ca3e053a..982c449c926fc 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 5e36eee7364c7..9107f2553e6f2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -26,6 +26,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.core.esql.action.EsqlResponse; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 116ff588f417a..e4f9191c24fcc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -22,6 +22,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java index eb1f4f95db6fe..e883a7dfca768 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.core.esql.action.ColumnInfo; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.util.Collections; import java.util.Iterator; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java index 850b909872635..2e45377634a5c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -28,6 +29,8 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -177,7 +180,13 @@ public boolean equals(Object obj) { } @Override - public List originalTypes() { - return field().getOriginalTypes(); + public ColumnInfoImpl columnInfo() { + List originalTypes = field().getOriginalTypes(); + if (originalTypes != null) { + // Sort the original types so they are easier to test against and prettier. + originalTypes = new ArrayList<>(originalTypes); + Collections.sort(originalTypes); + } + return new ColumnInfoImpl(name(), dataType().outputType(), originalTypes); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 72ca465f647b7..b1cbfa9011e4d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -36,13 +36,14 @@ import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.elasticsearch.xpack.esql.VerificationException; -import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.action.EsqlQueryTask; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.async.AsyncTaskManagementService; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.AbstractLookupService; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; @@ -55,8 +56,6 @@ import java.io.IOException; import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -316,17 +315,7 @@ private EsqlExecutionInfo createEsqlExecutionInfo(EsqlQueryRequest request) { } private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Configuration configuration, Result result) { - List columns = result.schema().stream().map(c -> { - List originalTypes; - if (c.originalTypes() == null) { - originalTypes = null; - } else { - // Sort the original types so they are easier to test against and prettier. - originalTypes = new ArrayList<>(c.originalTypes()); - Collections.sort(originalTypes); - } - return new ColumnInfoImpl(c.name(), c.dataType().outputType(), originalTypes); - }).toList(); + List columns = result.schema().stream().map(Attribute::columnInfo).toList(); EsqlQueryResponse.Profile profile = configuration.profile() ? new EsqlQueryResponse.Profile(result.profiles()) : null; threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_IS_RUNNING_HEADER, "?0"); if (task instanceof EsqlQueryTask asyncTask && request.keepOnCompletion()) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 5dc913af74165..c04858852e7f5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.UnsupportedEsFieldTests; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java index 723f6c89f62cd..054aa9639bd53 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java @@ -18,8 +18,8 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.util.StringUtils; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java index ec9bb14d2a265..f7db448e818b1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java @@ -16,9 +16,9 @@ import org.elasticsearch.compute.test.TestBlockFactory; import org.elasticsearch.geometry.Point; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; +import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.util.Arrays; import java.util.List; From 9ac0446b9027fac5f4ef1ae6e107a03f4593c221 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 25 Mar 2025 08:45:13 -0400 Subject: [PATCH 7/7] Revert "Move" This reverts commit 09c515e7c6389eb02f8fb7f859af3d9e5a149bb4. --- .../xpack/esql/core/expression/Attribute.java | 10 ++++++---- .../test/esql/qa/action/CoreEsqlActionIT.java | 2 +- .../esql/action/AsyncEsqlQueryActionIT.java | 1 - .../esql/action/CrossClusterAsyncQueryIT.java | 1 - ...CrossClusterEnrichUnavailableClustersIT.java | 1 - .../CrossClusterQueryUnavailableRemotesIT.java | 1 - .../xpack/esql/action/EnrichIT.java | 1 - .../xpack/esql/action/EsqlActionIT.java | 1 - .../xpack/esql/action/TimeSeriesIT.java | 1 - .../xpack/esql/action}/ColumnInfoImpl.java | 2 +- .../xpack/esql/action/EsqlQueryResponse.java | 1 - .../xpack/esql/action/PositionToXContent.java | 1 - .../esql/action/ResponseXContentUtils.java | 1 - .../function/UnsupportedAttribute.java | 13 ++----------- .../esql/plugin/TransportEsqlQueryAction.java | 17 ++++++++++++++--- .../esql/action/EsqlQueryResponseTests.java | 1 - .../xpack/esql/formatter/TextFormatTests.java | 2 +- .../esql/formatter/TextFormatterTests.java | 2 +- 18 files changed, 26 insertions(+), 33 deletions(-) rename x-pack/plugin/{esql-core/src/main/java/org/elasticsearch/xpack/esql/core => esql/src/main/java/org/elasticsearch/xpack/esql/action}/ColumnInfoImpl.java (99%) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java index 7f0800644e13a..d419f7368a72d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.core.expression; import org.elasticsearch.core.Nullable; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -140,9 +139,12 @@ public String nodeString() { protected abstract String label(); /** - * The description of this column in the HTTP response. + * If this field is unsupported this contains the underlying ES types. If there + * is a type conflict this will have many elements, some or all of which may + * be actually supported types. */ - public ColumnInfoImpl columnInfo() { - return new ColumnInfoImpl(name(), dataType().outputType(), null); + @Nullable + public List originalTypes() { + return null; } } diff --git a/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java b/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java index 7b6543427a981..46fff385b5398 100644 --- a/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java +++ b/x-pack/plugin/esql/qa/action/src/internalClusterTest/java/org/elasticsearch/test/esql/qa/action/CoreEsqlActionIT.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.esql.action.EsqlQueryRequestBuilder; import org.elasticsearch.xpack.core.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.core.esql.action.EsqlResponse; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; +import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java index 39a8b8c66d656..41bac7f09df77 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AsyncEsqlQueryActionIT.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.core.async.DeleteAsyncResultRequest; import org.elasticsearch.xpack.core.async.GetAsyncResultRequest; import org.elasticsearch.xpack.core.async.TransportDeleteAsyncResultAction; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.hamcrest.core.IsEqual; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java index f9a555e47096a..cb404c0c93006 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.elasticsearch.xpack.core.async.AsyncStopRequest; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.io.IOException; import java.util.Map; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java index 3c1b113c1e8d7..40ea21371e513 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterEnrichUnavailableClustersIT.java @@ -12,7 +12,6 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.transport.RemoteClusterAware; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.plan.logical.Enrich; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java index 521a64132efa6..667ac23461000 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryUnavailableRemotesIT.java @@ -9,7 +9,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.core.Tuple; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import java.util.List; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index 0a36c686ee8e1..19157b636dffc 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -47,7 +47,6 @@ import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction; import org.elasticsearch.xpack.enrich.EnrichPlugin; import org.elasticsearch.xpack.esql.EsqlTestUtils; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.plan.logical.Enrich; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index 604f1cca352de..db11757097507 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.esql.action.ColumnInfo; import org.elasticsearch.xpack.esql.VerificationException; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 982c449c926fc..0e692ca3e053a 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlTestUtils; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.junit.Before; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ColumnInfoImpl.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java similarity index 99% rename from x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ColumnInfoImpl.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java index 4dc43223a4cff..b37c1a30ae0a3 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ColumnInfoImpl.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.core; +package org.elasticsearch.xpack.esql.action; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 9107f2553e6f2..5e36eee7364c7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -26,7 +26,6 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.core.esql.action.EsqlResponse; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index e4f9191c24fcc..116ff588f417a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -22,7 +22,6 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java index e883a7dfca768..eb1f4f95db6fe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java @@ -13,7 +13,6 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.core.esql.action.ColumnInfo; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.util.Collections; import java.util.Iterator; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java index 2e45377634a5c..850b909872635 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/UnsupportedAttribute.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -29,8 +28,6 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -180,13 +177,7 @@ public boolean equals(Object obj) { } @Override - public ColumnInfoImpl columnInfo() { - List originalTypes = field().getOriginalTypes(); - if (originalTypes != null) { - // Sort the original types so they are easier to test against and prettier. - originalTypes = new ArrayList<>(originalTypes); - Collections.sort(originalTypes); - } - return new ColumnInfoImpl(name(), dataType().outputType(), originalTypes); + public List originalTypes() { + return field().getOriginalTypes(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index b1cbfa9011e4d..72ca465f647b7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -36,14 +36,13 @@ import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.action.EsqlQueryTask; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.async.AsyncTaskManagementService; -import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.AbstractLookupService; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; @@ -56,6 +55,8 @@ import java.io.IOException; import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -315,7 +316,17 @@ private EsqlExecutionInfo createEsqlExecutionInfo(EsqlQueryRequest request) { } private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Configuration configuration, Result result) { - List columns = result.schema().stream().map(Attribute::columnInfo).toList(); + List columns = result.schema().stream().map(c -> { + List originalTypes; + if (c.originalTypes() == null) { + originalTypes = null; + } else { + // Sort the original types so they are easier to test against and prettier. + originalTypes = new ArrayList<>(c.originalTypes()); + Collections.sort(originalTypes); + } + return new ColumnInfoImpl(c.name(), c.dataType().outputType(), originalTypes); + }).toList(); EsqlQueryResponse.Profile profile = configuration.profile() ? new EsqlQueryResponse.Profile(result.profiles()) : null; threadPool.getThreadContext().addResponseHeader(AsyncExecutionId.ASYNC_EXECUTION_IS_RUNNING_HEADER, "?0"); if (task instanceof EsqlQueryTask asyncTask && request.keepOnCompletion()) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index c04858852e7f5..5dc913af74165 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -51,7 +51,6 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.UnsupportedEsFieldTests; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java index 054aa9639bd53..723f6c89f62cd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatTests.java @@ -18,8 +18,8 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import org.elasticsearch.xpack.esql.core.util.StringUtils; import java.io.IOException; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java index f7db448e818b1..ec9bb14d2a265 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/formatter/TextFormatterTests.java @@ -16,9 +16,9 @@ import org.elasticsearch.compute.test.TestBlockFactory; import org.elasticsearch.geometry.Point; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.action.ColumnInfoImpl; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; -import org.elasticsearch.xpack.esql.core.ColumnInfoImpl; import java.util.Arrays; import java.util.List;