Skip to content

Commit 4bb4d4b

Browse files
astefanastefan
authored
ESQL: Don't push down filters on the right hand side of an inlinejoin (#127383)
* Don't push down filters on the right hand side of an inlinejoin.
1 parent 2f39c49 commit 4bb4d4b

File tree

6 files changed

+115
-46
lines changed

6 files changed

+115
-46
lines changed

‎docs/changelog/127383.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127383
2+
summary: Don't push down filters on the right hand side of an inlinejoin
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V3;
5050
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
5151
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
52-
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V5;
52+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V6;
5353
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;
5454
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
5555
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
@@ -127,7 +127,7 @@ protected void shouldSkipTest(String testName) throws IOException {
127127
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
128128
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
129129
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
130-
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V5.capabilityName()));
130+
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V6.capabilityName()));
131131
assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName()));
132132
// Unmapped fields require a coorect capability response from every cluster, which isn't currently implemented.
133133
assumeFalse("UNMAPPED FIELDS not yet supported in CCS", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()));

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec

+100-40
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
//
44

55
maxOfInt
6-
required_capability: inlinestats_v5
6+
required_capability: inlinestats_v6
77
// tag::max-languages[]
88
FROM employees
99
| KEEP emp_no, languages
@@ -25,7 +25,7 @@ emp_no:integer | languages:integer | max_lang:integer
2525
;
2626

2727
maxOfIntByKeyword
28-
required_capability: inlinestats_v5
28+
required_capability: inlinestats_v6
2929

3030
FROM employees
3131
| KEEP emp_no, languages, gender
@@ -43,7 +43,7 @@ emp_no:integer | languages:integer | max_lang:integer | gender:keyword
4343
;
4444

4545
maxOfLongByKeyword
46-
required_capability: inlinestats_v5
46+
required_capability: inlinestats_v6
4747

4848
FROM employees
4949
| KEEP emp_no, avg_worked_seconds, gender
@@ -58,7 +58,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | gender:
5858
;
5959

6060
maxOfLong
61-
required_capability: inlinestats_v5
61+
required_capability: inlinestats_v6
6262

6363
FROM employees
6464
| KEEP emp_no, avg_worked_seconds, gender
@@ -71,7 +71,7 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon
7171
;
7272

7373
maxOfLongByCalculatedKeyword
74-
required_capability: inlinestats_v5
74+
required_capability: inlinestats_v6
7575

7676
// tag::longest-tenured-by-first[]
7777
FROM employees
@@ -94,7 +94,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se
9494
;
9595

9696
maxOfLongByCalculatedNamedKeyword
97-
required_capability: inlinestats_v5
97+
required_capability: inlinestats_v6
9898

9999
FROM employees
100100
| KEEP emp_no, avg_worked_seconds, last_name
@@ -113,7 +113,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se
113113
;
114114

115115
maxOfLongByCalculatedDroppedKeyword
116-
required_capability: inlinestats_v5
116+
required_capability: inlinestats_v6
117117

118118
FROM employees
119119
| INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY l = SUBSTRING(last_name, 0, 1)
@@ -132,7 +132,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se
132132
;
133133

134134
maxOfLongByEvaledKeyword
135-
required_capability: inlinestats_v5
135+
required_capability: inlinestats_v6
136136

137137
FROM employees
138138
| EVAL l = SUBSTRING(last_name, 0, 1)
@@ -152,7 +152,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | l:keywo
152152
;
153153

154154
maxOfLongByInt
155-
required_capability: inlinestats_v5
155+
required_capability: inlinestats_v6
156156

157157
FROM employees
158158
| KEEP emp_no, avg_worked_seconds, languages
@@ -170,7 +170,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | languag
170170
;
171171

172172
maxOfLongByIntDouble
173-
required_capability: inlinestats_v5
173+
required_capability: inlinestats_v6
174174

175175
FROM employees
176176
| KEEP emp_no, avg_worked_seconds, languages, height
@@ -205,8 +205,8 @@ emp_no:integer | languages:integer | avg_worked_seconds:long | gender:keyword |
205205
10007 | 4 | 393084805 | F | 2.863684210555556E8 | 5
206206
;
207207

208-
byMultivaluedSimple-Ignore
209-
required_capability: join_planning_v1
208+
byMultivaluedSimple
209+
required_capability: inlinestats_v6
210210

211211
// tag::mv-group[]
212212
FROM airports
@@ -223,8 +223,8 @@ abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer
223223
// end::mv-group-result[]
224224
;
225225

226-
byMultivaluedMvExpand-Ignore
227-
required_capability: join_planning_v1
226+
byMultivaluedMvExpand
227+
required_capability: inlinestats_v6
228228

229229
// tag::mv-expand[]
230230
FROM airports
@@ -237,14 +237,14 @@ FROM airports
237237
;
238238

239239
// tag::mv-expand-result[]
240-
abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer
241-
GWL | mid | 9 | 2
242-
GWL | military | 9 | 4
240+
abbrev:keyword | scalerank:integer | min_scalerank:integer | type:keyword
241+
GWL |9 |2 |mid
242+
GWL |9 |4 |military
243243
// end::mv-expand-result[]
244244
;
245245

246246
byMvExpand
247-
required_capability: inlinestats_v5
247+
required_capability: inlinestats_v6
248248

249249
// tag::extreme-airports[]
250250
FROM airports
@@ -308,7 +308,7 @@ count:long | country:keyword | avg:double
308308
;
309309

310310
afterWhere
311-
required_capability: inlinestats_v5
311+
required_capability: inlinestats_v6
312312

313313
FROM airports
314314
| WHERE country != "United States"
@@ -367,7 +367,7 @@ abbrev:keyword | city:keyword | region:text | "COUNT(*)":long
367367
;
368368

369369
beforeStats
370-
required_capability: inlinestats_v5
370+
required_capability: inlinestats_v6
371371

372372
FROM airports
373373
| EVAL lat = ST_Y(location)
@@ -380,7 +380,7 @@ northern:long | southern:long
380380
;
381381

382382
beforeKeepSort
383-
required_capability: inlinestats_v5
383+
required_capability: inlinestats_v6
384384

385385
FROM employees
386386
| INLINESTATS max_salary = MAX(salary) by languages
@@ -394,8 +394,8 @@ emp_no:integer | languages:integer | max_salary:integer
394394
10003 | 4 | 74572
395395
;
396396

397-
beforeKeepWhere-Ignore
398-
required_capability: join_planning_v1
397+
beforeKeepWhere
398+
required_capability: inlinestats_v6
399399

400400
FROM employees
401401
| INLINESTATS max_salary = MAX(salary) by languages
@@ -407,8 +407,10 @@ emp_no:integer | languages:integer | max_salary:integer
407407
10003 | 4 | 74572
408408
;
409409

410-
beforeEnrich-Ignore
410+
beforeEnrich
411411
required_capability: join_planning_v1
412+
required_capability: inlinestats_v6
413+
required_capability: enrich_load
412414

413415
FROM airports
414416
| KEEP abbrev, type, city
@@ -419,10 +421,10 @@ FROM airports
419421
| LIMIT 3
420422
;
421423

422-
abbrev:keyword | type:keyword | city:keyword | "COUNT(*)":long | region:text
423-
ABJ | mid | Abidjan | 499 | Abidjan
424-
ABV | major | Abuja | 385 | Municipal Area Council
425-
ACA | major | Acapulco de Juárez | 385 | Acapulco de Juárez
424+
abbrev:keyword |city:keyword |"COUNT(*)":long|type:keyword | region:text
425+
ABJ |Abidjan |499 |mid |Abidjan
426+
ABV |Abuja |385 |major |Municipal Area Council
427+
ACA |Acapulco de Juárez|385 |major |Acapulco de Juárez
426428
;
427429

428430
beforeAndAfterEnrich-Ignore
@@ -501,7 +503,7 @@ Zürich | Zürich
501503
;
502504

503505
byConstant
504-
required_capability: inlinestats_v5
506+
required_capability: inlinestats_v6
505507

506508
FROM employees
507509
| KEEP emp_no, languages
@@ -520,7 +522,7 @@ emp_no:integer | languages:integer | max_lang:integer | y:integer
520522
;
521523

522524
aggConstant
523-
required_capability: inlinestats_v5
525+
required_capability: inlinestats_v6
524526

525527
FROM employees
526528
| KEEP emp_no
@@ -538,7 +540,7 @@ one:integer | emp_no:integer
538540
;
539541

540542
percentile
541-
required_capability: inlinestats_v5
543+
required_capability: inlinestats_v6
542544

543545
FROM employees
544546
| KEEP emp_no, salary
@@ -557,7 +559,7 @@ emp_no:integer | salary:integer | ninety_fifth_salary:double
557559
;
558560

559561
byTwoCalculated
560-
required_capability: inlinestats_v5
562+
required_capability: inlinestats_v6
561563

562564
FROM airports
563565
| WHERE abbrev IS NOT NULL
@@ -642,7 +644,7 @@ abbrev:keyword | scalerank:integer | location:geo_point
642644
;
643645

644646
groupShadowsField
645-
required_capability: inlinestats_v5
647+
required_capability: inlinestats_v6
646648

647649
FROM employees
648650
| KEEP emp_no, salary, hire_date
@@ -661,7 +663,7 @@ emp_no:integer | salary:integer | avg_salary:double | hire_date:datetime
661663
;
662664

663665
groupByExpression_And_ExistentField
664-
required_capability: inlinestats_v5
666+
required_capability: inlinestats_v6
665667
FROM employees
666668
| KEEP emp_no, languages, gender
667669
| EVAL x = "ABC"
@@ -679,7 +681,7 @@ emp_no:integer | languages:integer | x:keyword | max_lang:integer | y:keyword |
679681
;
680682

681683
groupByRenamedColumn
682-
required_capability: inlinestats_v5
684+
required_capability: inlinestats_v6
683685
FROM employees
684686
| KEEP emp_no, languages, gender
685687
| INLINESTATS max_lang = MAX(languages) BY y = gender
@@ -787,7 +789,7 @@ emp_no:integer | languages:integer | gender:keyword|max_lang:integer| y:keyword
787789
;
788790

789791
twoAggregatesGroupedBy_AField_And_AnExpression
790-
required_capability: inlinestats_v5
792+
required_capability: inlinestats_v6
791793

792794
FROM employees
793795
| KEEP emp_no, languages, gender, last_name
@@ -809,7 +811,7 @@ emp_no:integer |languages:integer|last_name:keyword|max_lang:integer|min_lang:in
809811
;
810812

811813
groupByMultipleRenamedColumns_InversedOrder
812-
required_capability: inlinestats_v5
814+
required_capability: inlinestats_v6
813815

814816
FROM employees
815817
| KEEP emp_no, languages, still_hired, gender
@@ -827,7 +829,7 @@ emp_no:integer |languages:integer|still_hired:boolean| gender:keyword|max_lang:i
827829
;
828830

829831
groupByMultipleRenamedColumns_InversedOrder_ComplexEval
830-
required_capability: inlinestats_v5
832+
required_capability: inlinestats_v6
831833

832834
FROM employees
833835
| KEEP emp_no, languages, still_hired, gender
@@ -846,7 +848,7 @@ emp_no:integer |languages:integer|still_hired:boolean| gender:keyword|multilingu
846848
;
847849

848850
groupByMultipleRenamedColumns_AndComplexEval
849-
required_capability: inlinestats_v5
851+
required_capability: inlinestats_v6
850852

851853
FROM employees
852854
| KEEP emp_no, languages, still_hired, gender
@@ -888,7 +890,7 @@ emp_no:integer |languages:integer|gender:keyword |first_name:keyword | x:keyw
888890
;
889891

890892
groupByRenamedExpression
891-
required_capability: inlinestats_v5
893+
required_capability: inlinestats_v6
892894

893895
FROM employees
894896
| KEEP emp_no, languages, gender, last_name
@@ -908,4 +910,62 @@ emp_no:integer |languages:integer|last_name:keyword|max_lang:integer|min_lang:in
908910
10049 |5 |Tramer |5 |5 |T |F
909911
10028 |null |Tempesti |1 |1 |T |M
910912
;
913+
914+
doubleFilterOnLeftAndRight_InlineStats_Sides
915+
required_capability: inlinestats_v6
911916

917+
FROM employees
918+
| INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages
919+
| KEEP emp_no, languages, *salary
920+
| WHERE salary > 65000 and languages > 2
921+
| SORT emp_no
922+
;
923+
924+
emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary:integer
925+
10007 |4 |74572 |74572 |27215
926+
10030 |3 |67492 |74970 |26436
927+
10045 |3 |74970 |74970 |26436
928+
10054 |4 |65367 |74572 |27215
929+
10062 |3 |65030 |74970 |26436
930+
10094 |5 |66817 |66817 |25324
931+
10097 |3 |71165 |74970 |26436
932+
10100 |4 |68431 |74572 |27215
933+
;
934+
935+
filterOnInlineStatsAggs
936+
required_capability: inlinestats_v6
937+
938+
FROM employees
939+
| INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages
940+
| KEEP emp_no, languages, *salary
941+
| WHERE min_salary > 27000 or max_salary < 70000
942+
| sort salary desc
943+
| limit 5
944+
;
945+
946+
emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary:integer
947+
10029 |null |74999 |74999 |28336
948+
10007 |4 |74572 |74572 |27215
949+
10027 |null |73851 |74999 |28336
950+
10099 |2 |73578 |73578 |29175
951+
10078 |2 |69904 |73578 |29175
952+
;
953+
954+
filterOnInlineStatsAggsValues_And_Groupings
955+
required_capability: inlinestats_v6
956+
957+
FROM employees
958+
| INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages
959+
| KEEP emp_no, languages, *salary
960+
| WHERE (min_salary > 27000 or max_salary < 70000) and languages > 3
961+
| sort salary
962+
| limit 5
963+
;
964+
965+
emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary:integer
966+
10015 |5 |25324 |66817 |25324
967+
10035 |5 |25945 |66817 |25324
968+
10057 |4 |27215 |74572 |27215
969+
10011 |5 |31120 |66817 |25324
970+
10066 |5 |31897 |66817 |25324
971+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -873,7 +873,7 @@ public enum Cap {
873873
* Fixes a series of issues with inlinestats which had an incomplete implementation after lookup and inlinestats
874874
* were refactored.
875875
*/
876-
INLINESTATS_V5(EsqlPlugin.INLINESTATS_FEATURE_FLAG),
876+
INLINESTATS_V6(EsqlPlugin.INLINESTATS_FEATURE_FLAG),
877877

878878
/**
879879
* Support partial_results

0 commit comments

Comments
 (0)