Skip to content

Commit d684b92

Browse files
trinity-1686atrinity-1686a
authored
allow querying non existing fields (#5308)
* make integration test idempotent * ignore missing field on lenient query
1 parent 489c9d6 commit d684b92

File tree

21 files changed

+123
-38
lines changed

21 files changed

+123
-38
lines changed

‎quickwit/quickwit-doc-mapper/src/doc_mapper.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@ mod tests {
361361
user_text: "json_field.toto.titi:hello".to_string(),
362362
default_fields: None,
363363
default_operator: BooleanOperand::And,
364+
lenient: false,
364365
}
365366
.parse_user_query(&[])
366367
.unwrap();

‎quickwit/quickwit-doc-mapper/src/tag_pruning.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,7 @@ mod test {
398398
user_text: user_query.to_string(),
399399
default_fields: None,
400400
default_operator: BooleanOperand::Or,
401+
lenient: false,
401402
}
402403
.into();
403404
let parsed_query_ast = query_ast.parse_user_query(&[]).unwrap();

‎quickwit/quickwit-jaeger/src/lib.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1182,7 +1182,8 @@ mod tests {
11821182
quickwit_query::query_ast::UserInputQuery {
11831183
user_text: "query".to_string(),
11841184
default_fields: None,
1185-
default_operator: quickwit_query::BooleanOperand::And
1185+
default_operator: quickwit_query::BooleanOperand::And,
1186+
lenient: false,
11861187
}
11871188
.into()
11881189
);

‎quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.expected.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@
190190
"opstamp": 10,
191191
"delete_query": {
192192
"index_uid": "my-index:00000000000000000000000001",
193-
"query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}}]}"
193+
"query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false}]}"
194194
}
195195
}
196196
]

‎quickwit/quickwit-metastore/test-data/file-backed-index/v0.9.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@
190190
"opstamp": 10,
191191
"delete_query": {
192192
"index_uid": "my-index:00000000000000000000000001",
193-
"query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}}}]}"
193+
"query_ast": "{\"type\":\"bool\",\"must\":[{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Harry\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false},{\"type\":\"full_text\",\"field\":\"body\",\"text\":\"Potter\",\"params\":{\"mode\":{\"type\":\"phrase_fallback_to_intersection\"}},\"lenient\":false}]}"
194194
}
195195
}
196196
]

‎quickwit/quickwit-query/src/elastic_query_dsl/match_bool_prefix.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ impl ConvertibleToQueryAst for MatchBoolPrefixQuery {
4848
field: self.field,
4949
text: self.params.query,
5050
params: full_text_params,
51+
lenient: self.params.lenient,
5152
}))
5253
}
5354
}

‎quickwit/quickwit-query/src/elastic_query_dsl/match_phrase_query.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ impl ConvertibleToQueryAst for MatchPhraseQuery {
5959
field: self.field,
6060
text: self.params.query,
6161
params: full_text_params,
62+
lenient: false,
6263
}))
6364
}
6465
}
@@ -159,6 +160,7 @@ mod tests {
159160
field,
160161
text,
161162
params,
163+
lenient: _,
162164
}) = ast
163165
else {
164166
panic!()

‎quickwit/quickwit-query/src/elastic_query_dsl/match_query.rs

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ pub(crate) struct MatchQueryParams {
4242
pub(crate) operator: BooleanOperand,
4343
#[serde(default)]
4444
pub(crate) zero_terms_query: MatchAllOrNone,
45-
// Regardless of this option Quickwit behaves in elasticsearch definition of
46-
// lenient. We include this property here just to accept user queries containing
47-
// this option.
48-
#[serde(default, rename = "lenient")]
49-
pub(crate) _lenient: bool,
45+
// Quickwit and Elastic have different notions of lenient. For us, it means it's okay to
46+
// disregard part of the query where which uses non-existing collumn (which Elastic does by
47+
// default). For Elastic, it covers type errors (searching text in an integer field).
48+
#[serde(default)]
49+
pub(crate) lenient: bool,
5050
}
5151

5252
impl ConvertibleToQueryAst for MatchQuery {
@@ -60,6 +60,7 @@ impl ConvertibleToQueryAst for MatchQuery {
6060
field: self.field,
6161
text: self.params.query,
6262
params: full_text_params,
63+
lenient: self.params.lenient,
6364
}))
6465
}
6566
}
@@ -88,7 +89,7 @@ impl From<String> for MatchQueryParams {
8889
query,
8990
zero_terms_query: Default::default(),
9091
operator: Default::default(),
91-
_lenient: false,
92+
lenient: false,
9293
}
9394
}
9495
}
@@ -137,14 +138,15 @@ mod tests {
137138
query: "hello".to_string(),
138139
operator: BooleanOperand::And,
139140
zero_terms_query: crate::MatchAllOrNone::MatchAll,
140-
_lenient: false,
141+
lenient: false,
141142
},
142143
};
143144
let ast = match_query.convert_to_query_ast().unwrap();
144145
let QueryAst::FullText(FullTextQuery {
145146
field,
146147
text,
147148
params,
149+
lenient: _,
148150
}) = ast
149151
else {
150152
panic!()

‎quickwit/quickwit-query/src/elastic_query_dsl/multi_match.rs

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use crate::elastic_query_dsl::phrase_prefix_query::{
3030
};
3131
use crate::elastic_query_dsl::{ConvertibleToQueryAst, ElasticQueryDslInner};
3232

33-
/// Multi match queries are a bit odd. They end up being expanded into another type query of query.
33+
/// Multi match queries are a bit odd. They end up being expanded into another type of query.
3434
/// In Quickwit, we operate this expansion in generic way at the time of deserialization.
3535
#[derive(Deserialize, Debug, Eq, PartialEq, Clone)]
3636
#[serde(try_from = "MultiMatchQueryForDeserialization")]
@@ -48,11 +48,11 @@ struct MultiMatchQueryForDeserialization {
4848
#[serde_as(deserialize_as = "OneOrMany<_, PreferMany>")]
4949
#[serde(default)]
5050
fields: Vec<String>,
51-
// Regardless of this option Quickwit behaves in elasticsearch definition of
52-
// lenient. We include this property here just to accept user queries containing
53-
// this option.
54-
#[serde(default, rename = "lenient")]
55-
_lenient: bool,
51+
// Quickwit and Elastic have different notions of lenient. For us, it means it's okay to
52+
// disregard part of the query where which uses non-existing collumn (which Elastic does by
53+
// default). For Elastic, it covers type errors (searching text in an integer field).
54+
#[serde(default)]
55+
lenient: bool,
5656
}
5757

5858
fn deserialize_match_query_for_one_field(
@@ -198,7 +198,7 @@ mod tests {
198198
query: "quick brown fox".to_string(),
199199
operator: crate::BooleanOperand::Or,
200200
zero_terms_query: Default::default(),
201-
_lenient: false,
201+
lenient: false,
202202
},
203203
}
204204
.into(),
@@ -208,7 +208,7 @@ mod tests {
208208
query: "quick brown fox".to_string(),
209209
operator: crate::BooleanOperand::Or,
210210
zero_terms_query: Default::default(),
211-
_lenient: false,
211+
lenient: false,
212212
},
213213
}
214214
.into(),
@@ -228,7 +228,7 @@ mod tests {
228228
query: "quick brown fox".to_string(),
229229
operator: crate::BooleanOperand::Or,
230230
zero_terms_query: Default::default(),
231-
_lenient: false,
231+
lenient: false,
232232
},
233233
}
234234
.into(),
@@ -238,7 +238,7 @@ mod tests {
238238
query: "quick brown fox".to_string(),
239239
operator: crate::BooleanOperand::Or,
240240
zero_terms_query: Default::default(),
241-
_lenient: false,
241+
lenient: false,
242242
},
243243
}
244244
.into(),
@@ -258,7 +258,7 @@ mod tests {
258258
query: "quick brown fox".to_string(),
259259
operator: crate::BooleanOperand::Or,
260260
zero_terms_query: Default::default(),
261-
_lenient: false,
261+
lenient: false,
262262
},
263263
}
264264
.into(),
@@ -268,7 +268,7 @@ mod tests {
268268
query: "quick brown fox".to_string(),
269269
operator: crate::BooleanOperand::Or,
270270
zero_terms_query: Default::default(),
271-
_lenient: false,
271+
lenient: false,
272272
},
273273
}
274274
.into(),
@@ -350,7 +350,7 @@ mod tests {
350350
query: "quick brown".to_string(),
351351
operator: crate::BooleanOperand::Or,
352352
zero_terms_query: Default::default(),
353-
_lenient: false,
353+
lenient: false,
354354
},
355355
}
356356
.into(),
@@ -360,7 +360,7 @@ mod tests {
360360
query: "quick brown".to_string(),
361361
operator: crate::BooleanOperand::Or,
362362
zero_terms_query: Default::default(),
363-
_lenient: false,
363+
lenient: false,
364364
},
365365
}
366366
.into(),

‎quickwit/quickwit-query/src/elastic_query_dsl/query_string_query.rs

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ pub(crate) struct QueryStringQuery {
4343
// Regardless of this option Quickwit behaves in elasticsearch definition of
4444
// lenient. We include this property here just to accept user queries containing
4545
// this option.
46-
#[serde(default, rename = "lenient")]
47-
_lenient: bool,
46+
#[serde(default)]
47+
lenient: bool,
4848
}
4949

5050
impl ConvertibleToQueryAst for QueryStringQuery {
@@ -60,6 +60,7 @@ impl ConvertibleToQueryAst for QueryStringQuery {
6060
user_text: self.query,
6161
default_fields,
6262
default_operator: self.default_operator,
63+
lenient: self.lenient,
6364
};
6465
Ok(user_text_query.into())
6566
}
@@ -79,7 +80,7 @@ mod tests {
7980
default_operator: crate::BooleanOperand::Or,
8081
default_field: None,
8182
boost: None,
82-
_lenient: false,
83+
lenient: false,
8384
};
8485
let QueryAst::UserInput(user_input_query) =
8586
query_string_query.convert_to_query_ast().unwrap()
@@ -101,7 +102,7 @@ mod tests {
101102
default_operator: crate::BooleanOperand::Or,
102103
default_field: Some("hello".to_string()),
103104
boost: None,
104-
_lenient: false,
105+
lenient: false,
105106
};
106107
let QueryAst::UserInput(user_input_query) =
107108
query_string_query.convert_to_query_ast().unwrap()
@@ -123,7 +124,7 @@ mod tests {
123124
default_operator: crate::BooleanOperand::Or,
124125
default_field: Some("hello".to_string()),
125126
boost: None,
126-
_lenient: false,
127+
lenient: false,
127128
};
128129
let err_msg = query_string_query
129130
.convert_to_query_ast()
@@ -140,7 +141,7 @@ mod tests {
140141
default_field: None,
141142
default_operator: crate::BooleanOperand::And,
142143
boost: None,
143-
_lenient: false,
144+
lenient: false,
144145
};
145146
let QueryAst::UserInput(user_input_query) =
146147
query_string_query.convert_to_query_ast().unwrap()
@@ -158,7 +159,7 @@ mod tests {
158159
default_field: None,
159160
default_operator: crate::BooleanOperand::Or,
160161
boost: None,
161-
_lenient: false,
162+
lenient: false,
162163
};
163164
let QueryAst::UserInput(user_input_query) =
164165
query_string_query.convert_to_query_ast().unwrap()
@@ -177,7 +178,7 @@ mod tests {
177178
default_field: None,
178179
default_operator: crate::BooleanOperand::Or,
179180
boost: None,
180-
_lenient: false,
181+
lenient: false,
181182
};
182183
let QueryAst::UserInput(user_input_query) =
183184
query_string_query.convert_to_query_ast().unwrap()
@@ -200,7 +201,8 @@ mod tests {
200201
assert!(matches!(query_ast, QueryAst::UserInput(UserInputQuery {
201202
user_text,
202203
default_fields,
203-
default_operator
204+
default_operator,
205+
lenient: _,
204206
}) if user_text == "hello world"
205207
&& default_operator == BooleanOperand::Or
206208
&& default_fields == Some(vec!["text".to_string()])));

‎quickwit/quickwit-query/src/query_ast/full_text_query.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ pub struct FullTextQuery {
227227
pub field: String,
228228
pub text: String,
229229
pub params: FullTextParams,
230+
pub lenient: bool,
230231
}
231232

232233
impl From<FullTextQuery> for QueryAst {
@@ -249,6 +250,7 @@ impl BuildTantivyAst for FullTextQuery {
249250
&self.params,
250251
schema,
251252
tokenizer_manager,
253+
self.lenient,
252254
)
253255
}
254256
}
@@ -323,6 +325,7 @@ mod tests {
323325
mode: BooleanOperand::And.into(),
324326
zero_terms_query: crate::MatchAllOrNone::MatchAll,
325327
},
328+
lenient: false,
326329
};
327330
let mut schema_builder = Schema::builder();
328331
schema_builder.add_text_field("body", TEXT);
@@ -348,6 +351,7 @@ mod tests {
348351
mode: FullTextMode::Phrase { slop: 1 },
349352
zero_terms_query: crate::MatchAllOrNone::MatchAll,
350353
},
354+
lenient: false,
351355
};
352356
let mut schema_builder = Schema::builder();
353357
schema_builder.add_text_field("body", TEXT);
@@ -378,6 +382,7 @@ mod tests {
378382
mode: FullTextMode::Phrase { slop: 1 },
379383
zero_terms_query: crate::MatchAllOrNone::MatchAll,
380384
},
385+
lenient: false,
381386
};
382387
let mut schema_builder = Schema::builder();
383388
schema_builder.add_text_field("body", TEXT);
@@ -407,6 +412,7 @@ mod tests {
407412
mode: BooleanOperand::And.into(),
408413
zero_terms_query: crate::MatchAllOrNone::MatchAll,
409414
},
415+
lenient: false,
410416
};
411417
let mut schema_builder = Schema::builder();
412418
schema_builder.add_text_field("body", TEXT);

‎quickwit/quickwit-query/src/query_ast/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,7 @@ pub fn query_ast_from_user_text(user_text: &str, default_fields: Option<Vec<Stri
316316
user_text: user_text.to_string(),
317317
default_fields,
318318
default_operator: BooleanOperand::And,
319+
lenient: false,
319320
}
320321
.into()
321322
}
@@ -334,6 +335,7 @@ mod tests {
334335
user_text: "*".to_string(),
335336
default_fields: Default::default(),
336337
default_operator: Default::default(),
338+
lenient: false,
337339
}
338340
.into();
339341
let schema = tantivy::schema::Schema::builder().build();
@@ -357,6 +359,7 @@ mod tests {
357359
user_text: "*".to_string(),
358360
default_fields: Default::default(),
359361
default_operator: Default::default(),
362+
lenient: false,
360363
}
361364
.into();
362365
let query_ast_with_parsed_user_query: QueryAst = query_ast.parse_user_query(&[]).unwrap();
@@ -378,6 +381,7 @@ mod tests {
378381
user_text: "*".to_string(),
379382
default_fields: Default::default(),
380383
default_operator: Default::default(),
384+
lenient: false,
381385
}
382386
.into();
383387
let bool_query_ast: QueryAst = BoolQuery {
@@ -412,6 +416,7 @@ mod tests {
412416
user_text: "field:hello field:toto".to_string(),
413417
default_fields: None,
414418
default_operator: crate::BooleanOperand::And,
419+
lenient: false,
415420
}
416421
.parse_user_query(&[])
417422
.unwrap();
@@ -427,6 +432,7 @@ mod tests {
427432
user_text: "field:hello field:toto".to_string(),
428433
default_fields: None,
429434
default_operator: crate::BooleanOperand::Or,
435+
lenient: false,
430436
}
431437
.parse_user_query(&[])
432438
.unwrap();

0 commit comments

Comments
 (0)