Skip to content

Commit 826f10f

Browse files
guilloadguilload
authored
Log PostgreSQL metastore error (quickwit-oss#5530)
1 parent a91d2e7 commit 826f10f

File tree

1 file changed

+23
-36
lines changed
  • quickwit/quickwit-metastore/src/metastore/postgres

1 file changed

+23
-36
lines changed

‎quickwit/quickwit-metastore/src/metastore/postgres/metastore.rs

Lines changed: 23 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ use futures::StreamExt;
2626
use itertools::Itertools;
2727
use quickwit_common::pretty::PrettySample;
2828
use quickwit_common::uri::Uri;
29-
use quickwit_common::{get_bool_from_env, ServiceStream};
29+
use quickwit_common::{get_bool_from_env, rate_limited_error, ServiceStream};
3030
use quickwit_config::{
3131
validate_index_id_pattern, IndexTemplate, IndexTemplateId, PostgresMetastoreConfig,
3232
};
@@ -304,17 +304,20 @@ async fn try_apply_delta_v2(
304304
/// We still use this macro for them in order to make the code
305305
/// "trivially correct".
306306
macro_rules! run_with_tx {
307-
($connection_pool:expr, $tx_refmut:ident, $x:block) => {{
307+
($connection_pool:expr, $tx_refmut:ident, $label:literal, $x:block) => {{
308308
let mut tx: Transaction<'_, Postgres> = $connection_pool.begin().await?;
309309
let $tx_refmut = &mut tx;
310310
let op_fut = move || async move { $x };
311311
let op_result: MetastoreResult<_> = op_fut().await;
312-
if op_result.is_ok() {
313-
debug!("committing transaction");
314-
tx.commit().await?;
315-
} else {
316-
warn!("rolling transaction back");
317-
tx.rollback().await?;
312+
match &op_result {
313+
Ok(_) => {
314+
debug!("committing transaction");
315+
tx.commit().await?;
316+
}
317+
Err(error) => {
318+
rate_limited_error!(limit_per_min = 60, error=%error, "failed to {}, rolling transaction back" , $label);
319+
tx.rollback().await?;
320+
}
318321
}
319322
op_result
320323
}};
@@ -331,22 +334,17 @@ where
331334
{
332335
let index_id = &index_uid.index_id;
333336
let mut index_metadata = index_metadata(tx, index_id, true).await?;
337+
334338
if index_metadata.index_uid != index_uid {
335339
return Err(MetastoreError::NotFound(EntityKind::Index {
336340
index_id: index_id.to_string(),
337341
}));
338342
}
339-
340343
if let MutationOccurred::No(()) = mutate_fn(&mut index_metadata)? {
341344
return Ok(index_metadata);
342345
}
346+
let index_metadata_json = serde_utils::to_json_str(&index_metadata)?;
343347

344-
let index_metadata_json = serde_json::to_string(&index_metadata).map_err(|error| {
345-
MetastoreError::JsonSerializeError {
346-
struct_name: "IndexMetadata".to_string(),
347-
message: error.to_string(),
348-
}
349-
})?;
350348
let update_index_res = sqlx::query(
351349
r#"
352350
UPDATE indexes
@@ -426,7 +424,7 @@ impl MetastoreService for PostgresqlMetastore {
426424
let doc_mapping = request.deserialize_doc_mapping()?;
427425

428426
let index_uid: IndexUid = request.index_uid().clone();
429-
let updated_index_metadata = run_with_tx!(self.connection_pool, tx, {
427+
let updated_index_metadata = run_with_tx!(self.connection_pool, tx, "update index", {
430428
mutate_index_metadata::<MetastoreError, _>(tx, index_uid, |index_metadata| {
431429
let mut mutation_occurred =
432430
index_metadata.set_retention_policy(retention_policy_opt);
@@ -622,7 +620,7 @@ impl MetastoreService for PostgresqlMetastore {
622620
tracing::Span::current().record("split_ids", format!("{split_ids:?}"));
623621

624622
// TODO: Remove transaction.
625-
run_with_tx!(self.connection_pool, tx, {
623+
run_with_tx!(self.connection_pool, tx, "stage splits", {
626624
let upserted_split_ids: Vec<String> = sqlx::query_scalar(r#"
627625
INSERT INTO splits
628626
(split_id, time_range_start, time_range_end, tags, split_metadata_json, delete_opstamp, maturity_timestamp, split_state, index_uid, node_id)
@@ -698,7 +696,7 @@ impl MetastoreService for PostgresqlMetastore {
698696
let staged_split_ids = request.staged_split_ids;
699697
let replaced_split_ids = request.replaced_split_ids;
700698

701-
run_with_tx!(self.connection_pool, tx, {
699+
run_with_tx!(self.connection_pool, tx, "publish splits", {
702700
let mut index_metadata = index_metadata(tx, &index_uid.index_id, true).await?;
703701
if index_metadata.index_uid != index_uid {
704702
return Err(MetastoreError::NotFound(EntityKind::Index {
@@ -744,12 +742,7 @@ impl MetastoreService for PostgresqlMetastore {
744742
})?;
745743
}
746744
}
747-
let index_metadata_json = serde_json::to_string(&index_metadata).map_err(|error| {
748-
MetastoreError::JsonSerializeError {
749-
struct_name: "IndexMetadata".to_string(),
750-
message: error.to_string(),
751-
}
752-
})?;
745+
let index_metadata_json = serde_utils::to_json_str(&index_metadata)?;
753746

754747
const PUBLISH_SPLITS_QUERY: &str = r#"
755748
-- Select the splits to update, regardless of their state.
@@ -854,8 +847,7 @@ impl MetastoreService for PostgresqlMetastore {
854847
}
855848
info!(
856849
%index_uid,
857-
"published {} splits and marked {} for deletion successfully",
858-
num_published_splits, num_marked_splits
850+
"published {num_published_splits} splits and marked {num_marked_splits} for deletion successfully"
859851
);
860852
Ok(EmptyResponse {})
861853
})
@@ -1077,7 +1069,7 @@ impl MetastoreService for PostgresqlMetastore {
10771069
async fn add_source(&self, request: AddSourceRequest) -> MetastoreResult<EmptyResponse> {
10781070
let source_config = request.deserialize_source_config()?;
10791071
let index_uid: IndexUid = request.index_uid().clone();
1080-
run_with_tx!(self.connection_pool, tx, {
1072+
run_with_tx!(self.connection_pool, tx, "add source", {
10811073
mutate_index_metadata::<MetastoreError, _>(tx, index_uid, |index_metadata| {
10821074
index_metadata.add_source(source_config)?;
10831075
Ok(MutationOccurred::Yes(()))
@@ -1091,7 +1083,7 @@ impl MetastoreService for PostgresqlMetastore {
10911083
#[instrument(skip(self))]
10921084
async fn toggle_source(&self, request: ToggleSourceRequest) -> MetastoreResult<EmptyResponse> {
10931085
let index_uid: IndexUid = request.index_uid().clone();
1094-
run_with_tx!(self.connection_pool, tx, {
1086+
run_with_tx!(self.connection_pool, tx, "toggle source", {
10951087
mutate_index_metadata(tx, index_uid, |index_metadata| {
10961088
if index_metadata.toggle_source(&request.source_id, request.enable)? {
10971089
Ok::<_, MetastoreError>(MutationOccurred::Yes(()))
@@ -1109,7 +1101,7 @@ impl MetastoreService for PostgresqlMetastore {
11091101
async fn delete_source(&self, request: DeleteSourceRequest) -> MetastoreResult<EmptyResponse> {
11101102
let index_uid: IndexUid = request.index_uid().clone();
11111103
let source_id = request.source_id.clone();
1112-
run_with_tx!(self.connection_pool, tx, {
1104+
run_with_tx!(self.connection_pool, tx, "delete source", {
11131105
mutate_index_metadata(tx, index_uid.clone(), |index_metadata| {
11141106
index_metadata.delete_source(&source_id)?;
11151107
Ok::<_, MetastoreError>(MutationOccurred::Yes(()))
@@ -1138,7 +1130,7 @@ impl MetastoreService for PostgresqlMetastore {
11381130
request: ResetSourceCheckpointRequest,
11391131
) -> MetastoreResult<EmptyResponse> {
11401132
let index_uid: IndexUid = request.index_uid().clone();
1141-
run_with_tx!(self.connection_pool, tx, {
1133+
run_with_tx!(self.connection_pool, tx, "reset source checkpoint", {
11421134
mutate_index_metadata(tx, index_uid, |index_metadata| {
11431135
if index_metadata.checkpoint.reset_source(&request.source_id) {
11441136
Ok::<_, MetastoreError>(MutationOccurred::Yes(()))
@@ -1178,12 +1170,7 @@ impl MetastoreService for PostgresqlMetastore {
11781170
/// Creates a delete task from a delete query.
11791171
#[instrument(skip(self))]
11801172
async fn create_delete_task(&self, delete_query: DeleteQuery) -> MetastoreResult<DeleteTask> {
1181-
let delete_query_json = serde_json::to_string(&delete_query).map_err(|error| {
1182-
MetastoreError::JsonSerializeError {
1183-
struct_name: "DeleteQuery".to_string(),
1184-
message: error.to_string(),
1185-
}
1186-
})?;
1173+
let delete_query_json = serde_utils::to_json_str(&delete_query)?;
11871174
let (create_timestamp, opstamp): (sqlx::types::time::PrimitiveDateTime, i64) =
11881175
sqlx::query_as(
11891176
r#"

0 commit comments

Comments
 (0)