Import wiredtiger: ce1f45b4908579eb3e9dd588ee6ed4267783fe13 from branch mongodb-8.1 (#36272)

GitOrigin-RevId: c441b67d7260844c1422bf259e23c054a33ee7d8

Author: lukech

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger",
     "branch": "mongodb-8.1",
-    "commit": "d6eda7a7e8b05772384c4cfe36b1c1d43344adfd"
+    "commit": "ce1f45b4908579eb3e9dd588ee6ed4267783fe13"
 }

src/third_party/wiredtiger/src/checkpoint/checkpoint_txn.c

@@ -1431,6 +1431,15 @@ __checkpoint_db_internal(WT_SESSION_IMPL *session, const char *cfg[])
     if (F_ISSET(&conn->log_mgr, WT_LOG_ENABLED))
         WT_ERR(__wt_log_flush(session, WT_LOG_FSYNC));
 
+    /*
+     * Stress point to stop just before we sync the metadata file. Used to recreate log recovery
+     * scenarios with an incomplete checkpoint.
+     */
+    WT_STAT_CONN_SET(session, checkpoint_stop_stress_active, 1);
+    /* Wait prior to flush the checkpoint stop log record. */
+    __checkpoint_timing_stress(session, WT_TIMING_STRESS_CHECKPOINT_STOP, &tsp);
+    WT_STAT_CONN_SET(session, checkpoint_stop_stress_active, 0);
+
     /*
      * Ensure that the metadata changes are durable before the checkpoint is resolved. Either
      * checkpointing the metadata or syncing the log file works. Recovery relies on the checkpoint
@@ -1459,11 +1468,6 @@ __checkpoint_db_internal(WT_SESSION_IMPL *session, const char *cfg[])
 
     __checkpoint_verbose_track(session, "metadata sync completed");
 
-    WT_STAT_CONN_SET(session, checkpoint_stop_stress_active, 1);
-    /* Wait prior to flush the checkpoint stop log record. */
-    __checkpoint_timing_stress(session, WT_TIMING_STRESS_CHECKPOINT_STOP, &tsp);
-    WT_STAT_CONN_SET(session, checkpoint_stop_stress_active, 0);
-
     /*
      * Now that the metadata is stable, re-open the metadata file for regular eviction by clearing
      * the checkpoint_pinned flag.

src/third_party/wiredtiger/src/conn/conn_dhandle.c

@@ -45,6 +45,10 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session)
     base = NULL;
     tmp = NULL;
 
+    /* We should never be looking at metadata before it's been recovered. */
+    WT_ASSERT_ALWAYS(session, !F_ISSET(S2C(session), WT_CONN_RECOVERING_METADATA),
+      "Assert failure: %s: attempt to open data handle during metadata recovery", session->name);
+
     /*
      * Read the object's entry from the metadata file, we're done if we don't find one.
      */

src/third_party/wiredtiger/src/cursor/cur_hs.c

@@ -73,7 +73,8 @@ __wt_curhs_cache(WT_SESSION_IMPL *session)
      * Make sure this session has a cached history store cursor, otherwise we can deadlock with a
      * session wanting exclusive access to a handle: that session will have a handle list write lock
      * and will be waiting on eviction to drain, we'll be inside eviction waiting on a handle list
-     * read lock to open a history store cursor.
+     * read lock to open a history store cursor. If we are recovering metadata, we aren't ready to
+     * open the history store, and we can't deadlock because we are single threaded.
      *
      * The test for the no-reconciliation flag is necessary because the session may already be doing
      * history store operations and if we open/close the existing history store cursor, we can
@@ -89,7 +90,8 @@ __wt_curhs_cache(WT_SESSION_IMPL *session)
      *
      * FIXME-WT-6037: This isn't reasonable and needs a better fix.
      */
-    if (F_ISSET(conn, WT_CONN_IN_MEMORY) || F_ISSET(session, WT_SESSION_NO_RECONCILE) ||
+    if (F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_RECOVERING_METADATA) ||
+      F_ISSET(session, WT_SESSION_NO_RECONCILE) ||
       (session->dhandle != NULL && WT_IS_METADATA(S2BT(session)->dhandle)) ||
       session == conn->default_session)
         return (0);

src/third_party/wiredtiger/src/evict/evict_inline.h

@@ -633,6 +633,11 @@ __wt_evict_app_assist_worker_check(
     if (didworkp != NULL)
         *didworkp = false;
 
+    /* It is not safe to proceed if the eviction server threads aren't setup yet. */
+    WT_CONNECTION_IMPL *conn = S2C(session);
+    if (!__wt_atomic_loadbool(&conn->evict_server_running))
+        return (0);
+
     /* Eviction causes reconciliation. So don't evict if we can't reconcile */
     if (F_ISSET(session, WT_SESSION_NO_RECONCILE))
         return (0);
@@ -655,9 +660,8 @@ __wt_evict_app_assist_worker_check(
         return (0);
 
     /* Setting cache_max_wait_us to 1 effectively means "disable eviction when possible" */
-    uint64_t cache_max_wait_us = session->cache_max_wait_us != 0 ?
-      session->cache_max_wait_us :
-      S2C(session)->evict->cache_max_wait_us;
+    uint64_t cache_max_wait_us =
+      session->cache_max_wait_us != 0 ? session->cache_max_wait_us : conn->evict->cache_max_wait_us;
     if (cache_max_wait_us == 1)
         return (0);
 
@@ -667,7 +671,7 @@ __wt_evict_app_assist_worker_check(
      * evict what we can. Otherwise, we are at a transaction boundary and we can work harder to make
      * sure there is free space in the cache.
      */
-    WT_TXN_GLOBAL *txn_global = &S2C(session)->txn_global;
+    WT_TXN_GLOBAL *txn_global = &conn->txn_global;
     WT_TXN_SHARED *txn_shared = WT_SESSION_TXN_SHARED(session);
     busy = busy || __wt_atomic_loadv64(&txn_shared->id) != WT_TXN_NONE ||
       session->hazards.num_active > 0 ||
@@ -687,7 +691,7 @@ __wt_evict_app_assist_worker_check(
         return (0);
 
     /* In memory configurations don't block when the cache is full. */
-    if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY))
+    if (F_ISSET(conn, WT_CONN_IN_MEMORY))
         return (0);
 
     /*
@@ -704,6 +708,13 @@ __wt_evict_app_assist_worker_check(
     if (!__wt_evict_needed(session, busy, readonly, &pct_full))
         return (0);
 
+    /*
+     * If the caller is holding shared resources, only evict if the cache is at any of its eviction
+     * targets.
+     */
+    if (busy && pct_full < 100.0)
+        return (0);
+
     /* Last check if application wants to prevent the thread from evicting. */
     if (!__evict_check_user_ok_with_eviction(session, interruptible))
         return (0);
@@ -715,7 +726,7 @@ __wt_evict_app_assist_worker_check(
     if (didworkp != NULL)
         *didworkp = true;
 
-    return (__wti_evict_app_assist_worker(session, busy, readonly, interruptible, pct_full));
+    return (__wti_evict_app_assist_worker(session, busy, readonly, interruptible));
 }
 
 /*

src/third_party/wiredtiger/src/evict/evict_lru.c

@@ -2815,7 +2815,7 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server)
  */
 int
 __wti_evict_app_assist_worker(
-  WT_SESSION_IMPL *session, bool busy, bool readonly, bool interruptible, double pct_full)
+  WT_SESSION_IMPL *session, bool busy, bool readonly, bool interruptible)
 {
     WT_DECL_RET;
     WT_TRACK_OP_DECL;
@@ -2835,18 +2835,11 @@ __wti_evict_app_assist_worker(
      * Before we enter the eviction generation, make sure this session has a cached history store
      * cursor, otherwise we can deadlock with a session wanting exclusive access to a handle: that
      * session will have a handle list write lock and will be waiting on eviction to drain, we'll be
-     * inside eviction waiting on a handle list read lock to open a history store cursor.
+     * inside eviction waiting on a handle list read lock to open a history store cursor. The
+     * eviction server should be started at this point so it is safe to open the history store.
      */
     WT_ERR(__wt_curhs_cache(session));
 
-    /*
-     * It is not safe to proceed if the eviction server threads aren't setup yet. Also, if the
-     * caller is holding shared resources, only evict if the cache is at any of its eviction
-     * targets.
-     */
-    if (!__wt_atomic_loadbool(&conn->evict_server_running) || (busy && pct_full < 100.0))
-        goto done;
-
     /* Wake the eviction server if we need to do work. */
     __wt_evict_server_wake(session);
 
@@ -2907,6 +2900,7 @@ __wti_evict_app_assist_worker(
         uint64_t max_progress = busy ? 5 : 20;
 
         /* See if eviction is still needed. */
+        double pct_full;
         if (!__wt_evict_needed(session, busy, readonly, &pct_full) ||
           (pct_full < 100.0 &&
             (__wt_atomic_loadv64(&evict->eviction_progress) > initial_progress + max_progress)))
@@ -2962,7 +2956,6 @@ __wti_evict_app_assist_worker(
         }
     }
 
-done:
     WT_TRACK_OP_END(session);
 
     return (ret);

src/third_party/wiredtiger/src/evict/evict_private.h

@@ -50,7 +50,7 @@ struct __wti_evict_queue {
 /* DO NOT EDIT: automatically built by prototypes.py: BEGIN */
 
 extern int __wti_evict_app_assist_worker(WT_SESSION_IMPL *session, bool busy, bool readonly,
-  bool interruptible, double pct_full) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+  bool interruptible) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern void __wti_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref);
 static WT_INLINE bool __wti_evict_hs_dirty(WT_SESSION_IMPL *session)
   WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));

src/third_party/wiredtiger/src/include/connection.h

@@ -813,11 +813,12 @@ struct __wt_connection_impl {
 #define WT_CONN_READY 0x00800000u
 #define WT_CONN_RECONFIGURING 0x01000000u
 #define WT_CONN_RECOVERING 0x02000000u
-#define WT_CONN_RECOVERY_COMPLETE 0x04000000u
-#define WT_CONN_RTS_THREAD_RUN 0x08000000u
-#define WT_CONN_SALVAGE 0x10000000u
-#define WT_CONN_TIERED_FIRST_FLUSH 0x20000000u
-#define WT_CONN_WAS_BACKUP 0x40000000u
+#define WT_CONN_RECOVERING_METADATA 0x04000000u
+#define WT_CONN_RECOVERY_COMPLETE 0x08000000u
+#define WT_CONN_RTS_THREAD_RUN 0x10000000u
+#define WT_CONN_SALVAGE 0x20000000u
+#define WT_CONN_TIERED_FIRST_FLUSH 0x40000000u
+#define WT_CONN_WAS_BACKUP 0x80000000u
     /* AUTOMATIC FLAG VALUE GENERATION STOP 32 */
     wt_shared uint32_t flags;
 };

src/third_party/wiredtiger/src/txn/txn_recover.c

@@ -1016,6 +1016,13 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[])
         r.metadata_only = true;
         r.backup_only = false;
     }
+
+    /*
+     * Start metadata recovery. Don't allow any Btree files to be opened, they depend on metadata
+     * that might be modified during recovery.
+     */
+    F_SET(conn, WT_CONN_RECOVERING_METADATA);
+
     /*
      * If this is a read-only connection, check if the checkpoint LSN in the metadata file is up to
      * date, indicating a clean shutdown.
@@ -1036,6 +1043,9 @@ __wt_txn_recover(WT_SESSION_IMPL *session, const char *cfg[])
         ret = __wt_log_scan(
           session, &metafile->ckpt_lsn, NULL, WT_LOGSCAN_RECOVER_METADATA, __txn_log_recover, &r);
     }
+    /* We're finished with metadata recovery, so allow other data files to be opened. */
+    F_CLR(conn, WT_CONN_RECOVERING_METADATA);
+
     if (F_ISSET(conn, WT_CONN_SALVAGE))
         ret = 0;
     /* We need to do some work after recovering backup information. Do that now. */

src/third_party/wiredtiger/test/catch2/sub_level_error/unit/test_sub_level_error_rollback.cpp

@@ -33,16 +33,19 @@ TEST_CASE("Test functions for error handling in rollback workflows",
     SECTION(
       "Test WT_CACHE_OVERFLOW in __wti_evict_app_assist_worker - not safe to proceed with eviction")
     {
+
         // If the eviction server isn't running, then the threads have not been set up yet and it's
         // not safe to evict.
-        CHECK(__wti_evict_app_assist_worker(session_impl, false, false, true, 100) == 0);
+        CHECK(__wt_evict_app_assist_worker_check(session_impl, false, false, true, NULL) == 0);
         check_error_info(err_info, 0, WT_NONE, WT_ERROR_INFO_SUCCESS);
 
         // Set the eviction server as running.
         conn_impl->evict_server_running = true;
         // The eviction sever is running, but the application is busy and the cache is less than 100
         // percent full.
-        CHECK(__wti_evict_app_assist_worker(session_impl, true, false, true, 99) == 0);
+        conn_impl->cache_size = 10 * 1024 * 1024;
+        conn_impl->cache->bytes_inmem = 9 * 1024 * 1024;
+        CHECK(__wt_evict_app_assist_worker_check(session_impl, true, false, true, NULL) == 0);
         check_error_info(err_info, 0, WT_NONE, WT_ERROR_INFO_SUCCESS);
     }
 
@@ -59,7 +62,7 @@ TEST_CASE("Test functions for error handling in rollback workflows",
         // Set transaction's update amount to 1 and ID to be equal to the oldest transaction ID.
         session_impl->txn->mod_count = 1;
         WT_SESSION_TXN_SHARED(session_impl)->id = S2C(session)->txn_global.oldest_id;
-        CHECK(__wti_evict_app_assist_worker(session_impl, false, false, true, 100) == WT_ROLLBACK);
+        CHECK(__wti_evict_app_assist_worker(session_impl, false, false, true) == WT_ROLLBACK);
         check_error_info(err_info, WT_ROLLBACK, WT_OLDEST_FOR_EVICTION,
           "Transaction has the oldest pinned transaction ID");
 
@@ -84,7 +87,7 @@ TEST_CASE("Test functions for error handling in rollback workflows",
         cursor->set_key(cursor, "key");
         cursor->set_value(cursor, "value");
 
-        CHECK(__wti_evict_app_assist_worker(session_impl, false, false, true, 100) == WT_ROLLBACK);
+        CHECK(__wti_evict_app_assist_worker(session_impl, false, false, true) == WT_ROLLBACK);
         check_error_info(err_info, WT_ROLLBACK, WT_CACHE_OVERFLOW, "Cache capacity has overflown");
 
         // Drop the table.