Import wiredtiger: d29619b755d79cb4406349e1309935b8c2fda2c2 from branch mongodb-4.6

ref: 54d5655b43..d29619b755
for: 4.5.1

WT-6274       SESSIONs shouldn't nest calls to get/release hs cursor
WT-6421       Avoid parsing metadata checkpoint for clean files
WT-6466       Don't reset the WT_SESSION timers when HS cursors are closed
WT-6551       Avoid a timing problem at the beginning of a test for log preallocation

Author: lukech

src/third_party/wiredtiger/dist/s_stat

@@ -22,6 +22,7 @@ search=`sed \
 
 # There are some fields that are used, but we can't detect it.
 cat << UNUSED_STAT_FIELDS
+btree_clean_checkpoint_timer
 lock_checkpoint_count
 lock_checkpoint_wait_application
 lock_checkpoint_wait_internal

src/third_party/wiredtiger/dist/stat_data.py

@@ -728,6 +728,7 @@ def __init__(self, name, desc, flags=''):
     # Btree statistics
     ##########################################
     BtreeStat('btree_checkpoint_generation', 'btree checkpoint generation', 'no_clear,no_scale'),
+    BtreeStat('btree_clean_checkpoint_timer', 'btree clean tree checkpoint expiration time', 'no_clear,no_scale'),
     BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,tree_walk'),
     BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,tree_walk'),
     BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,tree_walk'),
@@ -965,4 +966,4 @@ def __init__(self, name, desc, flags=''):
     SessionStat('write_time', 'page write from cache to disk time (usecs)'),
 ]
 
-session_stats = sorted(session_stats, key=attrgetter('desc'))
+session_stats = sorted(session_stats, key=attrgetter('desc'))

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-4.6",
-    "commit": "54d5655b43b91028ee1c90eadd0969c863a9e7e5"
+    "commit": "d29619b755d79cb4406349e1309935b8c2fda2c2"
 }

src/third_party/wiredtiger/src/btree/bt_debug.c

@@ -21,7 +21,7 @@ struct __wt_dbg {
     WT_ITEM *hs_key; /* History store lookups */
     WT_ITEM *hs_value;
     uint32_t session_flags;
-    bool hs_is_local, is_owner;
+    bool hs_is_local;
 
     /*
      * When using the standard event handlers, the debugging output has to do its own message
@@ -288,7 +288,7 @@ __debug_wrapup(WT_DBG *ds)
     msg = ds->msg;
 
     if (ds->hs_is_local)
-        WT_TRET(__wt_hs_cursor_close(session, ds->session_flags, ds->is_owner));
+        WT_TRET(__wt_hs_cursor_close(session, ds->session_flags));
 
     __wt_scr_free(session, &ds->key);
     __wt_scr_free(session, &ds->hs_key);
@@ -975,15 +975,14 @@ __wt_debug_cursor_tree_hs(void *cursor_arg, const char *ofile)
     WT_DECL_RET;
     WT_SESSION_IMPL *session;
     uint32_t session_flags;
-    bool is_owner;
 
     session = CUR2S(cursor_arg);
     session_flags = 0; /* [-Werror=maybe-uninitialized] */
 
-    WT_RET(__wt_hs_cursor(session, &session_flags, &is_owner));
+    WT_RET(__wt_hs_cursor_open(session, &session_flags));
     cbt = (WT_CURSOR_BTREE *)session->hs_cursor;
     WT_WITH_BTREE(session, CUR2BT(cbt), ret = __wt_debug_tree_all(session, NULL, NULL, ofile));
-    WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner));
+    WT_TRET(__wt_hs_cursor_close(session, session_flags));
 
     return (ret);
 }
@@ -1029,8 +1028,7 @@ __debug_page(WT_DBG *ds, WT_REF *ref, uint32_t flags)
      * doesn't work, we may be running in-memory.
      */
     if (!WT_IS_HS(S2BT(session))) {
-        if (session->hs_cursor == NULL &&
-          __wt_hs_cursor(session, &ds->session_flags, &ds->is_owner) == 0)
+        if (session->hs_cursor == NULL && __wt_hs_cursor_open(session, &ds->session_flags) == 0)
             ds->hs_is_local = true;
         if (session->hs_cursor != NULL) {
             WT_RET(__wt_scr_alloc(session, 0, &ds->hs_key));

src/third_party/wiredtiger/src/btree/bt_vrfy.c

@@ -173,7 +173,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[])
 #if 0
     /* FIXME-WT-6263: Temporarily disable history store verification. */
     uint32_t session_flags;
-    bool is_owner, skip_hs;
+    bool skip_hs;
 #endif
 
     btree = S2BT(session);
@@ -185,7 +185,6 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[])
 #if 0
     /* FIXME-WT-6263: Temporarily disable history store verification. */
     session_flags = 0; /* -Wuninitialized */
-    is_owner = false; /* -Wuninitialized */
 
     /*
      * Skip the history store explicit call if we're performing a metadata verification. The
@@ -283,9 +282,9 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[])
              */
             if (ret == 0 && (ckpt + 1)->name == NULL && !skip_hs) {
                 /* Open a history store cursor. */
-                WT_ERR(__wt_hs_cursor(session, &session_flags, &is_owner));
+                WT_ERR(__wt_hs_cursor_open(session, &session_flags);
                 WT_TRET(__wt_history_store_verify_one(session));
-                WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner));
+                WT_TRET(__wt_hs_cursor_close(session, session_flags);
                 /*
                  * We cannot error out here. If we got an error verifying the history store, we need
                  * to follow through with reacquiring the exclusive call below. We'll error out

src/third_party/wiredtiger/src/conn/conn_cache.c

@@ -208,7 +208,7 @@ __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[])
      */
     if (reconfigure)
         WT_RET(__wt_thread_group_resize(session, &conn->evict_threads, conn->evict_threads_min,
-          conn->evict_threads_max, WT_THREAD_CAN_WAIT | WT_THREAD_HS | WT_THREAD_PANIC_FAIL));
+          conn->evict_threads_max, WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL));
 
     return (0);
 }

src/third_party/wiredtiger/src/evict/evict_lru.c

@@ -274,7 +274,6 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread)
     WT_DECL_RET;
     uint32_t session_flags;
     bool did_work, was_intr;
-    bool is_owner;
 
     conn = S2C(session);
     cache = conn->cache;
@@ -284,10 +283,10 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread)
      * busy and then opens a different file (in this case, the HS file), it can deadlock with a
      * thread waiting for the first file to drain from the eviction queue. See WT-5946 for details.
      */
-    if (!F_ISSET(conn, WT_CONN_IN_MEMORY)) {
+    if (session->hs_cursor == NULL && !F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY)) {
         session_flags = 0; /* [-Werror=maybe-uninitialized] */
-        WT_RET(__wt_hs_cursor(session, &session_flags, &is_owner));
-        WT_RET(__wt_hs_cursor_close(session, session_flags, is_owner));
+        WT_RET(__wt_hs_cursor_open(session, &session_flags));
+        WT_RET(__wt_hs_cursor_close(session, session_flags));
     }
 
     if (conn->evict_server_running && __wt_spin_trylock(session, &cache->evict_pass_lock) == 0) {
@@ -493,7 +492,7 @@ __wt_evict_create(WT_SESSION_IMPL *session)
     /*
      * Create the eviction thread group. Set the group size to the maximum allowed sessions.
      */
-    session_flags = WT_THREAD_CAN_WAIT | WT_THREAD_HS | WT_THREAD_PANIC_FAIL;
+    session_flags = WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL;
     WT_RET(__wt_thread_group_create(session, &conn->evict_threads, "eviction-server",
       conn->evict_threads_min, conn->evict_threads_max, session_flags, __wt_evict_thread_chk,
       __wt_evict_thread_run, __wt_evict_thread_stop));

src/third_party/wiredtiger/src/evict/evict_page.c

@@ -96,7 +96,7 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32
     WT_PAGE *page;
     uint64_t time_start, time_stop;
     uint32_t session_flags;
-    bool clean_page, closing, force_evict_hs, inmem_split, is_owner, local_gen, tree_dead;
+    bool clean_page, closing, force_evict_hs, inmem_split, local_gen, tree_dead;
 
     conn = S2C(session);
     page = ref->page;
@@ -134,8 +134,8 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, uint8_t previous_state, uint32
       session->hs_cursor == NULL && !F_ISSET(session, WT_SESSION_NO_RECONCILE) &&
       session != conn->default_session) {
         session_flags = 0; /* [-Werror=maybe-uninitialized] */
-        WT_RET(__wt_hs_cursor(session, &session_flags, &is_owner));
-        WT_RET(__wt_hs_cursor_close(session, session_flags, is_owner));
+        WT_RET(__wt_hs_cursor_open(session, &session_flags));
+        WT_RET(__wt_hs_cursor_close(session, session_flags));
     }
 
     /*

src/third_party/wiredtiger/src/history/hs.c

@@ -63,17 +63,16 @@ __wt_hs_get_btree(WT_SESSION_IMPL *session, WT_BTREE **hs_btreep)
 {
     WT_DECL_RET;
     uint32_t session_flags;
-    bool is_owner;
 
     *hs_btreep = NULL;
     session_flags = 0; /* [-Werror=maybe-uninitialized] */
 
-    WT_RET(__wt_hs_cursor(session, &session_flags, &is_owner));
+    WT_RET(__wt_hs_cursor_open(session, &session_flags));
 
     *hs_btreep = CUR2BT(session->hs_cursor);
     WT_ASSERT(session, *hs_btreep != NULL);
 
-    WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner));
+    WT_TRET(__wt_hs_cursor_close(session, session_flags));
 
     return (ret);
 }
@@ -212,54 +211,32 @@ __wt_hs_destroy(WT_SESSION_IMPL *session)
  *     Open a new history store table cursor.
  */
 int
-__wt_hs_cursor_open(WT_SESSION_IMPL *session)
+__wt_hs_cursor_open(WT_SESSION_IMPL *session, uint32_t *session_flags)
 {
     WT_CURSOR *cursor;
     WT_DECL_RET;
     const char *open_cursor_cfg[] = {WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL};
 
+    /* Not allowed to open a cursor if you already have one */
+    WT_ASSERT(session, session->hs_cursor == NULL && !F_ISSET(session, WT_SESSION_HS_CURSOR));
+
     WT_WITHOUT_DHANDLE(
       session, ret = __wt_open_cursor(session, WT_HS_URI, NULL, open_cursor_cfg, &cursor));
     WT_RET(ret);
 
     /* History store cursors should always ignore tombstones. */
     F_SET(cursor, WT_CURSTD_IGNORE_TOMBSTONE);
 
-    session->hs_cursor = cursor;
-    F_SET(session, WT_SESSION_HS_CURSOR);
-
-    return (0);
-}
-
-/*
- * __wt_hs_cursor --
- *     Return a history store cursor, open one if not already open.
- */
-int
-__wt_hs_cursor(WT_SESSION_IMPL *session, uint32_t *session_flags, bool *is_owner)
-{
     /*
-     * We don't want to get tapped for eviction after we start using the history store cursor; save
-     * a copy of the current eviction state, we'll turn eviction off before we return.
-     *
-     * Don't cache history store table pages, we're here because of eviction problems and there's no
-     * reason to believe history store pages will be useful more than once.
+     * We don't want to get tapped for eviction after we start using the history store cursor. Save
+     * a copy of the current flag values. We'll restore them when the cursor is closed.
      */
     *session_flags = F_MASK(session, WT_HS_SESSION_FLAGS);
-    *is_owner = false;
-
-    /* Open a cursor if this session doesn't already have one. */
-    if (!F_ISSET(session, WT_SESSION_HS_CURSOR)) {
-        /* The caller is responsible for closing this cursor. */
-        *is_owner = true;
-        WT_RET(__wt_hs_cursor_open(session));
-    }
-
-    WT_ASSERT(session, session->hs_cursor != NULL);
-
-    /* Configure session to access the history store table. */
     F_SET(session, WT_HS_SESSION_FLAGS);
 
+    session->hs_cursor = cursor;
+    F_SET(session, WT_SESSION_HS_CURSOR);
+
     return (0);
 }
 
@@ -268,25 +245,13 @@ __wt_hs_cursor(WT_SESSION_IMPL *session, uint32_t *session_flags, bool *is_owner
  *     Discard a history store cursor.
  */
 int
-__wt_hs_cursor_close(WT_SESSION_IMPL *session, uint32_t session_flags, bool is_owner)
+__wt_hs_cursor_close(WT_SESSION_IMPL *session, uint32_t session_flags)
 {
-    /* Nothing to do if the session doesn't have a HS cursor opened. */
-    if (!F_ISSET(session, WT_SESSION_HS_CURSOR)) {
-        WT_ASSERT(session, session->hs_cursor == NULL);
-        return (0);
-    }
-    WT_ASSERT(session, session->hs_cursor != NULL);
+    /* Should only be called when session has an open history store cursor */
+    WT_ASSERT(session, session->hs_cursor != NULL && F_ISSET(session, WT_SESSION_HS_CURSOR));
 
     /*
-     * If we're not the owner, we're not responsible for closing this cursor. Reset the cursor to
-     * avoid pinning the page in cache.
-     */
-    if (!is_owner)
-        return (session->hs_cursor->reset(session->hs_cursor));
-
-    /*
-     * We turned off caching and eviction while the history store cursor was in use, restore the
-     * session's flags.
+     * Restore previous values of history store session flags.
      */
     F_CLR(session, WT_HS_SESSION_FLAGS);
     F_SET(session, session_flags);
@@ -1225,7 +1190,7 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma
     uint32_t hs_btree_id, session_flags;
     uint8_t *p, recno_key_buf[WT_INTPACK64_MAXSIZE], upd_type;
     int cmp;
-    bool is_owner, modify;
+    bool modify;
 
     hs_cursor = NULL;
     mod_upd = upd = NULL;
@@ -1237,7 +1202,6 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma
     hs_btree_id = S2BT(session)->id;
     session_flags = 0; /* [-Werror=maybe-uninitialized] */
     WT_NOT_READ(modify, false);
-    is_owner = false;
 
     WT_STAT_CONN_INCR(session, cursor_search_hs);
     WT_STAT_DATA_INCR(session, cursor_search_hs);
@@ -1258,7 +1222,7 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma
     WT_ERR(__wt_scr_alloc(session, 0, &hs_value));
 
     /* Open a history store table cursor. */
-    WT_ERR(__wt_hs_cursor(session, &session_flags, &is_owner));
+    WT_ERR(__wt_hs_cursor_open(session, &session_flags));
     hs_cursor = session->hs_cursor;
     hs_cbt = (WT_CURSOR_BTREE *)hs_cursor;
 
@@ -1423,7 +1387,7 @@ __wt_hs_find_upd(WT_SESSION_IMPL *session, WT_ITEM *key, const char *value_forma
         __wt_scr_free(session, &hs_value);
     WT_ASSERT(session, hs_key.mem == NULL && hs_key.memsize == 0);
 
-    WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner));
+    WT_TRET(__wt_hs_cursor_close(session, session_flags));
 
     __wt_free_update_list(session, &mod_upd);
     while (modifies.size > 0) {
@@ -1519,26 +1483,14 @@ __wt_hs_delete_key_from_ts(
   WT_SESSION_IMPL *session, uint32_t btree_id, const WT_ITEM *key, wt_timestamp_t ts)
 {
     WT_DECL_RET;
-    uint32_t session_flags;
-    bool is_owner;
-
-    session_flags = session->flags;
-
-    /*
-     * Some code paths such as schema removal involve deleting keys in metadata and assert that we
-     * shouldn't be opening new dhandles. We won't ever need to blow away history store content in
-     * these cases so let's just return early here.
-     */
-    if (F_ISSET(session, WT_SESSION_NO_DATA_HANDLES))
-        return (0);
 
-    WT_RET(__wt_hs_cursor(session, &session_flags, &is_owner));
+    /* If the operation can't open new handles, it should have figured that out before here. */
+    WT_ASSERT(session, !F_ISSET(session, WT_SESSION_NO_DATA_HANDLES));
 
     /* The tree structure can change while we try to insert the mod list, retry if that happens. */
     while ((ret = __hs_delete_key_from_ts_int(session, btree_id, key, ts)) == WT_RESTART)
         WT_STAT_CONN_INCR(session, cache_hs_insert_restart);
 
-    WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner));
     return (ret);
 }
 
@@ -1915,7 +1867,7 @@ __wt_history_store_verify(WT_SESSION_IMPL *session)
     uint64_t hs_counter;
     uint32_t btree_id, session_flags;
     char *uri_data;
-    bool is_owner, stop;
+    bool stop;
 
     /* We should never reach here if working in context of the default session. */
     WT_ASSERT(session, S2C(session)->default_session != session);
@@ -1925,10 +1877,9 @@ __wt_history_store_verify(WT_SESSION_IMPL *session)
     btree_id = WT_BTREE_ID_INVALID;
     session_flags = 0; /* [-Wconditional-uninitialized] */
     uri_data = NULL;
-    is_owner = false; /* [-Wconditional-uninitialized] */
 
     WT_ERR(__wt_scr_alloc(session, 0, &buf));
-    WT_ERR(__wt_hs_cursor(session, &session_flags, &is_owner));
+    WT_ERR(__wt_hs_cursor_open(session, &session_flags));
     cursor = session->hs_cursor;
     WT_ERR_NOTFOUND_OK(__wt_hs_cursor_next(session, cursor), true);
     stop = ret == WT_NOTFOUND ? true : false;
@@ -1960,7 +1911,7 @@ __wt_history_store_verify(WT_SESSION_IMPL *session)
         WT_ERR_NOTFOUND_OK(ret, false);
     }
 err:
-    WT_TRET(__wt_hs_cursor_close(session, session_flags, is_owner));
+    WT_TRET(__wt_hs_cursor_close(session, session_flags));
 
     __wt_scr_free(session, &buf);
     WT_ASSERT(session, hs_key.mem == NULL && hs_key.memsize == 0);

src/third_party/wiredtiger/src/include/btree.h

@@ -203,6 +203,21 @@ struct __wt_btree {
      */
     uint64_t file_max;
 
+/*
+ * We maintain a timer for a clean file to avoid excessive checking of checkpoint information that
+ * incurs a large processing penalty. We avoid that but will periodically incur the cost to clean up
+ * checkpoints that can be deleted.
+ */
+#define WT_BTREE_CLEAN_CKPT(session, btree, val)                          \
+    do {                                                                  \
+        (btree)->clean_ckpt_timer = (val);                                \
+        WT_STAT_DATA_SET((session), btree_clean_checkpoint_timer, (val)); \
+    } while (0)
+/* Statistics don't like UINT64_MAX, use INT64_MAX. It's still forever. */
+#define WT_BTREE_CLEAN_CKPT_FOREVER INT64_MAX
+#define WT_BTREE_CLEAN_MINUTES 10
+    uint64_t clean_ckpt_timer;
+
     /*
      * We flush pages from the tree (in order to make checkpoint faster), without a high-level lock.
      * To avoid multiple threads flushing at the same time, lock the tree.