Import wiredtiger-wiredtiger-mongodb-2.8-rc6-47-g5b3283e.tar.gz from wiredtiger branch mongodb-2.8

Author: Dan Pasette

src/third_party/wiredtiger/dist/api_data.py

@@ -309,8 +309,18 @@ def __cmp__(self, other):
             ]),
     Config('cache_size', '100MB', r'''
         maximum heap memory to allocate for the cache. A database should
-        configure either a cache_size or a shared_cache not both''',
+        configure either \c cache_size or \c shared_cache but not both''',
         min='1MB', max='10TB'),
+    Config('cache_overhead', '8', r'''
+        assume the heap allocator overhead is the specified percentage, and
+        adjust the cache size by that amount (for example, if the cache size is
+        100GB, a percentage of 10 means WiredTiger limits itself to allocating
+        90GB of memory).  This value is configurable because different heap
+        allocators have different overhead and different workloads will have
+        different heap allocation sizes and patterns, therefore applications
+        may need to adjust this value based on allocator choice and behavior
+        in measured workloads''',
+        min='0', max='30'),
     Config('checkpoint', '', r'''
         periodically checkpoint the database''',
         type='category', subconfig=[

src/third_party/wiredtiger/dist/s_all

@@ -69,6 +69,7 @@ run "sh ./s_copyright" "checking copyright notices"
 run "sh ./s_define" "checking for unused #defines"
 run "sh ./s_funcs" "checking for unused functions"
 run "sh ./s_getopt" "checking for incorrect getopt usage"
+run "sh ./s_lang" "checking for SWIG generated name conflicts"
 run "sh ./s_longlines" "checking for long lines"
 run "sh ./s_stat" "checking for unused statistics fields"
 run "sh ./s_string" "checking string spelling"

src/third_party/wiredtiger/dist/s_lang

@@ -0,0 +1,25 @@
+#! /bin/sh
+
+# Check lang directories for potential name conflicts
+t=__wt.$$
+trap 'rm -f $t; exit 0' 0 1 2 3 13 15
+
+cd ../lang
+
+for d in *; do
+	f=`find $d -name 'wiredtiger_wrap.c'`
+	test -z "$f" && continue
+
+	sed -e '/SWIGINTERN.*__wt_[a-z][a-z]*_[a-z]/!d' \
+	    -e '/__wt_[^(]*__.*(/d' \
+	    -e '/_wrap/d' \
+	    -e "/_${d}_/d" \
+	    $f > $t
+
+	test -s $t && {
+		echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+		echo "$l: potential SWIG naming conflict"
+		echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+		cat $t
+	}
+done

src/third_party/wiredtiger/dist/s_string.ok

@@ -73,6 +73,7 @@ Crummey
 CustomersPhone
 DATAITEMs
 DECL
+DECR
 DESC
 DHANDLE
 DLFCN
@@ -397,6 +398,7 @@ agc
 alfred
 alloc
 allocator
+allocators
 allocfile
 allocsize
 amd

src/third_party/wiredtiger/dist/stat_data.py

@@ -182,7 +182,10 @@ def __init__(self, name, desc, flags=''):
         'eviction server unable to reach eviction goal'),
     CacheStat('cache_eviction_split', 'pages split during eviction'),
     CacheStat('cache_eviction_walk', 'pages walked for eviction'),
+    CacheStat('cache_eviction_worker_evicting',
+        'eviction worker thread evicting pages'),
     CacheStat('cache_inmem_split', 'in-memory page splits'),
+    CacheStat('cache_overhead', 'percentage overhead', 'no_clear,no_scale'),
     CacheStat('cache_pages_dirty',
         'tracked dirty pages in the cache', 'no_scale'),
     CacheStat('cache_pages_inuse',

src/third_party/wiredtiger/lang/python/wiredtiger.i

@@ -157,7 +157,7 @@ from packing import pack, unpack
 %define DESTRUCTOR(class, method)
 %feature("shadow") class::method %{
 	def method(self, *args):
-		'''close(self, config) -> int
+		'''method(self, config) -> int
 		
 		@copydoc class::method'''
 		try:
@@ -171,6 +171,26 @@ DESTRUCTOR(__wt_connection, close)
 DESTRUCTOR(__wt_cursor, close)
 DESTRUCTOR(__wt_session, close)
 
+/*
+ * OVERRIDE_METHOD must be used when overriding or extending an existing
+ * method in the C interface.  It creates Python method() that calls
+ * _method(), which is the extended version of the method.  This works
+ * around potential naming conflicts.  Without this technique, for example,
+ * defining __wt_cursor::equals() creates the wrapper function
+ * __wt_cursor_equals(), which may be defined in the WT library.
+ */
+%define OVERRIDE_METHOD(cclass, pyclass, method, pyargs)
+%extend cclass {
+%pythoncode %{
+	def method(self, *args):
+		'''method pyargs -> int
+
+		@copydoc class::method'''
+		return self._##method(*args)
+%}
+};
+%enddef
+
 /* Don't require empty config strings. */
 %typemap(default) const char *config { $1 = NULL; }
 %typemap(default) WT_CURSOR *to_dup { $1 = NULL; }
@@ -389,9 +409,9 @@ NOTFOUND_OK(__wt_cursor::remove)
 NOTFOUND_OK(__wt_cursor::search)
 NOTFOUND_OK(__wt_cursor::update)
 
-COMPARE_OK(__wt_cursor::compare)
-COMPARE_OK(__wt_cursor::equals)
-COMPARE_NOTFOUND_OK(__wt_cursor::search_near)
+COMPARE_OK(__wt_cursor::_compare)
+COMPARE_OK(__wt_cursor::_equals)
+COMPARE_NOTFOUND_OK(__wt_cursor::_search_near)
 
 /* Lastly, some methods need no (additional) error checking. */
 %exception __wt_connection::get_home;
@@ -428,6 +448,10 @@ COMPARE_NOTFOUND_OK(__wt_cursor::search_near)
 %ignore __wt_cursor::equals(WT_CURSOR *, WT_CURSOR *, int *);
 %ignore __wt_cursor::search_near(WT_CURSOR *, int *);
 
+OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, compare, (self, other))
+OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, equals, (self, other))
+OVERRIDE_METHOD(__wt_cursor, WT_CURSOR, search_near, (self))
+
 /* SWIG magic to turn Python byte strings into data / size. */
 %apply (char *STRING, int LENGTH) { (char *data, int size) };
 
@@ -685,7 +709,7 @@ typedef int int_void;
 	}
 
 	/* compare: special handling. */
-	int compare(WT_CURSOR *other) {
+	int _compare(WT_CURSOR *other) {
 		int cmp = 0;
 		int ret = 0;
 		if (other == NULL) {
@@ -709,7 +733,7 @@ typedef int int_void;
 	}
 
 	/* equals: special handling. */
-	int equals(WT_CURSOR *other) {
+	int _equals(WT_CURSOR *other) {
 		int cmp = 0;
 		int ret = 0;
 		if (other == NULL) {
@@ -728,7 +752,7 @@ typedef int int_void;
 	}
 
 	/* search_near: special handling. */
-	int search_near() {
+	int _search_near() {
 		int cmp = 0;
 		int ret = $self->search_near($self, &cmp);
 		/*
@@ -828,7 +852,7 @@ typedef int int_void;
 };
 
 %extend __wt_session {
-	int log_printf(const char *msg) {
+	int _log_printf(const char *msg) {
 		return self->log_printf(self, "%s", msg);
 	}
 
@@ -892,6 +916,8 @@ int verbose_build();
 %ignore __wt_connection::get_extension_api;
 %ignore __wt_session::log_printf;
 
+OVERRIDE_METHOD(__wt_session, WT_SESSION, log_printf, (self, msg))
+
 %ignore wiredtiger_struct_pack;
 %ignore wiredtiger_struct_size;
 %ignore wiredtiger_struct_unpack;

src/third_party/wiredtiger/src/block/block_compact.c

@@ -50,6 +50,8 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block)
 	block->allocfirst = block->allocfirst_save;
 	__wt_spin_unlock(session, &block->live_lock);
 
+	block->compact_pct_tenths = 0;
+
 	return (0);
 }
 
@@ -64,7 +66,7 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, int *skipp)
 	WT_EXT *ext;
 	WT_EXTLIST *el;
 	WT_FH *fh;
-	wt_off_t avail, ninety;
+	wt_off_t avail_eighty, avail_ninety, eighty, ninety;
 
 	*skipp = 1;				/* Return a default skip. */
 
@@ -84,31 +86,53 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, int *skipp)
 	if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT))
 		WT_ERR(__block_dump_avail(session, block));
 
-	/* Sum the number of available bytes in the first 90% of the file. */
-	avail = 0;
+	/* Sum the available bytes in the first 80% and 90% of the file. */
+	avail_eighty = avail_ninety = 0;
 	ninety = fh->size - fh->size / 10;
+	eighty = fh->size - ((fh->size / 10) * 2);
 
 	el = &block->live.avail;
 	WT_EXT_FOREACH(ext, el->off)
-		if (ext->off < ninety)
-			avail += ext->size;
-
-	/*
-	 * If at least 10% of the total file is available and in the first 90%
-	 * of the file, we'll try compaction.
-	 */
-	if (avail >= fh->size / 10)
-		*skipp = 0;
+		if (ext->off < ninety) {
+			avail_ninety += ext->size;
+			if (ext->off < eighty)
+				avail_eighty += ext->size;
+		}
 
 	WT_ERR(__wt_verbose(session, WT_VERB_COMPACT,
 	    "%s: %" PRIuMAX "MB (%" PRIuMAX ") available space in the first "
-	    "90%% of the file, require 10%% or %" PRIuMAX "MB (%" PRIuMAX
-	    ") to perform compaction, compaction %s",
+	    "80%% of the file",
+	    block->name,
+	    (uintmax_t)avail_eighty / WT_MEGABYTE, (uintmax_t)avail_eighty));
+	WT_ERR(__wt_verbose(session, WT_VERB_COMPACT,
+	    "%s: %" PRIuMAX "MB (%" PRIuMAX ") available space in the first "
+	    "90%% of the file",
+	    block->name,
+	    (uintmax_t)avail_ninety / WT_MEGABYTE, (uintmax_t)avail_ninety));
+	WT_ERR(__wt_verbose(session, WT_VERB_COMPACT,
+	    "%s: require 10%% or %" PRIuMAX "MB (%" PRIuMAX ") in the first "
+	    "90%% of the file to perform compaction, compaction %s",
 	    block->name,
-	    (uintmax_t)avail / WT_MEGABYTE, (uintmax_t)avail,
 	    (uintmax_t)(fh->size / 10) / WT_MEGABYTE, (uintmax_t)fh->size / 10,
 	    *skipp ? "skipped" : "proceeding"));
 
+	/*
+	 * If at least 20% of the total file is available and in the first 80%
+	 * of the file, we'll try compaction on the last 20% of the file; else,
+	 * if at least 10% of the total file is available and in the first 90%
+	 * of the file, we'll try compaction on the last 10% of the file.
+	 *
+	 * We could push this further, but there's diminishing returns, a mostly
+	 * empty file can be processed quickly, so more aggressive compaction is
+	 * less useful.
+	 */
+	if (avail_ninety >= fh->size / 10) {
+		*skipp = 0;
+		block->compact_pct_tenths = 1;
+		if (avail_eighty >= ((fh->size / 10) * 2))
+			block->compact_pct_tenths = 2;
+	}
+
 err:	__wt_spin_unlock(session, &block->live_lock);
 
 	return (ret);
@@ -126,7 +150,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session,
 	WT_EXT *ext;
 	WT_EXTLIST *el;
 	WT_FH *fh;
-	wt_off_t ninety, offset;
+	wt_off_t limit, offset;
 	uint32_t size, cksum;
 
 	WT_UNUSED(addr_size);
@@ -138,21 +162,24 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session,
 	WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &cksum));
 
 	/*
-	 * If this block is in the last 10% of the file and there's a block on
-	 * the available list that's in the first 90% of the file, rewrite the
-	 * block.  Checking the available list is necessary (otherwise writing
-	 * the block would extend the file), but there's an obvious race if the
-	 * file is sufficiently busy.
+	 * If this block is in the chosen percentage of the file and there's a
+	 * block on the available list that's appears before that percentage of
+	 * the file, rewrite the block.  Checking the available list is
+	 * necessary (otherwise writing the block would extend the file), but
+	 * there's an obvious race if the file is sufficiently busy.
 	 */
 	__wt_spin_lock(session, &block->live_lock);
-	ninety = fh->size - fh->size / 10;
-	if (offset > ninety) {
+	limit = fh->size - ((fh->size / 10) * block->compact_pct_tenths);
+	if (offset > limit) {
 		el = &block->live.avail;
-		WT_EXT_FOREACH(ext, el->off)
-			if (ext->off < ninety && ext->size >= size) {
+		WT_EXT_FOREACH(ext, el->off) {
+			if (ext->off >= limit)
+				break;
+			if (ext->size >= size) {
 				*skipp = 0;
 				break;
 			}
+		}
 	}
 	__wt_spin_unlock(session, &block->live_lock);
 

src/third_party/wiredtiger/src/btree/bt_handle.c

@@ -659,7 +659,8 @@ __btree_page_sizes(WT_SESSION_IMPL *session)
 	 * cache, it may not have been set.
 	 */
 	WT_RET(__wt_config_gets(session, cfg, "memory_page_max", &cval));
-	btree->maxmempage = WT_MAX((uint64_t)cval.val, 50 * btree->maxleafpage);
+	btree->maxmempage =
+	    WT_MAX((uint64_t)cval.val, 50 * (uint64_t)btree->maxleafpage);
 	cache_size = S2C(session)->cache_size;
 	if (cache_size > 0)
 		btree->maxmempage = WT_MIN(btree->maxmempage, cache_size / 2);

src/third_party/wiredtiger/src/btree/bt_split.c

@@ -13,7 +13,7 @@
  * estimate of allocation overhead to every object.
  */
 #define	WT_MEMSIZE_ADD(total, len)	do {				\
-	total += (len) + WT_ALLOC_OVERHEAD;				\
+	total += (len);							\
 } while (0)
 #define	WT_MEMSIZE_TRANSFER(from_decr, to_incr, len) do {		\
 	WT_MEMSIZE_ADD(from_decr, len);					\
@@ -728,14 +728,6 @@ __split_multi_inmem(
 	 */
 	page->modify->first_dirty_txn = WT_TXN_FIRST;
 
-	/*
-	 * XXX Don't allow this page to be evicted immediately.
-	 *
-	 * In some cases involving forced eviction during truncates, a reader
-	 * ends up looking at an evicted page.  This is a temporary workaround.
-	 */
-	page->modify->inmem_split_txn = __wt_txn_new_id(session);
-
 err:	/* Free any resources that may have been cached in the cursor. */
 	WT_TRET(__wt_btcur_close(&cbt));
 
@@ -889,8 +881,7 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 	for (i = 0, deleted_entries = 0; i < parent_entries; ++i) {
 		next_ref = pindex->index[i];
 		WT_ASSERT(session, next_ref->state != WT_REF_SPLIT);
-		if (next_ref->state == WT_REF_DELETED &&
-		    next_ref->page_del == NULL &&
+		if (__wt_delete_page_skip(session, next_ref) &&
 		    WT_ATOMIC_CAS4(next_ref->state,
 		    WT_REF_DELETED, WT_REF_SPLIT))
 			deleted_entries++;
@@ -986,6 +977,18 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new,
 					    session, 0, ikey, size));
 					WT_MEMSIZE_ADD(parent_decr, size);
 				}
+				/*
+				 * The page_del structure can be freed
+				 * immediately: it is only read when the ref
+				 * state is WT_REF_DELETED.  The size of the
+				 * structures wasn't added to the parent: don't
+				 * decrement.
+				 */
+				if (next_ref->page_del != NULL) {
+					__wt_free(session,
+					    next_ref->page_del->update_list);
+					__wt_free(session, next_ref->page_del);
+				}
 			}
 
 			WT_TRET(__split_safe_free(