Bug#21221500: ADD SUPPORT FOR GCC __ATOMIC BUILTINS

This patch adds support for the GCC __atomic builtins introduced
in GCC 4.7. These new builtins allows a more efficient
implementation of the MySQL atomics API. Most importantly the
new builtins include functions for atomic load, so that
my_atomic_load can be implemented as a real load instead of a store.

The new __atomic builtins are only used for platforms
where the old GCC __sync builtins are not available.
In particular this includes platforms supported by Ubuntu
(such as powerpc) where the server would otherwise not build.

Author: jhauglid

cmake/os/WindowsCache.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -210,6 +210,7 @@ SET(HAVE_BUILTIN_UNREACHABLE CACHE  INTERNAL "")
 SET(HAVE_BUILTIN_EXPECT CACHE  INTERNAL "")
 SET(HAVE_BUILTIN_STPCPY CACHE  INTERNAL "")
 SET(HAVE_GCC_ATOMIC_BUILTINS CACHE  INTERNAL "")
+SET(HAVE_GCC_SYNC_BUILTINS CACHE  INTERNAL "")
 # Derived result HAVE_VALGRIND
 
 # IPV6

config.h.cmake

@@ -198,6 +198,7 @@
 #cmakedefine HAVE_BUILTIN_EXPECT 1
 #cmakedefine HAVE_BUILTIN_STPCPY 1
 #cmakedefine HAVE_GCC_ATOMIC_BUILTINS 1
+#cmakedefine HAVE_GCC_SYNC_BUILTINS 1
 #cmakedefine HAVE_VALGRIND
 
 /* IPV6 */

configure.cmake

@@ -750,6 +750,33 @@ int main()
 }" HAVE_BUILTIN_STPCPY)
 ENDIF()
 
+CHECK_CXX_SOURCE_COMPILES("
+  int main()
+  {
+    int foo= -10; int bar= 10;
+    long long int foo64= -10; long long int bar64= 10;
+    if (!__atomic_fetch_add(&foo, bar, __ATOMIC_SEQ_CST) || foo)
+      return -1;
+    bar= __atomic_exchange_n(&foo, bar, __ATOMIC_SEQ_CST);
+    if (bar || foo != 10)
+      return -1;
+    bar= __atomic_compare_exchange_n(&bar, &foo, 15, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    if (bar)
+      return -1;
+    if (!__atomic_fetch_add(&foo64, bar64, __ATOMIC_SEQ_CST) || foo64)
+      return -1;
+    bar64= __atomic_exchange_n(&foo64, bar64, __ATOMIC_SEQ_CST);
+    if (bar64 || foo64 != 10)
+      return -1;
+    bar64= __atomic_compare_exchange_n(&bar64, &foo64, 15, 0,
+                                       __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    if (bar64)
+      return -1;
+    return 0;
+  }"
+  HAVE_GCC_ATOMIC_BUILTINS)
+
 CHECK_CXX_SOURCE_COMPILES("
   int main()
   {
@@ -773,7 +800,7 @@ CHECK_CXX_SOURCE_COMPILES("
       return -1;
     return 0;
   }"
-  HAVE_GCC_ATOMIC_BUILTINS)
+  HAVE_GCC_SYNC_BUILTINS)
 
 IF(WITH_VALGRIND)
   SET(VALGRIND_HEADERS "valgrind/memcheck.h;valgrind/valgrind.h")

include/atomic/gcc_atomic.h

@@ -0,0 +1,94 @@
+#ifndef GCC_ATOMIC_INCLUDED
+#define GCC_ATOMIC_INCLUDED
+
+/* Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+/* New GCC __atomic builtins introduced in GCC 4.7 */
+
+static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
+{
+  return __atomic_compare_exchange_n(a, cmp, set, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set)
+{
+  return __atomic_compare_exchange_n(a, cmp, set, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set)
+{
+  return __atomic_compare_exchange_n(a, cmp, set, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+static inline int32 my_atomic_add32(int32 volatile *a, int32 v)
+{
+  return __atomic_fetch_add(a, v, __ATOMIC_SEQ_CST);
+}
+
+static inline int64 my_atomic_add64(int64 volatile *a, int64 v)
+{
+  return __atomic_fetch_add(a, v, __ATOMIC_SEQ_CST);
+}
+
+static inline int32 my_atomic_fas32(int32 volatile *a, int32 v)
+{
+  return __atomic_exchange_n(a, v, __ATOMIC_SEQ_CST);
+}
+
+static inline int64 my_atomic_fas64(int64 volatile *a, int64 v)
+{
+  return __atomic_exchange_n(a, v, __ATOMIC_SEQ_CST);
+}
+
+static inline void * my_atomic_fasptr(void * volatile *a, void * v)
+{
+  return __atomic_exchange_n(a, v, __ATOMIC_SEQ_CST);
+}
+
+static inline int32 my_atomic_load32(int32 volatile *a)
+{
+  return __atomic_load_n(a, __ATOMIC_SEQ_CST);
+}
+
+static inline int64 my_atomic_load64(int64 volatile *a)
+{
+  return __atomic_load_n(a, __ATOMIC_SEQ_CST);
+}
+
+static inline void* my_atomic_loadptr(void * volatile *a)
+{
+  return __atomic_load_n(a, __ATOMIC_SEQ_CST);
+}
+
+static inline void my_atomic_store32(int32 volatile *a, int32 v)
+{
+  __atomic_store_n(a, v, __ATOMIC_SEQ_CST);
+}
+
+static inline void my_atomic_store64(int64 volatile *a, int64 v)
+{
+  __atomic_store_n(a, v, __ATOMIC_SEQ_CST);
+}
+
+static inline void my_atomic_storeptr(void * volatile *a, void *v)
+{
+  __atomic_store_n(a, v, __ATOMIC_SEQ_CST);
+}
+
+#endif /* GCC_ATOMIC_INCLUDED */

include/atomic/gcc_builtins.h renamed to include/atomic/gcc_sync.h

@@ -1,7 +1,7 @@
-#ifndef ATOMIC_GCC_BUILTINS_INCLUDED
-#define ATOMIC_GCC_BUILTINS_INCLUDED
+#ifndef GCC_SYNC_INCLUDED
+#define GCC_SYNC_INCLUDED
 
-/* Copyright (c) 2008, 2014, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -16,6 +16,8 @@
    along with this program; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
 
+/* Old GCC __sync builtins introduced in GCC 4.1 */
+
 static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set)
 {
   int32 cmp_val= *cmp;
@@ -101,4 +103,4 @@ static inline void my_atomic_storeptr(void * volatile *a, void *v)
   (void) __sync_lock_test_and_set(a, v);
 }
 
-#endif /* ATOMIC_GCC_BUILTINS_INCLUDED */
+#endif /* GCC_SYNC_INCLUDED */

include/my_atomic.h

@@ -1,7 +1,7 @@
 #ifndef MY_ATOMIC_INCLUDED
 #define MY_ATOMIC_INCLUDED
 
-/* Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -56,8 +56,10 @@
 #  include "atomic/generic-msvc.h"
 #elif defined(HAVE_SOLARIS_ATOMIC)
 #  include "atomic/solaris.h"
-#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
-#  include "atomic/gcc_builtins.h"
+#elif defined(HAVE_GCC_SYNC_BUILTINS)   /* Use old __sync by default */
+#  include "atomic/gcc_sync.h"
+#elif defined(HAVE_GCC_ATOMIC_BUILTINS) /* Use __atomic on e.g. powerpc */
+#  include "atomic/gcc_atomic.h"
 #else
 #  error Native atomics support not found!
 #endif

mysys/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved.
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -66,7 +66,7 @@ ENDIF()
 
 ADD_CONVENIENCE_LIBRARY(mysys ${MYSYS_SOURCES})
 TARGET_LINK_LIBRARIES(mysys dbug strings ${ZLIB_LIBRARY} 
- ${LIBNSL} ${LIBM} ${LIBRT})
+ ${LIBNSL} ${LIBM} ${LIBRT} ${LIBATOMIC})
 DTRACE_INSTRUMENT(mysys)
 
 # Need explicit pthread for gcc -fsanitize=address

plugin/innodb_memcached/daemon_memcached/daemon/memcached.c

@@ -3,10 +3,10 @@
  *  memcached - memory caching daemon
  *
  *       http://www.danga.com/memcached/
- *  Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ *  Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved.
  *  Copyright 2003 Danga Interactive, Inc.  All rights reserved.
- *  This file was modified by Oracle on 28-08-2015.
- *  Modifications copyright (c) 2015, Oracle and/or its affiliates.
+ *  This file was modified by Oracle on 28-08-2015 and 23-03-2016.
+ *  Modifications copyright (c) 2015, 2016, Oracle and/or its affiliates.
  *  All rights reserved.
  *
  *  Use and distribution licensed under the BSD license.  See
@@ -89,7 +89,7 @@ static inline void item_set_cas(const void *cookie, item *it, uint64_t cas) {
 #define STATS_MISS(conn, op, key, nkey) \
     STATS_TWO(conn, op##_misses, cmd_##op, key, nkey)
 
-#if defined(HAVE_GCC_ATOMIC_BUILTINS)
+#if defined(HAVE_GCC_SYNC_BUILTINS)
 
 #define STATS_NOKEY(conn, op)	\
 do { \
@@ -115,7 +115,7 @@ do { \
 
 #define MEMCACHED_ATOMIC_MSG	"InnoDB MEMCACHED: Memcached uses atomic increment \n"
 
-#else /* HAVE_GCC_ATOMIC_BUILTINS */
+#else /* HAVE_GCC_SYNC_BUILTINS */
 #define STATS_NOKEY(conn, op) { \
     struct thread_stats *thread_stats = \
         get_thread_stats(conn); \
@@ -142,7 +142,7 @@ do { \
 }
 
 #define MEMCACHED_ATOMIC_MSG	"InnoDB Memcached: Memcached DOES NOT use atomic increment"
-#endif /* HAVE_GCC_ATOMIC_BUILTINS */
+#endif /* HAVE_GCC_SYNC_BUILTINS */
 
 volatile sig_atomic_t memcached_shutdown;
 volatile sig_atomic_t memcached_initialized;

plugin/innodb_memcached/innodb_memcache/src/innodb_api.c

@@ -938,7 +938,7 @@ mci_get_cas(
 {
 	static uint64_t cas_id = 0;
 
-#if defined(HAVE_GCC_ATOMIC_BUILTINS)
+#if defined(HAVE_GCC_SYNC_BUILTINS)
 	return(__sync_add_and_fetch(&cas_id, 1));
 #else
 	pthread_mutex_lock(&eng->cas_mutex);

storage/innobase/include/os0atomic.h

@@ -1,5 +1,5 @@
 /*****************************************************************************
-Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, Google Inc.
 
 Portions of this file contain modifications contributed and copyrighted by
@@ -208,6 +208,8 @@ amount to decrement. There is no atomic substract function on Windows */
 Returns true if swapped, ptr is pointer to target, old_val is value to
 compare to, new_val is the value to swap in. */
 
+#if defined(HAVE_GCC_SYNC_BUILTINS)
+
 # define os_compare_and_swap(ptr, old_val, new_val) \
 	__sync_bool_compare_and_swap(ptr, old_val, new_val)
 
@@ -220,9 +222,47 @@ compare to, new_val is the value to swap in. */
 # define os_compare_and_swap_uint32(ptr, old_val, new_val) \
 	os_compare_and_swap(ptr, old_val, new_val)
 
+#else
+
+UNIV_INLINE
+bool
+os_compare_and_swap_ulint(volatile ulint* ptr, ulint old_val, ulint new_val)
+{
+  return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+UNIV_INLINE
+bool
+os_compare_and_swap_lint(volatile lint* ptr, lint old_val, lint new_val)
+{
+  return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
+				     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+UNIV_INLINE
+bool
+os_compare_and_swap_uint32(volatile ib_uint32_t* ptr, ib_uint32_t old_val, ib_uint32_t new_val)
+{
+  return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+#endif /* HAVE_GCC_SYNC_BUILTINS */
+
 # ifdef HAVE_IB_ATOMIC_PTHREAD_T_GCC
+#if defined(HAVE_GCC_SYNC_BUILTINS)
 #  define os_compare_and_swap_thread_id(ptr, old_val, new_val) \
 	os_compare_and_swap(ptr, old_val, new_val)
+#else
+UNIV_INLINE
+bool
+os_compare_and_swap_thread_id(volatile os_thread_id_t* ptr, os_thread_id_t old_val, os_thread_id_t new_val)
+{
+  return __atomic_compare_exchange_n(ptr, &old_val, new_val, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+#endif /* HAVE_GCC_SYNC_BUILTINS */
 #  define INNODB_RW_LOCKS_USE_ATOMICS
 #  define IB_ATOMICS_STARTUP_MSG \
 	"Mutexes and rw_locks use GCC atomic builtins"
@@ -235,8 +275,13 @@ compare to, new_val is the value to swap in. */
 Returns the resulting value, ptr is pointer to target, amount is the
 amount of increment. */
 
+#if defined(HAVE_GCC_SYNC_BUILTINS)
 # define os_atomic_increment(ptr, amount) \
 	__sync_add_and_fetch(ptr, amount)
+#else
+# define os_atomic_increment(ptr, amount) \
+	__atomic_add_fetch(ptr, amount, __ATOMIC_SEQ_CST)
+#endif /* HAVE_GCC_SYNC_BUILTINS */
 
 # define os_atomic_increment_lint(ptr, amount) \
 	os_atomic_increment(ptr, amount)
@@ -253,8 +298,13 @@ amount of increment. */
 /* Returns the resulting value, ptr is pointer to target, amount is the
 amount to decrement. */
 
+#if defined(HAVE_GCC_SYNC_BUILTINS)
 # define os_atomic_decrement(ptr, amount) \
 	__sync_sub_and_fetch(ptr, amount)
+#else
+# define os_atomic_decrement(ptr, amount) \
+	__atomic_sub_fetch(ptr, amount, __ATOMIC_SEQ_CST)
+#endif /* HAVE_GCC_SYNC_BUILTINS */
 
 # define os_atomic_decrement_lint(ptr, amount) \
 	os_atomic_decrement(ptr, amount)