WL#16938 Startup option server_memory

Description:
  The MySQL Server auto-tunes the default values of certain
configuration parameters based on the total physical memory available to
the server. This information may come from the cgroup configuration or
from the system hardware configuration.
  This worklog allows the user to configure the amount of memory
considered by the server when determining the default values. It adds a
system variable called server_memory as a read_only non_persistent
variable with a default value of 0 indiciating no limits set.
  In case the server is configured to use more memory than what the
server can access, the server throws an error during bootstrap and
halts.

Note:
  If the value is not provided, server_memory will default to 0. When
the value is 0 (either default or manually set to 0), the server will
fallback to available physical memory using cgroup or hardware as
implemented in WL#16484.

Change-Id: I39310955d952fef2b65a767c74481660daef4f2c

Author: ram1048

components/library_mysys/my_system.cc

@@ -48,7 +48,17 @@
 #include <unistd.h>
 #endif
 
-uint64_t my_physical_memory() noexcept {
+namespace {
+ulonglong configured_memory{0};
+}  // anonymous namespace
+
+/**
+  Get the total physical memory accessible to the server. Tries to read cgroup
+  limits first. If failed or no limits were found, then tries to read from the
+  system using corresponding system APIs.
+  @return  total physical memory available to the server, or 0 on failure
+*/
+static inline uint64_t total_physical_memory() noexcept {
   try {
     uint64_t mem = 0;
 #ifdef _WIN32
@@ -77,6 +87,26 @@ uint64_t my_physical_memory() noexcept {
   }
 }
 
+bool init_my_physical_memory(ulonglong memory) {
+  if (memory == 0) {
+    /* No limit. Physical memory available should be returned */
+    return true;
+  }
+  if (memory <= total_physical_memory()) {
+    configured_memory = memory;
+    return true;
+  }
+
+  return false;
+}
+
+uint64_t my_physical_memory() noexcept {
+  if (configured_memory != 0) {
+    return configured_memory;
+  }
+  return total_physical_memory();
+}
+
 uint32_t my_num_vcpus() noexcept {
   try {
     uint32_t n_vcpus = 0;

include/mysql/components/library_mysys/my_system.h

@@ -25,6 +25,18 @@
 
 #include <cstdint>
 
+using ulonglong = unsigned long long;
+
+/**
+  Initialize the my_physical_memory function using server_memory option
+  @param[in]  memory  Value of the server_memory startup option
+
+  @note The input value of 0 indicates no limits, and underlying container/host
+  configuration must be used
+  @return true on success, false if input memory value is invalid
+*/
+bool init_my_physical_memory(ulonglong memory);
+
 /**
   Determine the total physical memory available in bytes.
 

mysql-test/r/mysqld--help-notwin.result

@@ -1328,6 +1328,9 @@ The following options may be given as the first argument:
  --server-id=#       Uniquely identifies the server instance in the community
  of replication partners
  --server-id-bits=#  Set number of significant bits in server-id
+ --server-memory=#   Memory (in bytes) used by the MySQL Server when
+ auto-tuning the default values of the configuration
+ parameters which depend on memory
  --session-track-gtids=name 
  Controls the amount of global transaction ids to be
  included in the response packet sent by the
@@ -1960,6 +1963,7 @@ select-into-disk-sync FALSE
 select-into-disk-sync-delay 0
 server-id 1
 server-id-bits 32
+server-memory 0
 session-track-gtids OFF
 session-track-schema TRUE
 session-track-state-change FALSE

mysql-test/r/mysqld--help-win.result

@@ -1331,6 +1331,9 @@ The following options may be given as the first argument:
  --server-id=#       Uniquely identifies the server instance in the community
  of replication partners
  --server-id-bits=#  Set number of significant bits in server-id
+ --server-memory=#   Memory (in bytes) used by the MySQL Server when
+ auto-tuning the default values of the configuration
+ parameters which depend on memory
  --session-track-gtids=name 
  Controls the amount of global transaction ids to be
  included in the response packet sent by the
@@ -1972,6 +1975,7 @@ select-into-disk-sync FALSE
 select-into-disk-sync-delay 0
 server-id 1
 server-id-bits 32
+server-memory 0
 session-track-gtids OFF
 session-track-schema TRUE
 session-track-state-change FALSE

mysql-test/suite/sys_vars/r/server_memory_basic.result

@@ -0,0 +1,48 @@
+select @@global.server_memory;
+@@global.server_memory
+0
+select @@session.server_memory;
+ERROR HY000: Variable 'server_memory' is a GLOBAL variable
+show global variables like 'server_memory';
+Variable_name	Value
+server_memory	0
+show session variables like 'server_memory';
+Variable_name	Value
+server_memory	0
+select * from performance_schema.global_variables where variable_name='server_memory';
+VARIABLE_NAME	VARIABLE_VALUE
+server_memory	0
+select * from performance_schema.session_variables where variable_name='server_memory';
+VARIABLE_NAME	VARIABLE_VALUE
+server_memory	0
+set global server_memory=1G;
+ERROR HY000: Variable 'server_memory' is a read only variable
+set session server_memory=1G;
+ERROR HY000: Variable 'server_memory' is a read only variable
+set persist server_memory=1G;
+ERROR HY000: Variable 'server_memory' is a read only variable
+set persist_only server_memory=1G;
+ERROR HY000: Variable 'server_memory' is a non persistent read only variable
+# restart: --server-memory=512M
+select @@server_memory;
+@@server_memory
+536870912
+select * from performance_schema.global_variables where variable_name='server_memory';
+VARIABLE_NAME	VARIABLE_VALUE
+server_memory	536870912
+select * from performance_schema.session_variables where variable_name='server_memory';
+VARIABLE_NAME	VARIABLE_VALUE
+server_memory	536870912
+# restart: --server-memory=512M
+call mtr.add_suppression("Warning.* option 'server_memory': value -1 adjusted to 0.");
+select @@server_memory;
+@@server_memory
+536870912
+select * from performance_schema.global_variables where variable_name='server_memory';
+VARIABLE_NAME	VARIABLE_VALUE
+server_memory	536870912
+select * from performance_schema.session_variables where variable_name='server_memory';
+VARIABLE_NAME	VARIABLE_VALUE
+server_memory	536870912
+# Cleanup by restart
+# restart

mysql-test/suite/sys_vars/t/server_memory_basic.test

@@ -0,0 +1,65 @@
+# Variable server_memory indicates base memory value to be used by the server
+# when auto-tuning default values of configuration parameters
+# Note: Value of 0 implies no limits, read from cgroup/hardware
+# Note: Very small value implies, configurations defaults to minimum value
+
+#
+# show that it's global
+#
+select @@global.server_memory;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+select @@session.server_memory;
+show global variables like 'server_memory';
+show session variables like 'server_memory';
+--disable_warnings
+select * from performance_schema.global_variables where variable_name='server_memory';
+select * from performance_schema.session_variables where variable_name='server_memory';
+--enable_warnings
+
+#
+# show that it's read-only
+#
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set global server_memory=1G;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set session server_memory=1G;
+
+#
+# show that it's non persistent
+#
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set persist server_memory=1G;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set persist_only server_memory=1G;
+
+#
+# Set the option and verify that it is reflected
+#
+--let $restart_parameters=restart: --server-memory=512M
+--source include/restart_mysqld.inc
+
+select @@server_memory;
+
+--disable_warnings
+select * from performance_schema.global_variables where variable_name='server_memory';
+select * from performance_schema.session_variables where variable_name='server_memory';
+--enable_warnings
+
+#
+# Value less than 0 is adjusted to 0
+#
+--let $restart_parameters=restart: --server-memory=512M
+--source include/restart_mysqld.inc
+call mtr.add_suppression("Warning.* option 'server_memory': value -1 adjusted to 0.");
+
+select @@server_memory;
+
+--disable_warnings
+select * from performance_schema.global_variables where variable_name='server_memory';
+select * from performance_schema.session_variables where variable_name='server_memory';
+--enable_warnings
+
+--echo # Cleanup by restart
+--let $restart_parameters=
+--source include/restart_mysqld.inc
+

share/messages_to_error_log.txt

@@ -13143,6 +13143,9 @@ ER_SERVER_STARTING_WITH_RESOURCE
 ER_INFO_AUTHENTICATION_OCI
    eng "authentication_oci plugin reported: %s."
 
+ER_SERVER_MEMORY_OPTION_INVALID
+   eng "MySQL Server configured to use %llu bytes but can only access %llu bytes."
+
 ################################################################################
 # Error numbers 50000 to 51999 are reserved. Please do not use them for
 # other error messages.

sql/mysqld.cc

@@ -1191,6 +1191,7 @@ char *my_bind_addr_str;
 char *my_admin_bind_addr_str;
 uint mysqld_admin_port;
 bool listen_admin_interface_in_separate_thread;
+ulonglong server_memory;
 static const char *default_collation_name;
 const char *default_storage_engine;
 const char *default_tmp_storage_engine;
@@ -6742,6 +6743,21 @@ int init_common_variables() {
 #endif
 
   if (get_options(&remaining_argc, &remaining_argv)) return 1;
+
+  /* Adjust memory to be used based on "server_memory" iff explicitly provided
+   */
+  if (server_memory > 0) {
+    if (!init_my_physical_memory(server_memory)) {
+      LogErr(ERROR_LEVEL, ER_SERVER_MEMORY_OPTION_INVALID, server_memory,
+             my_physical_memory());
+      return 1;
+    }
+
+    /* temptable_max_ram default is set according physical memory, needs to be
+  updated based on "server_memory" now */
+    update_temptable_max_ram_default();
+  }
+
   /*
     The opt_bin_log can be false (binary log is disabled) only if
     --skip-log-bin/--disable-log-bin is configured or while the

sql/mysqld.h

@@ -280,6 +280,7 @@ extern char *my_bind_addr_str;
 extern char *my_admin_bind_addr_str;
 extern uint mysqld_admin_port;
 extern bool listen_admin_interface_in_separate_thread;
+extern ulonglong server_memory;
 extern char glob_hostname[HOSTNAME_LENGTH + 1];
 extern char system_time_zone_dst_on[30], system_time_zone_dst_off[30];
 extern char *opt_init_file;

sql/sys_vars.cc

@@ -1191,6 +1191,13 @@ static Sys_var_bool Sys_use_separate_thread_for_admin(
     READ_ONLY NON_PERSIST GLOBAL_VAR(listen_admin_interface_in_separate_thread),
     CMD_LINE(OPT_ARG), DEFAULT(false));
 
+static Sys_var_ulonglong Sys_server_memory(
+    "server_memory",
+    "Memory (in bytes) used by the MySQL Server when auto-tuning the default "
+    "values of the configuration parameters which depend on memory",
+    READ_ONLY NON_PERSIST GLOBAL_VAR(server_memory), CMD_LINE(REQUIRED_ARG),
+    VALID_RANGE(0, ULLONG_MAX), DEFAULT(0), BLOCK_SIZE(1));
+
 static Sys_var_bool Sys_password_require_current(
     "password_require_current",
     "Current password is needed to be specified in order to change it",
@@ -5174,6 +5181,25 @@ static Sys_var_ulonglong Sys_temptable_max_ram(
                        1ULL << 30 /* 1 GiB */, 1ULL << 32 /* 4 GiB */)),
     BLOCK_SIZE(1));
 
+void update_temptable_max_ram_default() {
+  mysql_mutex_lock(&LOCK_global_system_variables);
+
+  assert(server_memory > 0);
+
+  /* Auto tune default value based on "server_memory" */
+  const ulonglong new_default =
+      std::clamp(ulonglong{3 * (my_physical_memory() / 100)},
+                 1ULL << 30 /* 1 GiB */, 1ULL << 32 /* 4 GiB */);
+  Sys_temptable_max_ram.update_default(new_default);
+
+  /* If "temptable_max_ram" is not set explicitly, update the default value */
+  if (Sys_temptable_max_ram.get_source() == COMPILED) {
+    temptable_max_ram = new_default;
+  }
+
+  mysql_mutex_unlock(&LOCK_global_system_variables);
+}
+
 static Sys_var_ulonglong Sys_temptable_max_mmap(
     "temptable_max_mmap",
     "Maximum amount of memory (in bytes) the TempTable storage engine is "