From 4d931ff183955f879002ba154e74068138b8ed82 Mon Sep 17 00:00:00 2001 From: liangwenjia <1757640220@qq.com> Date: Mon, 4 Mar 2024 15:15:32 +0800 Subject: [PATCH] add the foreground thread configuration changes take effect immediately --- plugin/thread_pool/numa_affinity_manager.cc | 88 +++++++++++++++++++-- plugin/thread_pool/numa_affinity_manager.h | 9 +++ plugin/thread_pool/threadpool_common.cc | 40 ++++++---- plugin/thread_pool/threadpool_unix.cc | 6 +- 4 files changed, 117 insertions(+), 26 deletions(-) diff --git a/plugin/thread_pool/numa_affinity_manager.cc b/plugin/thread_pool/numa_affinity_manager.cc index de1b19969..e8563a6b9 100644 --- a/plugin/thread_pool/numa_affinity_manager.cc +++ b/plugin/thread_pool/numa_affinity_manager.cc @@ -88,6 +88,13 @@ bool numa_affinity_manager::init_foreground_cpu_map() { } bool numa_affinity_manager::bind_foreground_thread(int group_id) { + if (!threadpool_sched_affinity) { + XLockGuard lk(lock); + my_thread_os_id_t tid = my_thread_os_id(); + foreground_threads.insert(std::pair(tid, -1)); + return false; + } + // for bind the purge thread if (!bind_background_ok) { XLockGuard lk(lock); @@ -98,13 +105,23 @@ bool numa_affinity_manager::bind_foreground_thread(int group_id) { } } - SLockGuard lk(lock); + bool ret = true; + XLockGuard lk(lock); if (initok) { - my_thread_os_id_t pid = my_thread_os_id(); + my_thread_os_id_t tid = my_thread_os_id(); bitmask *msk = foreground_cpu_map[group_id%foreground_cpu_map.size()].get(); - return (bind_thread(pid, msk) == 0); + ret = bind_thread(tid, msk); + if (ret) { + foreground_threads.insert(std::pair(tid, group_id)); + } } - return false; + return !ret; +} + +void numa_affinity_manager::update_bind_foreground_threads() { + XLockGuard lk(lock); + bind_foreground_threads(); + print_state(); } void numa_affinity_manager::update_bind_background_threads() { @@ -113,6 +130,13 @@ void numa_affinity_manager::update_bind_background_threads() { print_state(); } +void numa_affinity_manager::update_bind_threads() { + XLockGuard lk(lock); + bind_foreground_threads(); + bind_background_threads(); + print_state(); +} + void numa_affinity_manager::print_state() { #ifndef NDEBUG // assert has own lock of s or x @@ -165,11 +189,11 @@ int numa_affinity_manager::get_sys_numa() { return numa_num_configured_nodes(); } -bool numa_affinity_manager::bind_thread(my_thread_os_id_t pid, bitmask *msk) { +bool numa_affinity_manager::bind_thread(my_thread_os_id_t tid, bitmask *msk) { bool ret = false; int i = 0; for (i = 0; i < 10; i++) { - if (numa_sched_setaffinity(pid, msk) == 0) { + if (numa_sched_setaffinity(tid, msk) == 0) { ret = true; break; } @@ -177,7 +201,7 @@ bool numa_affinity_manager::bind_thread(my_thread_os_id_t pid, bitmask *msk) { std::this_thread::yield(); } std::cout << "bind thread " << (ret? "succeed":"failed") << ": " - << pid << " - " << cpumask_to_string(msk) << std::endl; + << tid << " - " << cpumask_to_string(msk) << std::endl; return ret; } @@ -335,6 +359,19 @@ void numa_affinity_manager::fetch_background_threads() { } } +void numa_affinity_manager::bind_foreground_threads() +{ + init_foreground_cpu_map(); + if (threadpool_sched_affinity) { + for (auto it = foreground_threads.begin(); it != foreground_threads.end(); it++) { + auto tid = it->first; + int group_id = it->second; + bitmask *msk = foreground_cpu_map[group_id%foreground_cpu_map.size()].get(); + bind_thread(tid, msk); + } + } +} + void numa_affinity_manager::bind_background_threads() { fetch_background_threads(); @@ -343,3 +380,40 @@ void numa_affinity_manager::bind_background_threads() apply_background_options(); } } + +void numa_affinity_manager::remove_foreground_thread() +{ + my_thread_os_id_t tid = my_thread_os_id(); + XLockGuard lk(lock); + foreground_threads.erase(tid); +} + +void numa_affinity_manager::unbind_foreground_threads() +{ + auto msk = numa_bitmask_alloc(cpu_count); + numa_bitmask_setall(msk); + for (auto it = foreground_threads.begin(); it != foreground_threads.end(); it++) { + auto tid = it->first; + bind_thread(tid, msk); + } + free_cpumask_func(msk); +} + +void numa_affinity_manager::unbind_background_threads() +{ + auto msk = numa_bitmask_alloc(cpu_count); + numa_bitmask_setall(msk); + for (auto &it : background_threads) { + for (auto tid : it.second) { + bind_thread(tid, msk); + } + } + free_cpumask_func(msk); +} + +void numa_affinity_manager::unbind_threads() +{ + SLockGuard lk(lock); + unbind_foreground_threads(); + unbind_background_threads(); +} \ No newline at end of file diff --git a/plugin/thread_pool/numa_affinity_manager.h b/plugin/thread_pool/numa_affinity_manager.h index c139a641f..e3c3eac31 100644 --- a/plugin/thread_pool/numa_affinity_manager.h +++ b/plugin/thread_pool/numa_affinity_manager.h @@ -34,7 +34,13 @@ public: bool init(); bool init_foreground_cpu_map(); bool bind_foreground_thread(int group_id); + void update_bind_threads(); + void update_bind_foreground_threads(); void update_bind_background_threads(); + void unbind_threads(); + void remove_foreground_thread(); + void unbind_foreground_threads(); + void unbind_background_threads(); void print_state(); static void free_cpumask_func(bitmask *ptr); @@ -42,6 +48,7 @@ public: static int get_sys_cpu_only(); static int get_sys_numa(); static bool bind_thread(my_thread_os_id_t id, bitmask *msk); + static bool unbind_thread(my_thread_os_id_t id); static bool check_cpustring(const char *opt); static const std::string cpumask_to_string(bitmask *msk); static bool normalize_cpustring(const std::string &cpu_string, std::string &out); @@ -51,6 +58,7 @@ protected: void apply_background_options(); void fetch_background_options(); void fetch_background_threads(); + void bind_foreground_threads(); void bind_background_threads(); @@ -64,6 +72,7 @@ private: std::map > background_cpu_map_opts; bool bind_background_ok{false}; std::map > background_threads; + std::map foreground_threads; }; #endif // NUMA_AFFINITY_MANAGER_H_ diff --git a/plugin/thread_pool/threadpool_common.cc b/plugin/thread_pool/threadpool_common.cc index 9084b2a3d..ce9dfb657 100644 --- a/plugin/thread_pool/threadpool_common.cc +++ b/plugin/thread_pool/threadpool_common.cc @@ -392,17 +392,25 @@ static int check_fix_sched_affinity_cpustring(MYSQL_THD thd, SYS_VAR *, void *sa return 0; } -static void fix_sched_affinity_cpustring(MYSQL_THD, SYS_VAR *, void *var_ptr, const void *save) { +static void fix_sched_affinity_background_cpustring(MYSQL_THD, SYS_VAR *, void *var_ptr, const void *save) { const char *new_option_val = *static_cast(save); *static_cast(var_ptr) = new_option_val; group_affinity.update_bind_background_threads(); } +static void fix_sched_affinity_foreground_cpustring(MYSQL_THD, SYS_VAR *, void *var_ptr, const void *save) { + const char *new_option_val = *static_cast(save); + *static_cast(var_ptr) = new_option_val; + group_affinity.update_bind_foreground_threads(); +} + static void fix_sched_affinity(MYSQL_THD, SYS_VAR *, void *var_ptr, const void *save) { bool newval = *static_cast(save); *static_cast(var_ptr) = newval; if (threadpool_sched_affinity) { - group_affinity.init(); + group_affinity.update_bind_threads(); + } else { + group_affinity.unbind_threads(); } } @@ -416,50 +424,50 @@ static MYSQL_SYSVAR_BOOL(sched_affinity, threadpool_sched_affinity, static MYSQL_SYSVAR_STR(sched_affinity_foreground_thread, threadpool_sched_affinity_foreground_thread, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "The set of cpus which foreground threads will run on.", - check_fix_sched_affinity_cpustring, // check func - fix_sched_affinity_cpustring, // update func + check_fix_sched_affinity_cpustring, // check func + fix_sched_affinity_foreground_cpustring, // update func nullptr); // default value static MYSQL_SYSVAR_STR(sched_affinity_log_writer, threadpool_sched_affinity_log_writer, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "log_writer", - check_fix_sched_affinity_cpustring, // check func - fix_sched_affinity_cpustring, // update func + check_fix_sched_affinity_cpustring, // check func + fix_sched_affinity_background_cpustring, // update func nullptr); // default value static MYSQL_SYSVAR_STR(sched_affinity_log_flusher, threadpool_sched_affinity_log_flusher, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "log_flusher", - check_fix_sched_affinity_cpustring, // check func - fix_sched_affinity_cpustring, // update func + check_fix_sched_affinity_cpustring, // check func + fix_sched_affinity_background_cpustring, // update func nullptr); // default value static MYSQL_SYSVAR_STR(sched_affinity_log_write_notifier, threadpool_sched_affinity_log_write_notifier, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "log_write_notifier", - check_fix_sched_affinity_cpustring, // check func - fix_sched_affinity_cpustring, // update func + check_fix_sched_affinity_cpustring, // check func + fix_sched_affinity_background_cpustring, // update func nullptr); // default value static MYSQL_SYSVAR_STR(sched_affinity_log_flush_notifier, threadpool_sched_affinity_log_flush_notifier, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "log_flush_notifier", - check_fix_sched_affinity_cpustring, // check func - fix_sched_affinity_cpustring, // update func + check_fix_sched_affinity_cpustring, // check func + fix_sched_affinity_background_cpustring, // update func nullptr); // default value static MYSQL_SYSVAR_STR(sched_affinity_log_checkpointer, threadpool_sched_affinity_log_checkpointer, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "log_checkpointer", - check_fix_sched_affinity_cpustring, // check func - fix_sched_affinity_cpustring, // update func + check_fix_sched_affinity_cpustring, // check func + fix_sched_affinity_background_cpustring, // update func nullptr); // default value static MYSQL_SYSVAR_STR(sched_affinity_purge_coordinator, threadpool_sched_affinity_purge_coordinator, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, "purge_coordinator", - check_fix_sched_affinity_cpustring, // check func - fix_sched_affinity_cpustring, // update func + check_fix_sched_affinity_cpustring, // check func + fix_sched_affinity_background_cpustring, // update func nullptr); // default value diff --git a/plugin/thread_pool/threadpool_unix.cc b/plugin/thread_pool/threadpool_unix.cc index 236013823..c966fa738 100644 --- a/plugin/thread_pool/threadpool_unix.cc +++ b/plugin/thread_pool/threadpool_unix.cc @@ -1627,9 +1627,7 @@ static void *worker_main(void *param) { thread_group_t *thread_group = static_cast(param); assert(thread_group != nullptr); - if (threadpool_sched_affinity) { - group_affinity.bind_foreground_thread(thread_group - all_groups); - } + group_affinity.bind_foreground_thread(thread_group - all_groups); /* Init per-thread structure */ worker_thread_t this_thread; @@ -1656,6 +1654,8 @@ static void *worker_main(void *param) { handle_event(connection); } + group_affinity.remove_foreground_thread(); + /* Thread shutdown: cleanup per-worker-thread structure. */ mysql_cond_destroy(&this_thread.cond); -- Gitee