From a74ad38cd4e2832572aa167977771778662ed4de Mon Sep 17 00:00:00 2001 From: ma_yulong Date: Wed, 11 Sep 2024 11:23:03 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=9D=A2=E5=90=91=E5=B8=A7?= =?UTF-8?q?=E7=9A=84=E5=85=B3=E8=81=94=E7=BA=BF=E7=A8=8B=E7=BB=84=E8=B0=83?= =?UTF-8?q?=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ma_yulong --- fs/proc/base.c | 86 ++++++++++++++++++ include/linux/sched.h | 6 ++ include/linux/sched/frame_rtg.h | 75 ++++++++++++++++ include/linux/sched/rtg.h | 65 ++++++++++++++ include/linux/sched/rtg_ctrl.h | 99 +++++++++++++++++++++ include/trace/events/rtg.h | 145 +++++++++++++++++++++++++++++++ kernel/sched/Makefile | 1 + kernel/sched/core.c | 71 ++++++++++++++- kernel/sched/cpufreq_schedutil.c | 35 ++++++++ kernel/sched/fair.c | 68 ++++++++++++++- kernel/sched/sched.h | 10 +++ 11 files changed, 659 insertions(+), 2 deletions(-) create mode 100755 include/linux/sched/frame_rtg.h create mode 100755 include/linux/sched/rtg.h create mode 100755 include/linux/sched/rtg_ctrl.h create mode 100755 include/trace/events/rtg.h diff --git a/fs/proc/base.c b/fs/proc/base.c index 835d83c1af86..a0c4fd012520 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -88,6 +88,9 @@ #include #include #include +#ifdef CONFIG_SCHED_RTG +#include +#endif #include #include #include @@ -1485,6 +1488,80 @@ static const struct file_operations proc_pid_sched_operations = { #endif +#ifdef CONFIG_SCHED_RTG +static const struct file_operations proc_rtg_operations = { + .open = proc_rtg_open, + .unlocked_ioctl = proc_rtg_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = proc_rtg_compat_ioctl, +#endif +}; +#endif + +#ifdef CONFIG_SCHED_RTG_DEBUG +static int sched_group_id_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + seq_printf(m, "%d\n", sched_get_group_id(p)); + + put_task_struct(p); + + return 0; +} + +static ssize_t +sched_group_id_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + char buffer[PROC_NUMBUF]; + int group_id, err; + + memset(buffer, 0, sizeof(buffer)); + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) { + err = -EFAULT; + goto out; + } + + err = kstrtoint(strstrip(buffer), 0, &group_id); + if (err) + goto out; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + err = sched_set_group_id(p, group_id); + + put_task_struct(p); + +out: + return err < 0 ? err : count; +} + +static int sched_group_id_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_group_id_show, inode); +} + +static const struct file_operations proc_pid_sched_group_id_operations = { + .open = sched_group_id_open, + .read = seq_read, + .write = sched_group_id_write, + .llseek = seq_lseek, + .release = single_release, +}; +#endif /* CONFIG_SCHED_RTG_DEBUG */ + #ifdef CONFIG_SCHED_AUTOGROUP /* * Print out autogroup related information: @@ -3425,6 +3502,12 @@ static const struct pid_entry tgid_base_stuff[] = { #ifdef CONFIG_ACCESS_TOKENID ONE("tokenid", S_IRUSR, proc_token_operations), #endif +#ifdef CONFIG_SCHED_RTG + REG("sched_rtg_ctrl", S_IRUGO|S_IWUGO, proc_rtg_operations), +#endif +#ifdef CONFIG_SCHED_RTG_DEBUG + REG("sched_group_id", S_IRUGO|S_IWUGO, proc_pid_sched_group_id_operations), +#endif #ifdef CONFIG_SECCOMP_CACHE_DEBUG ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache), #endif @@ -3767,6 +3850,9 @@ static const struct pid_entry tid_base_stuff[] = { #ifdef CONFIG_ACCESS_TOKENID ONE("tokenid", S_IRUSR, proc_token_operations), #endif +#ifdef CONFIG_SCHED_RTG_DEBUG + REG("sched_group_id", S_IRUGO|S_IWUGO, proc_pid_sched_group_id_operations), +#endif #ifdef CONFIG_SECCOMP_CACHE_DEBUG ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache), #endif diff --git a/include/linux/sched.h b/include/linux/sched.h index 7943095a925d..b5e8ea71b57f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -908,6 +908,12 @@ struct task_struct { unsigned int core_occupation; #endif +#ifdef CONFIG_SCHED_RTG + int rtg_depth; + struct related_thread_group *grp; + struct list_head grp_list; +#endif + #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif diff --git a/include/linux/sched/frame_rtg.h b/include/linux/sched/frame_rtg.h new file mode 100755 index 000000000000..6713f6771c9a --- /dev/null +++ b/include/linux/sched/frame_rtg.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Frame declaration + * + * Copyright (c) 2022-2023 Huawei Technologies Co., Ltd. + */ + +#ifndef __SCHED_FRAME_RTG_H +#define __SCHED_FRAME_RTG_H + +#ifdef CONFIG_SCHED_RTG_FRAME + +#define MAX_TID_NUM 5 + +struct frame_info { + /* + * use rtg load tracking in frame_info + * rtg->curr_window_load -=> the workload of current frame + * rtg->prev_window_load -=> the workload of last frame + * rtg->curr_window_exec -=> the thread's runtime of current frame + * rtg->prev_window_exec -=> the thread's runtime of last frame + * rtg->prev_window_time -=> the actual time of the last frame + */ + struct mutex lock; + struct related_thread_group *rtg; + int prio; + struct task_struct *thread[MAX_TID_NUM]; + atomic_t thread_prio[MAX_TID_NUM]; + int thread_num; + unsigned int frame_rate; // frame rate + u64 frame_time; + atomic_t curr_rt_thread_num; + atomic_t max_rt_thread_num; + atomic_t frame_sched_state; + atomic_t start_frame_freq; + atomic_t frame_state; + + /* + * frame_vload : the emergency level of current frame. + * max_vload_time : the timeline frame_load increase to FRAME_MAX_VLOAD + * it's always equal to 2 * frame_time / NSEC_PER_MSEC + * + * The closer to the deadline, the higher emergency of current + * frame, so the frame_vload is only related to frame time, + * and grown with time. + */ + u64 frame_vload; + int vload_margin; + int max_vload_time; + + u64 frame_util; + unsigned long status; + unsigned long prev_fake_load_util; + unsigned long prev_frame_load_util; + unsigned long prev_frame_time; + unsigned long prev_frame_exec; + unsigned long prev_frame_load; + unsigned int frame_min_util; + unsigned int frame_max_util; + unsigned int prev_min_util; + unsigned int prev_max_util; + unsigned int frame_boost_min_util; + + bool margin_imme; + bool timestamp_skipped; +}; + +struct frame_info *rtg_frame_info(int id); +static inline +struct related_thread_group *frame_info_rtg(const struct frame_info *frame_info) +{ + return frame_info->rtg; +} +#endif +#endif diff --git a/include/linux/sched/rtg.h b/include/linux/sched/rtg.h new file mode 100755 index 000000000000..ec738f49fd1e --- /dev/null +++ b/include/linux/sched/rtg.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SCHED_RTG_H +#define __SCHED_RTG_H + +#ifdef CONFIG_SCHED_RTG + +#define DEFAULT_RTG_GRP_ID 0 +#define DEFAULT_CGROUP_COLOC_ID 1 +#define MAX_NUM_CGROUP_COLOC_ID 21 + +struct group_cpu_time { + u64 window_start; + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; +}; + +struct group_ravg { + unsigned long curr_window_load; + unsigned long curr_window_exec; + unsigned long prev_window_load; + unsigned long prev_window_exec; + unsigned long normalized_util; +}; + +struct rtg_class; + +struct related_thread_group { + int id; + raw_spinlock_t lock; + struct list_head tasks; + struct list_head list; + + unsigned int nr_running; + struct group_ravg ravg; + u64 window_start; + u64 mark_start; + u64 prev_window_time; + /* rtg window information for WALT */ + unsigned int window_size; + const struct rtg_class *rtg_class; + struct sched_cluster *preferred_cluster; + int max_boost; + unsigned long util_invalid_interval; /* in nanoseconds */ + unsigned long util_update_timeout; /* in nanoseconds */ + unsigned long freq_update_interval; /* in nanoseconds */ + u64 last_util_update_time; + u64 last_freq_update_time; + void *private_data; +}; + +struct rtg_class { + void (*sched_update_rtg_tick)(struct related_thread_group *grp); +}; + +enum rtg_freq_update_flags { + RTG_FREQ_FORCE_UPDATE = (1 << 0), + RTG_FREQ_NORMAL_UPDATE = (1 << 1), +}; + +int sched_set_group_id(struct task_struct *p, unsigned int group_id); +unsigned int sched_get_group_id(struct task_struct *p); +#endif /* CONFIG_SCHED_RTG */ +#endif diff --git a/include/linux/sched/rtg_ctrl.h b/include/linux/sched/rtg_ctrl.h new file mode 100755 index 000000000000..b71dd74e7fc3 --- /dev/null +++ b/include/linux/sched/rtg_ctrl.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * rtg control interface + * + * Copyright (c) 2022-2023 Huawei Technologies Co., Ltd. + */ + +#ifndef __SCHED_RTG_CTL_H +#define __SCHED_RTG_CTL_H + +#include + +#define SYSTEM_SERVER_UID 1000 +#define MIN_APP_UID 10000 +#define MAX_BOOST_DURATION_MS 5000 + +#define RTG_SCHED_IPC_MAGIC 0XAB + +#define CMD_ID_SET_ENABLE \ + _IOWR(RTG_SCHED_IPC_MAGIC, SET_ENABLE, struct rtg_enable_data) +#define CMD_ID_SET_RTG \ + _IOWR(RTG_SCHED_IPC_MAGIC, SET_RTG, struct rtg_str_data) +#define CMD_ID_SET_CONFIG \ + _IOWR(RTG_SCHED_IPC_MAGIC, SET_CONFIG, struct rtg_str_data) +#define CMD_ID_SET_RTG_ATTR \ + _IOWR(RTG_SCHED_IPC_MAGIC, SET_RTG_ATTR, struct rtg_str_data) +#define CMD_ID_BEGIN_FRAME_FREQ \ + _IOWR(RTG_SCHED_IPC_MAGIC, BEGIN_FRAME_FREQ, struct proc_state_data) +#define CMD_ID_END_FRAME_FREQ \ + _IOWR(RTG_SCHED_IPC_MAGIC, END_FRAME_FREQ, struct proc_state_data) +#define CMD_ID_END_SCENE \ + _IOWR(RTG_SCHED_IPC_MAGIC, END_SCENE, struct proc_state_data) +#define CMD_ID_SET_MIN_UTIL \ + _IOWR(RTG_SCHED_IPC_MAGIC, SET_MIN_UTIL, struct proc_state_data) +#define CMD_ID_SET_MARGIN \ + _IOWR(RTG_SCHED_IPC_MAGIC, SET_MARGIN, struct proc_state_data) +#define CMD_ID_LIST_RTG \ + _IOWR(RTG_SCHED_IPC_MAGIC, LIST_RTG, struct rtg_info) +#define CMD_ID_LIST_RTG_THREAD \ + _IOWR(RTG_SCHED_IPC_MAGIC, LIST_RTG_THREAD, struct rtg_grp_data) +#define CMD_ID_SEARCH_RTG \ + _IOWR(RTG_SCHED_IPC_MAGIC, SEARCH_RTG, struct proc_state_data) +#define CMD_ID_GET_ENABLE \ + _IOWR(RTG_SCHED_IPC_MAGIC, GET_ENABLE, struct rtg_enable_data) + +int proc_rtg_open(struct inode *inode, struct file *filp); +long proc_rtg_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +#ifdef CONFIG_COMPAT +long proc_rtg_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +#endif + +enum ioctl_abi_format { + IOCTL_ABI_ARM32, + IOCTL_ABI_AARCH64, +}; + +enum rtg_sched_cmdid { + SET_ENABLE = 1, + SET_RTG, + SET_CONFIG, + SET_RTG_ATTR, + BEGIN_FRAME_FREQ = 5, + END_FRAME_FREQ, + END_SCENE, + SET_MIN_UTIL, + SET_MARGIN, + LIST_RTG = 10, + LIST_RTG_THREAD, + SEARCH_RTG, + GET_ENABLE, + RTG_CTRL_MAX_NR, +}; + +/* proc_state */ +enum grp_ctrl_cmd { + CMD_CREATE_RTG_GRP, + CMD_ADD_RTG_THREAD, + CMD_REMOVE_RTG_THREAD, + CMD_CLEAR_RTG_GRP, + CMD_DESTROY_RTG_GRP +}; + +struct rtg_enable_data { + int enable; + int len; + char *data; +}; + +struct rtg_str_data { + int type; + int len; + char *data; +}; + +struct proc_state_data { + int grp_id; + int state_param; +}; +#endif diff --git a/include/trace/events/rtg.h b/include/trace/events/rtg.h new file mode 100755 index 000000000000..b885e328de12 --- /dev/null +++ b/include/trace/events/rtg.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rtg + +#if !defined(_TRACE_RTG_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RTG_H + +#include +#include +#include + +struct rq; + +TRACE_EVENT(find_rtg_cpu, + + TP_PROTO(struct task_struct *p, const struct cpumask *perferred_cpumask, + char *msg, int cpu), + + TP_ARGS(p, perferred_cpumask, msg, cpu), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __bitmask(cpus, num_possible_cpus()) + __array(char, msg, TASK_COMM_LEN) + __field(int, cpu) + ), + + TP_fast_assign( + __entry->pid = p->pid; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __assign_bitmask(cpus, cpumask_bits(perferred_cpumask), num_possible_cpus()); + memcpy(__entry->msg, msg, min((size_t)TASK_COMM_LEN, strlen(msg)+1)); + __entry->cpu = cpu; + ), + + TP_printk("comm=%s pid=%d perferred_cpus=%s reason=%s target_cpu=%d", + __entry->comm, __entry->pid, __get_bitmask(cpus), __entry->msg, __entry->cpu) +); + +TRACE_EVENT(sched_rtg_task_each, + + TP_PROTO(unsigned int id, unsigned int nr_running, struct task_struct *task), + + TP_ARGS(id, nr_running, task), + + TP_STRUCT__entry( + __field(unsigned int, id) + __field(unsigned int, nr_running) + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __bitmask(allowed, num_possible_cpus()) + __field(int, cpu) + __field(int, state) + __field(bool, on_rq) + __field(int, on_cpu) + ), + + TP_fast_assign( + __entry->id = id; + __entry->nr_running = nr_running; + memcpy(__entry->comm, task->comm, TASK_COMM_LEN); + __entry->pid = task->pid; + __entry->prio = task->prio; + __assign_bitmask(allowed, cpumask_bits(&task->cpus_mask), num_possible_cpus()); + __entry->cpu = task_cpu(task); + __entry->state = task->state; + __entry->on_rq = task->on_rq; + __entry->on_cpu = task->on_cpu; + ), + + TP_printk("comm=%s pid=%d prio=%d allowed=%s cpu=%d state=%s%s on_rq=%d on_cpu=%d", + __entry->comm, __entry->pid, __entry->prio, __get_bitmask(allowed), __entry->cpu, + __entry->state & (TASK_REPORT_MAX) ? + __print_flags(__entry->state & (TASK_REPORT_MAX), "|", + { TASK_INTERRUPTIBLE, "S" }, + { TASK_UNINTERRUPTIBLE, "D" }, + { __TASK_STOPPED, "T" }, + { __TASK_TRACED, "t" }, + { EXIT_DEAD, "X" }, + { EXIT_ZOMBIE, "Z" }, + { TASK_DEAD, "x" }, + { TASK_WAKEKILL, "K"}, + { TASK_WAKING, "W"}) : "R", + __entry->state & TASK_STATE_MAX ? "+" : "", + __entry->on_rq, __entry->on_cpu) +); + +TRACE_EVENT(sched_rtg_valid_normalized_util, + + TP_PROTO(unsigned int id, unsigned int nr_running, + const struct cpumask *rtg_cpus, unsigned int valid), + + TP_ARGS(id, nr_running, rtg_cpus, valid), + + TP_STRUCT__entry( + __field(unsigned int, id) + __field(unsigned int, nr_running) + __bitmask(cpus, num_possible_cpus()) + __field(unsigned int, valid) + ), + + TP_fast_assign( + __entry->id = id; + __entry->nr_running = nr_running; + __assign_bitmask(cpus, cpumask_bits(rtg_cpus), num_possible_cpus()); + __entry->valid = valid; + ), + + TP_printk("id=%d nr_running=%d cpus=%s valid=%d", + __entry->id, __entry->nr_running, + __get_bitmask(cpus), __entry->valid) +); + +#ifdef CONFIG_SCHED_RTG_FRAME +TRACE_EVENT(rtg_frame_sched, + + TP_PROTO(int rtgid, const char *s, s64 value), + + TP_ARGS(rtgid, s, value), + TP_STRUCT__entry( + __field(int, rtgid) + __field(struct frame_info *, frame) + __field(pid_t, pid) + __string(str, s) + __field(s64, value) + ), + + TP_fast_assign( + __assign_str(str, s); + __entry->rtgid = rtgid != -1 ? rtgid : (current->grp ? current->grp->id : 0); + __entry->frame = rtg_frame_info(rtgid); + __entry->pid = __entry->frame ? ((__entry->frame->thread[0]) ? + ((__entry->frame->thread[0])->pid) : + current->tgid) : current->tgid; + __entry->value = value; + ), + TP_printk("C|%d|%s_%d|%lld", __entry->pid, __get_str(str), __entry->rtgid, __entry->value) +); +#endif +#endif /* _TRACE_RTG_H */ + +/* This part must be outside protection */ +#include diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index ea4f0b3c4773..3a6c3878cb7d 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -33,5 +33,6 @@ obj-y += fair.o obj-y += build_policy.o obj-y += build_utility.o obj-$(CONFIG_SCHED_WALT) += walt.o +obj-$(CONFIG_SCHED_RTG) += rtg/ obj-$(CONFIG_SCHED_RUNNING_AVG) += sched_avg.o obj-$(CONFIG_SCHED_CORE_CTRL) += core_ctl.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index dc3ee249258b..84654c67d694 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4678,6 +4678,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) #ifdef CONFIG_SMP p->wake_entry.u_flags = CSD_TYPE_TTWU; p->migration_pending = NULL; +#endif +#ifdef CONFIG_SCHED_RTG + p->rtg_depth = 0; #endif init_sched_mm_cid(p); } @@ -4897,7 +4900,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) if (unlikely(p->sched_reset_on_fork)) { if (task_has_dl_policy(p) || task_has_rt_policy(p)) { p->policy = SCHED_NORMAL; +#ifdef CONFIG_SCHED_RTG + if (current->rtg_depth != 0) + p->static_prio = current->static_prio; + else + p->static_prio = NICE_TO_PRIO(0); +#else p->static_prio = NICE_TO_PRIO(0); +#endif p->rt_priority = 0; } else if (PRIO_TO_NICE(p->static_prio) < 0) p->static_prio = NICE_TO_PRIO(0); @@ -5820,7 +5830,9 @@ void scheduler_tick(void) task_tick_mm_cid(rq, curr); rq_unlock(rq, &rf); - +#ifdef CONFIG_SCHED_RTG + sched_update_rtg_tick(curr); +#endif if (sched_feat(LATENCY_WARN) && resched_latency) resched_latency_warn(cpu, resched_latency); @@ -11228,6 +11240,11 @@ cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) if (IS_ERR(tg)) return ERR_PTR(-ENOMEM); +#ifdef CONFIG_SCHED_RTG_CGROUP + tg->colocate = false; + tg->colocate_update_disabled = false; +#endif + return &tg->css; } @@ -11283,6 +11300,26 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) } #endif +#if defined(CONFIG_UCLAMP_TASK_GROUP) && defined(CONFIG_SCHED_RTG_CGROUP) +static void schedgp_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + bool colocate; + struct task_group *tg; + + cgroup_taskset_first(tset, &css); + tg = css_tg(css); + + colocate = tg->colocate; + + cgroup_taskset_for_each(task, css, tset) + sync_cgroup_colocation(task, colocate); +} +#else +static void schedgp_attach(struct cgroup_taskset *tset) { } +#endif + static void cpu_cgroup_attach(struct cgroup_taskset *tset) { struct task_struct *task; @@ -11467,6 +11504,30 @@ static int cpu_uclamp_max_show(struct seq_file *sf, void *v) cpu_uclamp_print(sf, UCLAMP_MAX); return 0; } + +#ifdef CONFIG_SCHED_RTG_CGROUP +static u64 sched_colocate_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return (u64) tg->colocate; +} + +static int sched_colocate_write(struct cgroup_subsys_state *css, + struct cftype *cft, u64 colocate) +{ + struct task_group *tg = css_tg(css); + + if (tg->colocate_update_disabled) + return -EPERM; + + tg->colocate = !!colocate; + tg->colocate_update_disabled = true; + + return 0; +} +#endif /* CONFIG_SCHED_RTG_CGROUP */ #endif /* CONFIG_UCLAMP_TASK_GROUP */ #ifdef CONFIG_FAIR_GROUP_SCHED @@ -11939,6 +12000,14 @@ static struct cftype cpu_legacy_files[] = { .seq_show = cpu_uclamp_max_show, .write = cpu_uclamp_max_write, }, +#ifdef CONFIG_SCHED_RTG_CGROUP + { + .name = "uclamp.colocate", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = sched_colocate_read, + .write_u64 = sched_colocate_write, + }, +#endif #endif { } /* Terminate */ }; diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 035ed07966e0..927b86130390 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -6,6 +6,8 @@ * Author: Rafael J. Wysocki */ +#include "sched.h" +#include "rtg/rtg.h" #define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8) struct sugov_tunables { @@ -31,6 +33,10 @@ struct sugov_policy { struct mutex work_lock; struct kthread_worker worker; struct task_struct *thread; +#ifdef CONFIG_SCHED_RTG + unsigned long rtg_util; + unsigned int rtg_freq; +#endif bool work_in_progress; bool limits_changed; @@ -340,6 +346,11 @@ static void sugov_update_single_freq(struct update_util_data *hook, u64 time, unsigned int cached_freq = sg_policy->cached_raw_freq; unsigned long max_cap; unsigned int next_f; +#ifdef CONFIG_SCHED_RTG + unsigned long irq_flag; + + force_update = flags & SCHED_CPUFREQ_FORCE_UPDATE; +#endif max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); @@ -373,9 +384,17 @@ static void sugov_update_single_freq(struct update_util_data *hook, u64 time, if (sg_policy->policy->fast_switch_enabled) { cpufreq_driver_fast_switch(sg_policy->policy, next_f); } else { +#ifdef CONFIG_SCHED_RTG + raw_spin_lock_irqsave(&sg_policy->update_lock, irq_flag); +#else raw_spin_lock(&sg_policy->update_lock); +#endif sugov_deferred_update(sg_policy); +#ifdef CONFIG_SCHED_RTG + raw_spin_unlock_irqrestore(&sg_policy->update_lock, irq_flag); +#else raw_spin_unlock(&sg_policy->update_lock); +#endif } } @@ -435,6 +454,10 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) util = max(j_sg_cpu->util, util); } +#ifdef CONFIG_SCHED_RTG + sched_get_max_group_util(policy->cpus, &sg_policy->rtg_util, &sg_policy->rtg_freq); + util = max(sg_policy->rtg_util, util); +#endif return get_next_freq(sg_policy, util, max_cap); } @@ -444,8 +467,16 @@ sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags) struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); struct sugov_policy *sg_policy = sg_cpu->sg_policy; unsigned int next_f; +#ifdef CONFIG_SCHED_RTG + unsigned long irq_flag; +#endif +#ifdef CONFIG_SCHED_RTG + force_update = flags & SCHED_CPUFREQ_FORCE_UPDATE; + raw_spin_lock_irqsave(&sg_policy->update_lock, irq_flag); +#else raw_spin_lock(&sg_policy->update_lock); +#endif sugov_iowait_boost(sg_cpu, time, flags); sg_cpu->last_update = time; @@ -469,7 +500,11 @@ sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags) sugov_deferred_update(sg_policy); } unlock: +#ifdef CONFIG_SCHED_RTG + raw_spin_unlock_irqrestore(&sg_policy->update_lock, irq_flag); +#else raw_spin_unlock(&sg_policy->update_lock); +#endif } static void sugov_work(struct kthread_work *work) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 0c0680a89052..8d6c84e89452 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5096,8 +5096,26 @@ static inline int task_fits_cpu(struct task_struct *p, int cpu) return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); } +#ifdef CONFIG_SCHED_RTG +bool task_fits_max(struct task_struct *p, int cpu) +{ + unsigned long capacity = capacity_orig_of(cpu); + unsigned long max_capacity = cpu_rq(cpu)->rd->max_cpu_capacity; + + if (capacity == max_capacity) + return true; + + return task_fits_cpu(p, cpu); +} +#endif + static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { + bool task_fits = false; +#ifdef CONFIG_SCHED_RTG + int cpu = cpu_of(rq); + struct cpumask *rtg_target = NULL; +#endif if (!sched_asym_cpucap_active()) return; @@ -5106,7 +5124,17 @@ static inline void update_misfit_status(struct task_struct *p, struct rq *rq) return; } - if (task_fits_cpu(p, cpu_of(rq))) { +#ifdef CONFIG_SCHED_RTG + rtg_target = find_rtg_target(p); + if (rtg_target) + task_fits = capacity_orig_of(cpu) >= + capacity_orig_of(cpumask_first(rtg_target)); + else + task_fits = task_fits_cpu(p, cpu_of(rq)); +#else + task_fits = task_fits_cpu(p, cpu_of(rq)); +#endif + if (task_fits) { rq->misfit_task_load = 0; return; } @@ -7881,6 +7909,13 @@ static inline void eenv_task_busy_time(struct energy_env *eenv, eenv->task_busy_time = busy_time; } +#ifdef CONFIG_SCHED_RTG +unsigned long capacity_spare_without(int cpu, struct task_struct *p) +{ + return max_t(long, capacity_of(cpu) - cpu_util_without(cpu, p), 0); +} +#endif + /* * Compute the perf_domain (PD) busy time for compute_energy(). Based on the * utilization for each @pd_cpus, it however doesn't take into account @@ -8228,6 +8263,12 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) int want_affine = 0; /* SD_flags and WF_flags share the first nibble */ int sd_flag = wake_flags & 0xF; +#ifdef CONFIG_SCHED_RTG + int target_cpu = -1; + target_cpu = find_rtg_cpu(p); + if (target_cpu >= 0) + return target_cpu; +#endif /* * required for stable ->cpus_allowed @@ -9130,6 +9171,13 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) /* Record that we found at least one task that could run on dst_cpu */ env->flags &= ~LBF_ALL_PINNED; + +#ifdef CONFIG_SCHED_RTG + if (env->flags & LBF_IGNORE_PREFERRED_CLUSTER_TASKS && + !preferred_cluster(cpu_rq(env->dst_cpu)->cluster, p)) + return 0; +#endif + if (task_on_cpu(env->src_rq, p)) { schedstat_inc(p->stats.nr_failed_migrations_running); return 0; @@ -9224,6 +9272,9 @@ static int detach_tasks(struct lb_env *env) unsigned long util, load; struct task_struct *p; int detached = 0; +#ifdef CONFIG_SCHED_RTG + int orig_loop = env->loop; +#endif lockdep_assert_rq_held(env->src_rq); @@ -9239,6 +9290,12 @@ static int detach_tasks(struct lb_env *env) if (env->imbalance <= 0) return 0; +#ifdef CONFIG_SCHED_RTG + if (!same_cluster(env->dst_cpu, env->src_cpu)) + env->flags |= LBF_IGNORE_PREFERRED_CLUSTER_TASKS; + +redo: +#endif while (!list_empty(tasks)) { /* * We don't want to steal all, otherwise we may be treated likewise, @@ -9344,6 +9401,15 @@ static int detach_tasks(struct lb_env *env) list_move(&p->se.group_node, tasks); } +#ifdef CONFIG_SCHED_RTG + if (env->flags & LBF_IGNORE_PREFERRED_CLUSTER_TASKS && !detached) { + tasks = &env->src_rq->cfs_tasks; + env->flags &= ~LBF_IGNORE_PREFERRED_CLUSTER_TASKS; + env->loop = orig_loop; + goto redo; + } +#endif + /* * Right now, this is one of only two places we collect this stat * so we can safely collect detach_one_task() stats here rather diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e17bf7cb98d7..97b574c917db 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -493,6 +493,16 @@ struct task_group { struct uclamp_se uclamp[UCLAMP_CNT]; #endif +#ifdef CONFIG_SCHED_RTG_CGROUP + /* + * Controls whether tasks of this cgroup should be colocated with each + * other and tasks of other cgroups that have the same flag turned on. + */ + bool colocate; + + /* Controls whether further updates are allowed to the colocate flag */ + bool colocate_update_disabled; +#endif }; #ifdef CONFIG_FAIR_GROUP_SCHED -- Gitee