| From fe8613546614bed3f089a1fce3411ada03620f5e Mon Sep 17 00:00:00 2001 |
| From: Ricardo Neri <ricardo.neri-calderon@linux.intel.com> |
| Date: Fri, 10 Sep 2021 18:18:19 -0700 |
| Subject: [PATCH] BACKPORT: FROMGIT: sched/fair: Consider SMT in ASYM_PACKING |
| load balance |
| |
| When deciding to pull tasks in ASYM_PACKING, it is necessary not only to |
| check for the idle state of the destination CPU, dst_cpu, but also of |
| its SMT siblings. |
| |
| If dst_cpu is idle but its SMT siblings are busy, performance suffers |
| if it pulls tasks from a medium priority CPU that does not have SMT |
| siblings. |
| |
| Implement asym_smt_can_pull_tasks() to inspect the state of the SMT |
| siblings of both dst_cpu and the CPUs in the candidate busiest group. |
| |
| Signed-off-by: Ricardo Neri <ricardo.neri-calderon@linux.intel.com> |
| Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> |
| Reviewed-by: Joel Fernandes (Google) <joel@joelfernandes.org> |
| Reviewed-by: Len Brown <len.brown@intel.com> |
| Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org> |
| Link: https://lkml.kernel.org/r/20210911011819.12184-7-ricardo.neri-calderon@linux.intel.com |
| Change-Id: Ib94f2ab5dd21ae7ceb81c800d9c6724948a28ea2 |
| |
| (cherry picked from commit eac6f3841f1dac7b6f43002056b63f44cc1f1543 |
| git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git master) |
| |
| Conflicts: |
| kernel/sched/fair.c |
| |
| Backport Notes: |
| minor conflict due to code overlap in API static bool update_nohz_stats(struct rq *rq, bool force) and new function |
| asym_smt_can_pull_tasks |
| |
| Added the below three lines: |
| |
| return false; |
| #endif |
| } |
| |
| BUG=b:179699891 |
| TEST=basic boot on brya and check /proc/sys/kernel/sched_itmt_enabled |
| should be 1 |
| TEST=spin N+1 threads where N = number of physical cores - expect |
| N threads to run on N physical cores and the N+1st thread |
| to run on an atom CPU |
| |
| Signed-off-by: George D Sworo <george.d.sworo@intel.com> |
| Change-Id: I9b962caa959262eec8f960e40769e41761fa84c3 |
| Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/3174725 |
| Reviewed-by: Alex Levin <levinale@google.com> |
| --- |
| kernel/sched/fair.c | 92 +++++++++++++++++++++++++++++++++++++++++++++ |
| 1 file changed, 92 insertions(+) |
| |
| diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c |
| --- a/kernel/sched/fair.c |
| +++ b/kernel/sched/fair.c |
| @@ -8604,10 +8604,96 @@ group_type group_classify(unsigned int imbalance_pct, |
| return group_has_spare; |
| } |
| |
| +/** |
| + * asym_smt_can_pull_tasks - Check whether the load balancing CPU can pull tasks |
| + * @dst_cpu: Destination CPU of the load balancing |
| + * @sds: Load-balancing data with statistics of the local group |
| + * @sgs: Load-balancing statistics of the candidate busiest group |
| + * @sg: The candidate busiest group |
| + * |
| + * Check the state of the SMT siblings of both @sds::local and @sg and decide |
| + * if @dst_cpu can pull tasks. |
| + * |
| + * If @dst_cpu does not have SMT siblings, it can pull tasks if two or more of |
| + * the SMT siblings of @sg are busy. If only one CPU in @sg is busy, pull tasks |
| + * only if @dst_cpu has higher priority. |
| + * |
| + * If both @dst_cpu and @sg have SMT siblings, and @sg has exactly one more |
| + * busy CPU than @sds::local, let @dst_cpu pull tasks if it has higher priority. |
| + * Bigger imbalances in the number of busy CPUs will be dealt with in |
| + * update_sd_pick_busiest(). |
| + * |
| + * If @sg does not have SMT siblings, only pull tasks if all of the SMT siblings |
| + * of @dst_cpu are idle and @sg has lower priority. |
| + */ |
| +static bool asym_smt_can_pull_tasks(int dst_cpu, struct sd_lb_stats *sds, |
| + struct sg_lb_stats *sgs, |
| + struct sched_group *sg) |
| +{ |
| +#ifdef CONFIG_SCHED_SMT |
| + bool local_is_smt, sg_is_smt; |
| + int sg_busy_cpus; |
| + |
| + local_is_smt = sds->local->flags & SD_SHARE_CPUCAPACITY; |
| + sg_is_smt = sg->flags & SD_SHARE_CPUCAPACITY; |
| + |
| + sg_busy_cpus = sgs->group_weight - sgs->idle_cpus; |
| + |
| + if (!local_is_smt) { |
| + /* |
| + * If we are here, @dst_cpu is idle and does not have SMT |
| + * siblings. Pull tasks if candidate group has two or more |
| + * busy CPUs. |
| + */ |
| + if (sg_busy_cpus >= 2) /* implies sg_is_smt */ |
| + return true; |
| + |
| + /* |
| + * @dst_cpu does not have SMT siblings. @sg may have SMT |
| + * siblings and only one is busy. In such case, @dst_cpu |
| + * can help if it has higher priority and is idle (i.e., |
| + * it has no running tasks). |
| + */ |
| + return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu); |
| + } |
| + |
| + /* @dst_cpu has SMT siblings. */ |
| + |
| + if (sg_is_smt) { |
| + int local_busy_cpus = sds->local->group_weight - |
| + sds->local_stat.idle_cpus; |
| + int busy_cpus_delta = sg_busy_cpus - local_busy_cpus; |
| + |
| + if (busy_cpus_delta == 1) |
| + return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu); |
| + |
| + return false; |
| + } |
| + |
| + /* |
| + * @sg does not have SMT siblings. Ensure that @sds::local does not end |
| + * up with more than one busy SMT sibling and only pull tasks if there |
| + * are not busy CPUs (i.e., no CPU has running tasks). |
| + */ |
| + if (!sds->local_stat.sum_nr_running) |
| + return sched_asym_prefer(dst_cpu, sg->asym_prefer_cpu); |
| + |
| + return false; |
| +#else |
| + /* Always return false so that callers deal with non-SMT cases. */ |
| + return false; |
| +#endif |
| +} |
| + |
| static inline bool |
| sched_asym(struct lb_env *env, struct sd_lb_stats *sds, struct sg_lb_stats *sgs, |
| struct sched_group *group) |
| { |
| + /* Only do SMT checks if either local or candidate have SMT siblings */ |
| + if ((sds->local->flags & SD_SHARE_CPUCAPACITY) || |
| + (group->flags & SD_SHARE_CPUCAPACITY)) |
| + return asym_smt_can_pull_tasks(env->dst_cpu, sds, sgs, group); |
| + |
| return sched_asym_prefer(env->dst_cpu, group->asym_prefer_cpu); |
| } |
| |
| @@ -9613,6 +9699,12 @@ static struct rq *find_busiest_queue(struct lb_env *env, |
| nr_running == 1) |
| continue; |
| |
| + /* Make sure we only pull tasks from a CPU of lower priority */ |
| + if ((env->sd->flags & SD_ASYM_PACKING) && |
| + sched_asym_prefer(i, env->dst_cpu) && |
| + nr_running == 1) |
| + continue; |
| + |
| switch (env->migration_type) { |
| case migrate_load: |
| /* |
| -- |
| 2.33.0.882.g93a45727a2-goog |
| |