sched.h (542898c5aa5c6a3179dffb1d1606884a63f75fed) sched.h (0fb3978b0aac3a5c08637aed03cc2d65f793508f)
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Scheduler internal types and methods:
4 */
5#include <linux/sched.h>
6
7#include <linux/sched/autogroup.h>
8#include <linux/sched/clock.h>

--- 1648 unchanged lines hidden (view full) ---

1657enum numa_topology_type {
1658 NUMA_DIRECT,
1659 NUMA_GLUELESS_MESH,
1660 NUMA_BACKPLANE,
1661};
1662extern enum numa_topology_type sched_numa_topology_type;
1663extern int sched_max_numa_distance;
1664extern bool find_numa_distance(int distance);
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Scheduler internal types and methods:
4 */
5#include <linux/sched.h>
6
7#include <linux/sched/autogroup.h>
8#include <linux/sched/clock.h>

--- 1648 unchanged lines hidden (view full) ---

1657enum numa_topology_type {
1658 NUMA_DIRECT,
1659 NUMA_GLUELESS_MESH,
1660 NUMA_BACKPLANE,
1661};
1662extern enum numa_topology_type sched_numa_topology_type;
1663extern int sched_max_numa_distance;
1664extern bool find_numa_distance(int distance);
1665extern void sched_init_numa(void);
1665extern void sched_init_numa(int offline_node);
1666extern void sched_update_numa(int cpu, bool online);
1666extern void sched_domains_numa_masks_set(unsigned int cpu);
1667extern void sched_domains_numa_masks_clear(unsigned int cpu);
1668extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu);
1669#else
1667extern void sched_domains_numa_masks_set(unsigned int cpu);
1668extern void sched_domains_numa_masks_clear(unsigned int cpu);
1669extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu);
1670#else
1670static inline void sched_init_numa(void) { }
1671static inline void sched_init_numa(int offline_node) { }
1672static inline void sched_update_numa(int cpu, bool online) { }
1671static inline void sched_domains_numa_masks_set(unsigned int cpu) { }
1672static inline void sched_domains_numa_masks_clear(unsigned int cpu) { }
1673static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
1674{
1675 return nr_cpu_ids;
1676}
1677#endif
1678

--- 1157 unchanged lines hidden (view full) ---

2836 cpu_of(rq)));
2837 if (data)
2838 data->func(data, rq_clock(rq), flags);
2839}
2840#else
2841static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
2842#endif /* CONFIG_CPU_FREQ */
2843
1673static inline void sched_domains_numa_masks_set(unsigned int cpu) { }
1674static inline void sched_domains_numa_masks_clear(unsigned int cpu) { }
1675static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
1676{
1677 return nr_cpu_ids;
1678}
1679#endif
1680

--- 1157 unchanged lines hidden (view full) ---

2838 cpu_of(rq)));
2839 if (data)
2840 data->func(data, rq_clock(rq), flags);
2841}
2842#else
2843static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
2844#endif /* CONFIG_CPU_FREQ */
2845
2844#ifdef CONFIG_UCLAMP_TASK
2845unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id);
2846
2847/**
2848 * uclamp_rq_util_with - clamp @util with @rq and @p effective uclamp values.
2849 * @rq: The rq to clamp against. Must not be NULL.
2850 * @util: The util value to clamp.
2851 * @p: The task to clamp against. Can be NULL if you want to clamp
2852 * against @rq only.
2853 *
2854 * Clamps the passed @util to the max(@rq, @p) effective uclamp values.
2855 *
2856 * If sched_uclamp_used static key is disabled, then just return the util
2857 * without any clamping since uclamp aggregation at the rq level in the fast
2858 * path is disabled, rendering this operation a NOP.
2859 *
2860 * Use uclamp_eff_value() if you don't care about uclamp values at rq level. It
2861 * will return the correct effective uclamp value of the task even if the
2862 * static key is disabled.
2863 */
2864static __always_inline
2865unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,
2866 struct task_struct *p)
2867{
2868 unsigned long min_util = 0;
2869 unsigned long max_util = 0;
2870
2871 if (!static_branch_likely(&sched_uclamp_used))
2872 return util;
2873
2874 if (p) {
2875 min_util = uclamp_eff_value(p, UCLAMP_MIN);
2876 max_util = uclamp_eff_value(p, UCLAMP_MAX);
2877
2878 /*
2879 * Ignore last runnable task's max clamp, as this task will
2880 * reset it. Similarly, no need to read the rq's min clamp.
2881 */
2882 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
2883 goto out;
2884 }
2885
2886 min_util = max_t(unsigned long, min_util, READ_ONCE(rq->uclamp[UCLAMP_MIN].value));
2887 max_util = max_t(unsigned long, max_util, READ_ONCE(rq->uclamp[UCLAMP_MAX].value));
2888out:
2889 /*
2890 * Since CPU's {min,max}_util clamps are MAX aggregated considering
2891 * RUNNABLE tasks with _different_ clamps, we can end up with an
2892 * inversion. Fix it now when the clamps are applied.
2893 */
2894 if (unlikely(min_util >= max_util))
2895 return min_util;
2896
2897 return clamp(util, min_util, max_util);
2898}
2899
2900/*
2901 * When uclamp is compiled in, the aggregation at rq level is 'turned off'
2902 * by default in the fast path and only gets turned on once userspace performs
2903 * an operation that requires it.
2904 *
2905 * Returns true if userspace opted-in to use uclamp and aggregation at rq level
2906 * hence is active.
2907 */
2908static inline bool uclamp_is_used(void)
2909{
2910 return static_branch_likely(&sched_uclamp_used);
2911}
2912#else /* CONFIG_UCLAMP_TASK */
2913static inline
2914unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,
2915 struct task_struct *p)
2916{
2917 return util;
2918}
2919
2920static inline bool uclamp_is_used(void)
2921{
2922 return false;
2923}
2924#endif /* CONFIG_UCLAMP_TASK */
2925
2926#ifdef arch_scale_freq_capacity
2927# ifndef arch_scale_freq_invariant
2928# define arch_scale_freq_invariant() true
2929# endif
2930#else
2931# define arch_scale_freq_invariant() false
2932#endif
2933

--- 81 unchanged lines hidden (view full) ---

3015}
3016
3017static inline unsigned long cpu_util_rt(struct rq *rq)
3018{
3019 return READ_ONCE(rq->avg_rt.util_avg);
3020}
3021#endif
3022
2846#ifdef arch_scale_freq_capacity
2847# ifndef arch_scale_freq_invariant
2848# define arch_scale_freq_invariant() true
2849# endif
2850#else
2851# define arch_scale_freq_invariant() false
2852#endif
2853

--- 81 unchanged lines hidden (view full) ---

2935}
2936
2937static inline unsigned long cpu_util_rt(struct rq *rq)
2938{
2939 return READ_ONCE(rq->avg_rt.util_avg);
2940}
2941#endif
2942
2943#ifdef CONFIG_UCLAMP_TASK
2944unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id);
2945
2946/**
2947 * uclamp_rq_util_with - clamp @util with @rq and @p effective uclamp values.
2948 * @rq: The rq to clamp against. Must not be NULL.
2949 * @util: The util value to clamp.
2950 * @p: The task to clamp against. Can be NULL if you want to clamp
2951 * against @rq only.
2952 *
2953 * Clamps the passed @util to the max(@rq, @p) effective uclamp values.
2954 *
2955 * If sched_uclamp_used static key is disabled, then just return the util
2956 * without any clamping since uclamp aggregation at the rq level in the fast
2957 * path is disabled, rendering this operation a NOP.
2958 *
2959 * Use uclamp_eff_value() if you don't care about uclamp values at rq level. It
2960 * will return the correct effective uclamp value of the task even if the
2961 * static key is disabled.
2962 */
2963static __always_inline
2964unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,
2965 struct task_struct *p)
2966{
2967 unsigned long min_util = 0;
2968 unsigned long max_util = 0;
2969
2970 if (!static_branch_likely(&sched_uclamp_used))
2971 return util;
2972
2973 if (p) {
2974 min_util = uclamp_eff_value(p, UCLAMP_MIN);
2975 max_util = uclamp_eff_value(p, UCLAMP_MAX);
2976
2977 /*
2978 * Ignore last runnable task's max clamp, as this task will
2979 * reset it. Similarly, no need to read the rq's min clamp.
2980 */
2981 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
2982 goto out;
2983 }
2984
2985 min_util = max_t(unsigned long, min_util, READ_ONCE(rq->uclamp[UCLAMP_MIN].value));
2986 max_util = max_t(unsigned long, max_util, READ_ONCE(rq->uclamp[UCLAMP_MAX].value));
2987out:
2988 /*
2989 * Since CPU's {min,max}_util clamps are MAX aggregated considering
2990 * RUNNABLE tasks with _different_ clamps, we can end up with an
2991 * inversion. Fix it now when the clamps are applied.
2992 */
2993 if (unlikely(min_util >= max_util))
2994 return min_util;
2995
2996 return clamp(util, min_util, max_util);
2997}
2998
2999/* Is the rq being capped/throttled by uclamp_max? */
3000static inline bool uclamp_rq_is_capped(struct rq *rq)
3001{
3002 unsigned long rq_util;
3003 unsigned long max_util;
3004
3005 if (!static_branch_likely(&sched_uclamp_used))
3006 return false;
3007
3008 rq_util = cpu_util_cfs(cpu_of(rq)) + cpu_util_rt(rq);
3009 max_util = READ_ONCE(rq->uclamp[UCLAMP_MAX].value);
3010
3011 return max_util != SCHED_CAPACITY_SCALE && rq_util >= max_util;
3012}
3013
3014/*
3015 * When uclamp is compiled in, the aggregation at rq level is 'turned off'
3016 * by default in the fast path and only gets turned on once userspace performs
3017 * an operation that requires it.
3018 *
3019 * Returns true if userspace opted-in to use uclamp and aggregation at rq level
3020 * hence is active.
3021 */
3022static inline bool uclamp_is_used(void)
3023{
3024 return static_branch_likely(&sched_uclamp_used);
3025}
3026#else /* CONFIG_UCLAMP_TASK */
3027static inline
3028unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long util,
3029 struct task_struct *p)
3030{
3031 return util;
3032}
3033
3034static inline bool uclamp_rq_is_capped(struct rq *rq) { return false; }
3035
3036static inline bool uclamp_is_used(void)
3037{
3038 return false;
3039}
3040#endif /* CONFIG_UCLAMP_TASK */
3041
3023#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
3024static inline unsigned long cpu_util_irq(struct rq *rq)
3025{
3026 return rq->avg_irq.util_avg;
3027}
3028
3029static inline
3030unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)

--- 90 unchanged lines hidden --- 		3042#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
3043static inline unsigned long cpu_util_irq(struct rq *rq)
3044{
3045 return rq->avg_irq.util_avg;
3046}
3047
3048static inline
3049unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)

--- 90 unchanged lines hidden ---