1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * itmt.c: Support Intel Turbo Boost Max Technology 3.0
4 *
5 * (C) Copyright 2016 Intel Corporation
6 * Author: Tim Chen <tim.c.chen@linux.intel.com>
7 *
8 * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT),
9 * the maximum turbo frequencies of some cores in a CPU package may be
10 * higher than for the other cores in the same package. In that case,
11 * better performance can be achieved by making the scheduler prefer
12 * to run tasks on the CPUs with higher max turbo frequencies.
13 *
14 * This file provides functions and data structures for enabling the
15 * scheduler to favor scheduling on cores can be boosted to a higher
16 * frequency under ITMT.
17 */
18
19 #include <linux/sched.h>
20 #include <linux/cpumask.h>
21 #include <linux/cpuset.h>
22 #include <linux/mutex.h>
23 #include <linux/sysctl.h>
24 #include <linux/nodemask.h>
25
26 static DEFINE_MUTEX(itmt_update_mutex);
27 DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
28
29 /* Boolean to track if system has ITMT capabilities */
30 static bool __read_mostly sched_itmt_capable;
31
32 /*
33 * Boolean to control whether we want to move processes to cpu capable
34 * of higher turbo frequency for cpus supporting Intel Turbo Boost Max
35 * Technology 3.0.
36 *
37 * It can be set via /proc/sys/kernel/sched_itmt_enabled
38 */
39 unsigned int __read_mostly sysctl_sched_itmt_enabled;
40
sched_itmt_update_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)41 static int sched_itmt_update_handler(struct ctl_table *table, int write,
42 void *buffer, size_t *lenp, loff_t *ppos)
43 {
44 unsigned int old_sysctl;
45 int ret;
46
47 mutex_lock(&itmt_update_mutex);
48
49 if (!sched_itmt_capable) {
50 mutex_unlock(&itmt_update_mutex);
51 return -EINVAL;
52 }
53
54 old_sysctl = sysctl_sched_itmt_enabled;
55 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
56
57 if (!ret && write && old_sysctl != sysctl_sched_itmt_enabled) {
58 x86_topology_update = true;
59 rebuild_sched_domains();
60 }
61
62 mutex_unlock(&itmt_update_mutex);
63
64 return ret;
65 }
66
67 static struct ctl_table itmt_kern_table[] = {
68 {
69 .procname = "sched_itmt_enabled",
70 .data = &sysctl_sched_itmt_enabled,
71 .maxlen = sizeof(unsigned int),
72 .mode = 0644,
73 .proc_handler = sched_itmt_update_handler,
74 .extra1 = SYSCTL_ZERO,
75 .extra2 = SYSCTL_ONE,
76 },
77 {}
78 };
79
80 static struct ctl_table_header *itmt_sysctl_header;
81
82 /**
83 * sched_set_itmt_support() - Indicate platform supports ITMT
84 *
85 * This function is used by the OS to indicate to scheduler that the platform
86 * is capable of supporting the ITMT feature.
87 *
88 * The current scheme has the pstate driver detects if the system
89 * is ITMT capable and call sched_set_itmt_support.
90 *
91 * This must be done only after sched_set_itmt_core_prio
92 * has been called to set the cpus' priorities.
93 * It must not be called with cpu hot plug lock
94 * held as we need to acquire the lock to rebuild sched domains
95 * later.
96 *
97 * Return: 0 on success
98 */
sched_set_itmt_support(void)99 int sched_set_itmt_support(void)
100 {
101 mutex_lock(&itmt_update_mutex);
102
103 if (sched_itmt_capable) {
104 mutex_unlock(&itmt_update_mutex);
105 return 0;
106 }
107
108 itmt_sysctl_header = register_sysctl("kernel", itmt_kern_table);
109 if (!itmt_sysctl_header) {
110 mutex_unlock(&itmt_update_mutex);
111 return -ENOMEM;
112 }
113
114 sched_itmt_capable = true;
115
116 sysctl_sched_itmt_enabled = 1;
117
118 x86_topology_update = true;
119 rebuild_sched_domains();
120
121 mutex_unlock(&itmt_update_mutex);
122
123 return 0;
124 }
125
126 /**
127 * sched_clear_itmt_support() - Revoke platform's support of ITMT
128 *
129 * This function is used by the OS to indicate that it has
130 * revoked the platform's support of ITMT feature.
131 *
132 * It must not be called with cpu hot plug lock
133 * held as we need to acquire the lock to rebuild sched domains
134 * later.
135 */
sched_clear_itmt_support(void)136 void sched_clear_itmt_support(void)
137 {
138 mutex_lock(&itmt_update_mutex);
139
140 if (!sched_itmt_capable) {
141 mutex_unlock(&itmt_update_mutex);
142 return;
143 }
144 sched_itmt_capable = false;
145
146 if (itmt_sysctl_header) {
147 unregister_sysctl_table(itmt_sysctl_header);
148 itmt_sysctl_header = NULL;
149 }
150
151 if (sysctl_sched_itmt_enabled) {
152 /* disable sched_itmt if we are no longer ITMT capable */
153 sysctl_sched_itmt_enabled = 0;
154 x86_topology_update = true;
155 rebuild_sched_domains();
156 }
157
158 mutex_unlock(&itmt_update_mutex);
159 }
160
arch_asym_cpu_priority(int cpu)161 int arch_asym_cpu_priority(int cpu)
162 {
163 return per_cpu(sched_core_priority, cpu);
164 }
165
166 /**
167 * sched_set_itmt_core_prio() - Set CPU priority based on ITMT
168 * @prio: Priority of @cpu
169 * @cpu: The CPU number
170 *
171 * The pstate driver will find out the max boost frequency
172 * and call this function to set a priority proportional
173 * to the max boost frequency. CPUs with higher boost
174 * frequency will receive higher priority.
175 *
176 * No need to rebuild sched domain after updating
177 * the CPU priorities. The sched domains have no
178 * dependency on CPU priorities.
179 */
sched_set_itmt_core_prio(int prio,int cpu)180 void sched_set_itmt_core_prio(int prio, int cpu)
181 {
182 per_cpu(sched_core_priority, cpu) = prio;
183 }
184