xref: /openbmc/linux/arch/x86/kernel/itmt.c (revision f89d416a)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * itmt.c: Support Intel Turbo Boost Max Technology 3.0
4  *
5  * (C) Copyright 2016 Intel Corporation
6  * Author: Tim Chen <tim.c.chen@linux.intel.com>
7  *
8  * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT),
9  * the maximum turbo frequencies of some cores in a CPU package may be
10  * higher than for the other cores in the same package.  In that case,
11  * better performance can be achieved by making the scheduler prefer
12  * to run tasks on the CPUs with higher max turbo frequencies.
13  *
14  * This file provides functions and data structures for enabling the
15  * scheduler to favor scheduling on cores can be boosted to a higher
16  * frequency under ITMT.
17  */
18 
19 #include <linux/sched.h>
20 #include <linux/cpumask.h>
21 #include <linux/cpuset.h>
22 #include <linux/mutex.h>
23 #include <linux/sysctl.h>
24 #include <linux/nodemask.h>
25 
26 static DEFINE_MUTEX(itmt_update_mutex);
27 DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
28 
29 /* Boolean to track if system has ITMT capabilities */
30 static bool __read_mostly sched_itmt_capable;
31 
32 /*
33  * Boolean to control whether we want to move processes to cpu capable
34  * of higher turbo frequency for cpus supporting Intel Turbo Boost Max
35  * Technology 3.0.
36  *
37  * It can be set via /proc/sys/kernel/sched_itmt_enabled
38  */
39 unsigned int __read_mostly sysctl_sched_itmt_enabled;
40 
41 static int sched_itmt_update_handler(struct ctl_table *table, int write,
42 				     void __user *buffer, size_t *lenp,
43 				     loff_t *ppos)
44 {
45 	unsigned int old_sysctl;
46 	int ret;
47 
48 	mutex_lock(&itmt_update_mutex);
49 
50 	if (!sched_itmt_capable) {
51 		mutex_unlock(&itmt_update_mutex);
52 		return -EINVAL;
53 	}
54 
55 	old_sysctl = sysctl_sched_itmt_enabled;
56 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
57 
58 	if (!ret && write && old_sysctl != sysctl_sched_itmt_enabled) {
59 		x86_topology_update = true;
60 		rebuild_sched_domains();
61 	}
62 
63 	mutex_unlock(&itmt_update_mutex);
64 
65 	return ret;
66 }
67 
68 static struct ctl_table itmt_kern_table[] = {
69 	{
70 		.procname	= "sched_itmt_enabled",
71 		.data		= &sysctl_sched_itmt_enabled,
72 		.maxlen		= sizeof(unsigned int),
73 		.mode		= 0644,
74 		.proc_handler	= sched_itmt_update_handler,
75 		.extra1		= SYSCTL_ZERO,
76 		.extra2		= SYSCTL_ONE,
77 	},
78 	{}
79 };
80 
81 static struct ctl_table itmt_root_table[] = {
82 	{
83 		.procname	= "kernel",
84 		.mode		= 0555,
85 		.child		= itmt_kern_table,
86 	},
87 	{}
88 };
89 
90 static struct ctl_table_header *itmt_sysctl_header;
91 
92 /**
93  * sched_set_itmt_support() - Indicate platform supports ITMT
94  *
95  * This function is used by the OS to indicate to scheduler that the platform
96  * is capable of supporting the ITMT feature.
97  *
98  * The current scheme has the pstate driver detects if the system
99  * is ITMT capable and call sched_set_itmt_support.
100  *
101  * This must be done only after sched_set_itmt_core_prio
102  * has been called to set the cpus' priorities.
103  * It must not be called with cpu hot plug lock
104  * held as we need to acquire the lock to rebuild sched domains
105  * later.
106  *
107  * Return: 0 on success
108  */
109 int sched_set_itmt_support(void)
110 {
111 	mutex_lock(&itmt_update_mutex);
112 
113 	if (sched_itmt_capable) {
114 		mutex_unlock(&itmt_update_mutex);
115 		return 0;
116 	}
117 
118 	itmt_sysctl_header = register_sysctl_table(itmt_root_table);
119 	if (!itmt_sysctl_header) {
120 		mutex_unlock(&itmt_update_mutex);
121 		return -ENOMEM;
122 	}
123 
124 	sched_itmt_capable = true;
125 
126 	sysctl_sched_itmt_enabled = 1;
127 
128 	x86_topology_update = true;
129 	rebuild_sched_domains();
130 
131 	mutex_unlock(&itmt_update_mutex);
132 
133 	return 0;
134 }
135 
136 /**
137  * sched_clear_itmt_support() - Revoke platform's support of ITMT
138  *
139  * This function is used by the OS to indicate that it has
140  * revoked the platform's support of ITMT feature.
141  *
142  * It must not be called with cpu hot plug lock
143  * held as we need to acquire the lock to rebuild sched domains
144  * later.
145  */
146 void sched_clear_itmt_support(void)
147 {
148 	mutex_lock(&itmt_update_mutex);
149 
150 	if (!sched_itmt_capable) {
151 		mutex_unlock(&itmt_update_mutex);
152 		return;
153 	}
154 	sched_itmt_capable = false;
155 
156 	if (itmt_sysctl_header) {
157 		unregister_sysctl_table(itmt_sysctl_header);
158 		itmt_sysctl_header = NULL;
159 	}
160 
161 	if (sysctl_sched_itmt_enabled) {
162 		/* disable sched_itmt if we are no longer ITMT capable */
163 		sysctl_sched_itmt_enabled = 0;
164 		x86_topology_update = true;
165 		rebuild_sched_domains();
166 	}
167 
168 	mutex_unlock(&itmt_update_mutex);
169 }
170 
171 int arch_asym_cpu_priority(int cpu)
172 {
173 	return per_cpu(sched_core_priority, cpu);
174 }
175 
176 /**
177  * sched_set_itmt_core_prio() - Set CPU priority based on ITMT
178  * @prio:	Priority of cpu core
179  * @core_cpu:	The cpu number associated with the core
180  *
181  * The pstate driver will find out the max boost frequency
182  * and call this function to set a priority proportional
183  * to the max boost frequency. CPU with higher boost
184  * frequency will receive higher priority.
185  *
186  * No need to rebuild sched domain after updating
187  * the CPU priorities. The sched domains have no
188  * dependency on CPU priorities.
189  */
190 void sched_set_itmt_core_prio(int prio, int core_cpu)
191 {
192 	int cpu, i = 1;
193 
194 	for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) {
195 		int smt_prio;
196 
197 		/*
198 		 * Ensure that the siblings are moved to the end
199 		 * of the priority chain and only used when
200 		 * all other high priority cpus are out of capacity.
201 		 */
202 		smt_prio = prio * smp_num_siblings / i;
203 		per_cpu(sched_core_priority, cpu) = smt_prio;
204 		i++;
205 	}
206 }
207