1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * A simple wrapper around refcount. An allocated sched_core_cookie's 5 * address is used to compute the cookie of the task. 6 */ 7 struct sched_core_cookie { 8 refcount_t refcnt; 9 }; 10 11 static unsigned long sched_core_alloc_cookie(void) 12 { 13 struct sched_core_cookie *ck = kmalloc(sizeof(*ck), GFP_KERNEL); 14 if (!ck) 15 return 0; 16 17 refcount_set(&ck->refcnt, 1); 18 sched_core_get(); 19 20 return (unsigned long)ck; 21 } 22 23 static void sched_core_put_cookie(unsigned long cookie) 24 { 25 struct sched_core_cookie *ptr = (void *)cookie; 26 27 if (ptr && refcount_dec_and_test(&ptr->refcnt)) { 28 kfree(ptr); 29 sched_core_put(); 30 } 31 } 32 33 static unsigned long sched_core_get_cookie(unsigned long cookie) 34 { 35 struct sched_core_cookie *ptr = (void *)cookie; 36 37 if (ptr) 38 refcount_inc(&ptr->refcnt); 39 40 return cookie; 41 } 42 43 /* 44 * sched_core_update_cookie - replace the cookie on a task 45 * @p: the task to update 46 * @cookie: the new cookie 47 * 48 * Effectively exchange the task cookie; caller is responsible for lifetimes on 49 * both ends. 50 * 51 * Returns: the old cookie 52 */ 53 static unsigned long sched_core_update_cookie(struct task_struct *p, 54 unsigned long cookie) 55 { 56 unsigned long old_cookie; 57 struct rq_flags rf; 58 struct rq *rq; 59 bool enqueued; 60 61 rq = task_rq_lock(p, &rf); 62 63 /* 64 * Since creating a cookie implies sched_core_get(), and we cannot set 65 * a cookie until after we've created it, similarly, we cannot destroy 66 * a cookie until after we've removed it, we must have core scheduling 67 * enabled here. 68 */ 69 SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq)); 70 71 enqueued = sched_core_enqueued(p); 72 if (enqueued) 73 sched_core_dequeue(rq, p, DEQUEUE_SAVE); 74 75 old_cookie = p->core_cookie; 76 p->core_cookie = cookie; 77 78 if (enqueued) 79 sched_core_enqueue(rq, p); 80 81 /* 82 * If task is currently running, it may not be compatible anymore after 83 * the cookie change, so enter the scheduler on its CPU to schedule it 84 * away. 85 * 86 * Note that it is possible that as a result of this cookie change, the 87 * core has now entered/left forced idle state. Defer accounting to the 88 * next scheduling edge, rather than always forcing a reschedule here. 89 */ 90 if (task_running(rq, p)) 91 resched_curr(rq); 92 93 task_rq_unlock(rq, p, &rf); 94 95 return old_cookie; 96 } 97 98 static unsigned long sched_core_clone_cookie(struct task_struct *p) 99 { 100 unsigned long cookie, flags; 101 102 raw_spin_lock_irqsave(&p->pi_lock, flags); 103 cookie = sched_core_get_cookie(p->core_cookie); 104 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 105 106 return cookie; 107 } 108 109 void sched_core_fork(struct task_struct *p) 110 { 111 RB_CLEAR_NODE(&p->core_node); 112 p->core_cookie = sched_core_clone_cookie(current); 113 } 114 115 void sched_core_free(struct task_struct *p) 116 { 117 sched_core_put_cookie(p->core_cookie); 118 } 119 120 static void __sched_core_set(struct task_struct *p, unsigned long cookie) 121 { 122 cookie = sched_core_get_cookie(cookie); 123 cookie = sched_core_update_cookie(p, cookie); 124 sched_core_put_cookie(cookie); 125 } 126 127 /* Called from prctl interface: PR_SCHED_CORE */ 128 int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, 129 unsigned long uaddr) 130 { 131 unsigned long cookie = 0, id = 0; 132 struct task_struct *task, *p; 133 struct pid *grp; 134 int err = 0; 135 136 if (!static_branch_likely(&sched_smt_present)) 137 return -ENODEV; 138 139 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID); 140 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD_GROUP != PIDTYPE_TGID); 141 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_PROCESS_GROUP != PIDTYPE_PGID); 142 143 if (type > PIDTYPE_PGID || cmd >= PR_SCHED_CORE_MAX || pid < 0 || 144 (cmd != PR_SCHED_CORE_GET && uaddr)) 145 return -EINVAL; 146 147 rcu_read_lock(); 148 if (pid == 0) { 149 task = current; 150 } else { 151 task = find_task_by_vpid(pid); 152 if (!task) { 153 rcu_read_unlock(); 154 return -ESRCH; 155 } 156 } 157 get_task_struct(task); 158 rcu_read_unlock(); 159 160 /* 161 * Check if this process has the right to modify the specified 162 * process. Use the regular "ptrace_may_access()" checks. 163 */ 164 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) { 165 err = -EPERM; 166 goto out; 167 } 168 169 switch (cmd) { 170 case PR_SCHED_CORE_GET: 171 if (type != PIDTYPE_PID || uaddr & 7) { 172 err = -EINVAL; 173 goto out; 174 } 175 cookie = sched_core_clone_cookie(task); 176 if (cookie) { 177 /* XXX improve ? */ 178 ptr_to_hashval((void *)cookie, &id); 179 } 180 err = put_user(id, (u64 __user *)uaddr); 181 goto out; 182 183 case PR_SCHED_CORE_CREATE: 184 cookie = sched_core_alloc_cookie(); 185 if (!cookie) { 186 err = -ENOMEM; 187 goto out; 188 } 189 break; 190 191 case PR_SCHED_CORE_SHARE_TO: 192 cookie = sched_core_clone_cookie(current); 193 break; 194 195 case PR_SCHED_CORE_SHARE_FROM: 196 if (type != PIDTYPE_PID) { 197 err = -EINVAL; 198 goto out; 199 } 200 cookie = sched_core_clone_cookie(task); 201 __sched_core_set(current, cookie); 202 goto out; 203 204 default: 205 err = -EINVAL; 206 goto out; 207 }; 208 209 if (type == PIDTYPE_PID) { 210 __sched_core_set(task, cookie); 211 goto out; 212 } 213 214 read_lock(&tasklist_lock); 215 grp = task_pid_type(task, type); 216 217 do_each_pid_thread(grp, type, p) { 218 if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) { 219 err = -EPERM; 220 goto out_tasklist; 221 } 222 } while_each_pid_thread(grp, type, p); 223 224 do_each_pid_thread(grp, type, p) { 225 __sched_core_set(p, cookie); 226 } while_each_pid_thread(grp, type, p); 227 out_tasklist: 228 read_unlock(&tasklist_lock); 229 230 out: 231 sched_core_put_cookie(cookie); 232 put_task_struct(task); 233 return err; 234 } 235 236 #ifdef CONFIG_SCHEDSTATS 237 238 /* REQUIRES: rq->core's clock recently updated. */ 239 void __sched_core_account_forceidle(struct rq *rq) 240 { 241 const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); 242 u64 delta, now = rq_clock(rq->core); 243 struct rq *rq_i; 244 struct task_struct *p; 245 int i; 246 247 lockdep_assert_rq_held(rq); 248 249 WARN_ON_ONCE(!rq->core->core_forceidle_count); 250 251 if (rq->core->core_forceidle_start == 0) 252 return; 253 254 delta = now - rq->core->core_forceidle_start; 255 if (unlikely((s64)delta <= 0)) 256 return; 257 258 rq->core->core_forceidle_start = now; 259 260 if (WARN_ON_ONCE(!rq->core->core_forceidle_occupation)) { 261 /* can't be forced idle without a running task */ 262 } else if (rq->core->core_forceidle_count > 1 || 263 rq->core->core_forceidle_occupation > 1) { 264 /* 265 * For larger SMT configurations, we need to scale the charged 266 * forced idle amount since there can be more than one forced 267 * idle sibling and more than one running cookied task. 268 */ 269 delta *= rq->core->core_forceidle_count; 270 delta = div_u64(delta, rq->core->core_forceidle_occupation); 271 } 272 273 for_each_cpu(i, smt_mask) { 274 rq_i = cpu_rq(i); 275 p = rq_i->core_pick ?: rq_i->curr; 276 277 if (p == rq_i->idle) 278 continue; 279 280 __schedstat_add(p->stats.core_forceidle_sum, delta); 281 } 282 } 283 284 void __sched_core_tick(struct rq *rq) 285 { 286 if (!rq->core->core_forceidle_count) 287 return; 288 289 if (rq != rq->core) 290 update_rq_clock(rq->core); 291 292 __sched_core_account_forceidle(rq); 293 } 294 295 #endif /* CONFIG_SCHEDSTATS */ 296