1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* 4 * A simple wrapper around refcount. An allocated sched_core_cookie's 5 * address is used to compute the cookie of the task. 6 */ 7 struct sched_core_cookie { 8 refcount_t refcnt; 9 }; 10 11 static unsigned long sched_core_alloc_cookie(void) 12 { 13 struct sched_core_cookie *ck = kmalloc(sizeof(*ck), GFP_KERNEL); 14 if (!ck) 15 return 0; 16 17 refcount_set(&ck->refcnt, 1); 18 sched_core_get(); 19 20 return (unsigned long)ck; 21 } 22 23 static void sched_core_put_cookie(unsigned long cookie) 24 { 25 struct sched_core_cookie *ptr = (void *)cookie; 26 27 if (ptr && refcount_dec_and_test(&ptr->refcnt)) { 28 kfree(ptr); 29 sched_core_put(); 30 } 31 } 32 33 static unsigned long sched_core_get_cookie(unsigned long cookie) 34 { 35 struct sched_core_cookie *ptr = (void *)cookie; 36 37 if (ptr) 38 refcount_inc(&ptr->refcnt); 39 40 return cookie; 41 } 42 43 /* 44 * sched_core_update_cookie - replace the cookie on a task 45 * @p: the task to update 46 * @cookie: the new cookie 47 * 48 * Effectively exchange the task cookie; caller is responsible for lifetimes on 49 * both ends. 50 * 51 * Returns: the old cookie 52 */ 53 static unsigned long sched_core_update_cookie(struct task_struct *p, 54 unsigned long cookie) 55 { 56 unsigned long old_cookie; 57 struct rq_flags rf; 58 struct rq *rq; 59 60 rq = task_rq_lock(p, &rf); 61 62 /* 63 * Since creating a cookie implies sched_core_get(), and we cannot set 64 * a cookie until after we've created it, similarly, we cannot destroy 65 * a cookie until after we've removed it, we must have core scheduling 66 * enabled here. 67 */ 68 SCHED_WARN_ON((p->core_cookie || cookie) && !sched_core_enabled(rq)); 69 70 if (sched_core_enqueued(p)) 71 sched_core_dequeue(rq, p, DEQUEUE_SAVE); 72 73 old_cookie = p->core_cookie; 74 p->core_cookie = cookie; 75 76 /* 77 * Consider the cases: !prev_cookie and !cookie. 78 */ 79 if (cookie && task_on_rq_queued(p)) 80 sched_core_enqueue(rq, p); 81 82 /* 83 * If task is currently running, it may not be compatible anymore after 84 * the cookie change, so enter the scheduler on its CPU to schedule it 85 * away. 86 * 87 * Note that it is possible that as a result of this cookie change, the 88 * core has now entered/left forced idle state. Defer accounting to the 89 * next scheduling edge, rather than always forcing a reschedule here. 90 */ 91 if (task_running(rq, p)) 92 resched_curr(rq); 93 94 task_rq_unlock(rq, p, &rf); 95 96 return old_cookie; 97 } 98 99 static unsigned long sched_core_clone_cookie(struct task_struct *p) 100 { 101 unsigned long cookie, flags; 102 103 raw_spin_lock_irqsave(&p->pi_lock, flags); 104 cookie = sched_core_get_cookie(p->core_cookie); 105 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 106 107 return cookie; 108 } 109 110 void sched_core_fork(struct task_struct *p) 111 { 112 RB_CLEAR_NODE(&p->core_node); 113 p->core_cookie = sched_core_clone_cookie(current); 114 } 115 116 void sched_core_free(struct task_struct *p) 117 { 118 sched_core_put_cookie(p->core_cookie); 119 } 120 121 static void __sched_core_set(struct task_struct *p, unsigned long cookie) 122 { 123 cookie = sched_core_get_cookie(cookie); 124 cookie = sched_core_update_cookie(p, cookie); 125 sched_core_put_cookie(cookie); 126 } 127 128 /* Called from prctl interface: PR_SCHED_CORE */ 129 int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, 130 unsigned long uaddr) 131 { 132 unsigned long cookie = 0, id = 0; 133 struct task_struct *task, *p; 134 struct pid *grp; 135 int err = 0; 136 137 if (!static_branch_likely(&sched_smt_present)) 138 return -ENODEV; 139 140 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID); 141 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD_GROUP != PIDTYPE_TGID); 142 BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_PROCESS_GROUP != PIDTYPE_PGID); 143 144 if (type > PIDTYPE_PGID || cmd >= PR_SCHED_CORE_MAX || pid < 0 || 145 (cmd != PR_SCHED_CORE_GET && uaddr)) 146 return -EINVAL; 147 148 rcu_read_lock(); 149 if (pid == 0) { 150 task = current; 151 } else { 152 task = find_task_by_vpid(pid); 153 if (!task) { 154 rcu_read_unlock(); 155 return -ESRCH; 156 } 157 } 158 get_task_struct(task); 159 rcu_read_unlock(); 160 161 /* 162 * Check if this process has the right to modify the specified 163 * process. Use the regular "ptrace_may_access()" checks. 164 */ 165 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) { 166 err = -EPERM; 167 goto out; 168 } 169 170 switch (cmd) { 171 case PR_SCHED_CORE_GET: 172 if (type != PIDTYPE_PID || uaddr & 7) { 173 err = -EINVAL; 174 goto out; 175 } 176 cookie = sched_core_clone_cookie(task); 177 if (cookie) { 178 /* XXX improve ? */ 179 ptr_to_hashval((void *)cookie, &id); 180 } 181 err = put_user(id, (u64 __user *)uaddr); 182 goto out; 183 184 case PR_SCHED_CORE_CREATE: 185 cookie = sched_core_alloc_cookie(); 186 if (!cookie) { 187 err = -ENOMEM; 188 goto out; 189 } 190 break; 191 192 case PR_SCHED_CORE_SHARE_TO: 193 cookie = sched_core_clone_cookie(current); 194 break; 195 196 case PR_SCHED_CORE_SHARE_FROM: 197 if (type != PIDTYPE_PID) { 198 err = -EINVAL; 199 goto out; 200 } 201 cookie = sched_core_clone_cookie(task); 202 __sched_core_set(current, cookie); 203 goto out; 204 205 default: 206 err = -EINVAL; 207 goto out; 208 }; 209 210 if (type == PIDTYPE_PID) { 211 __sched_core_set(task, cookie); 212 goto out; 213 } 214 215 read_lock(&tasklist_lock); 216 grp = task_pid_type(task, type); 217 218 do_each_pid_thread(grp, type, p) { 219 if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS)) { 220 err = -EPERM; 221 goto out_tasklist; 222 } 223 } while_each_pid_thread(grp, type, p); 224 225 do_each_pid_thread(grp, type, p) { 226 __sched_core_set(p, cookie); 227 } while_each_pid_thread(grp, type, p); 228 out_tasklist: 229 read_unlock(&tasklist_lock); 230 231 out: 232 sched_core_put_cookie(cookie); 233 put_task_struct(task); 234 return err; 235 } 236 237 #ifdef CONFIG_SCHEDSTATS 238 239 /* REQUIRES: rq->core's clock recently updated. */ 240 void __sched_core_account_forceidle(struct rq *rq) 241 { 242 const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); 243 u64 delta, now = rq_clock(rq->core); 244 struct rq *rq_i; 245 struct task_struct *p; 246 int i; 247 248 lockdep_assert_rq_held(rq); 249 250 WARN_ON_ONCE(!rq->core->core_forceidle_count); 251 252 if (rq->core->core_forceidle_start == 0) 253 return; 254 255 delta = now - rq->core->core_forceidle_start; 256 if (unlikely((s64)delta <= 0)) 257 return; 258 259 rq->core->core_forceidle_start = now; 260 261 if (WARN_ON_ONCE(!rq->core->core_forceidle_occupation)) { 262 /* can't be forced idle without a running task */ 263 } else if (rq->core->core_forceidle_count > 1 || 264 rq->core->core_forceidle_occupation > 1) { 265 /* 266 * For larger SMT configurations, we need to scale the charged 267 * forced idle amount since there can be more than one forced 268 * idle sibling and more than one running cookied task. 269 */ 270 delta *= rq->core->core_forceidle_count; 271 delta = div_u64(delta, rq->core->core_forceidle_occupation); 272 } 273 274 for_each_cpu(i, smt_mask) { 275 rq_i = cpu_rq(i); 276 p = rq_i->core_pick ?: rq_i->curr; 277 278 if (p == rq_i->idle) 279 continue; 280 281 /* 282 * Note: this will account forceidle to the current cpu, even 283 * if it comes from our SMT sibling. 284 */ 285 __account_forceidle_time(p, delta); 286 } 287 } 288 289 void __sched_core_tick(struct rq *rq) 290 { 291 if (!rq->core->core_forceidle_count) 292 return; 293 294 if (rq != rq->core) 295 update_rq_clock(rq->core); 296 297 __sched_core_account_forceidle(rq); 298 } 299 300 #endif /* CONFIG_SCHEDSTATS */ 301