1 #include <linux/kdebug.h> 2 #include <linux/kprobes.h> 3 #include <linux/export.h> 4 #include <linux/notifier.h> 5 #include <linux/rcupdate.h> 6 #include <linux/vmalloc.h> 7 #include <linux/reboot.h> 8 9 /* 10 * Notifier list for kernel code which wants to be called 11 * at shutdown. This is used to stop any idling DMA operations 12 * and the like. 13 */ 14 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); 15 16 /* 17 * Notifier chain core routines. The exported routines below 18 * are layered on top of these, with appropriate locking added. 19 */ 20 21 static int notifier_chain_register(struct notifier_block **nl, 22 struct notifier_block *n) 23 { 24 while ((*nl) != NULL) { 25 WARN_ONCE(((*nl) == n), "double register detected"); 26 if (n->priority > (*nl)->priority) 27 break; 28 nl = &((*nl)->next); 29 } 30 n->next = *nl; 31 rcu_assign_pointer(*nl, n); 32 return 0; 33 } 34 35 static int notifier_chain_cond_register(struct notifier_block **nl, 36 struct notifier_block *n) 37 { 38 while ((*nl) != NULL) { 39 if ((*nl) == n) 40 return 0; 41 if (n->priority > (*nl)->priority) 42 break; 43 nl = &((*nl)->next); 44 } 45 n->next = *nl; 46 rcu_assign_pointer(*nl, n); 47 return 0; 48 } 49 50 static int notifier_chain_unregister(struct notifier_block **nl, 51 struct notifier_block *n) 52 { 53 while ((*nl) != NULL) { 54 if ((*nl) == n) { 55 rcu_assign_pointer(*nl, n->next); 56 return 0; 57 } 58 nl = &((*nl)->next); 59 } 60 return -ENOENT; 61 } 62 63 /** 64 * notifier_call_chain - Informs the registered notifiers about an event. 65 * @nl: Pointer to head of the blocking notifier chain 66 * @val: Value passed unmodified to notifier function 67 * @v: Pointer passed unmodified to notifier function 68 * @nr_to_call: Number of notifier functions to be called. Don't care 69 * value of this parameter is -1. 70 * @nr_calls: Records the number of notifications sent. Don't care 71 * value of this field is NULL. 72 * @returns: notifier_call_chain returns the value returned by the 73 * last notifier function called. 74 */ 75 static int notifier_call_chain(struct notifier_block **nl, 76 unsigned long val, void *v, 77 int nr_to_call, int *nr_calls) 78 { 79 int ret = NOTIFY_DONE; 80 struct notifier_block *nb, *next_nb; 81 82 nb = rcu_dereference_raw(*nl); 83 84 while (nb && nr_to_call) { 85 next_nb = rcu_dereference_raw(nb->next); 86 87 #ifdef CONFIG_DEBUG_NOTIFIERS 88 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { 89 WARN(1, "Invalid notifier called!"); 90 nb = next_nb; 91 continue; 92 } 93 #endif 94 ret = nb->notifier_call(nb, val, v); 95 96 if (nr_calls) 97 (*nr_calls)++; 98 99 if (ret & NOTIFY_STOP_MASK) 100 break; 101 nb = next_nb; 102 nr_to_call--; 103 } 104 return ret; 105 } 106 NOKPROBE_SYMBOL(notifier_call_chain); 107 108 /* 109 * Atomic notifier chain routines. Registration and unregistration 110 * use a spinlock, and call_chain is synchronized by RCU (no locks). 111 */ 112 113 /** 114 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain 115 * @nh: Pointer to head of the atomic notifier chain 116 * @n: New entry in notifier chain 117 * 118 * Adds a notifier to an atomic notifier chain. 119 * 120 * Currently always returns zero. 121 */ 122 int atomic_notifier_chain_register(struct atomic_notifier_head *nh, 123 struct notifier_block *n) 124 { 125 unsigned long flags; 126 int ret; 127 128 spin_lock_irqsave(&nh->lock, flags); 129 ret = notifier_chain_register(&nh->head, n); 130 spin_unlock_irqrestore(&nh->lock, flags); 131 return ret; 132 } 133 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); 134 135 /** 136 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain 137 * @nh: Pointer to head of the atomic notifier chain 138 * @n: Entry to remove from notifier chain 139 * 140 * Removes a notifier from an atomic notifier chain. 141 * 142 * Returns zero on success or %-ENOENT on failure. 143 */ 144 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, 145 struct notifier_block *n) 146 { 147 unsigned long flags; 148 int ret; 149 150 spin_lock_irqsave(&nh->lock, flags); 151 ret = notifier_chain_unregister(&nh->head, n); 152 spin_unlock_irqrestore(&nh->lock, flags); 153 synchronize_rcu(); 154 return ret; 155 } 156 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); 157 158 /** 159 * __atomic_notifier_call_chain - Call functions in an atomic notifier chain 160 * @nh: Pointer to head of the atomic notifier chain 161 * @val: Value passed unmodified to notifier function 162 * @v: Pointer passed unmodified to notifier function 163 * @nr_to_call: See the comment for notifier_call_chain. 164 * @nr_calls: See the comment for notifier_call_chain. 165 * 166 * Calls each function in a notifier chain in turn. The functions 167 * run in an atomic context, so they must not block. 168 * This routine uses RCU to synchronize with changes to the chain. 169 * 170 * If the return value of the notifier can be and'ed 171 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() 172 * will return immediately, with the return value of 173 * the notifier function which halted execution. 174 * Otherwise the return value is the return value 175 * of the last notifier function called. 176 */ 177 int __atomic_notifier_call_chain(struct atomic_notifier_head *nh, 178 unsigned long val, void *v, 179 int nr_to_call, int *nr_calls) 180 { 181 int ret; 182 183 rcu_read_lock(); 184 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); 185 rcu_read_unlock(); 186 return ret; 187 } 188 EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain); 189 NOKPROBE_SYMBOL(__atomic_notifier_call_chain); 190 191 int atomic_notifier_call_chain(struct atomic_notifier_head *nh, 192 unsigned long val, void *v) 193 { 194 return __atomic_notifier_call_chain(nh, val, v, -1, NULL); 195 } 196 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); 197 NOKPROBE_SYMBOL(atomic_notifier_call_chain); 198 199 /* 200 * Blocking notifier chain routines. All access to the chain is 201 * synchronized by an rwsem. 202 */ 203 204 /** 205 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain 206 * @nh: Pointer to head of the blocking notifier chain 207 * @n: New entry in notifier chain 208 * 209 * Adds a notifier to a blocking notifier chain. 210 * Must be called in process context. 211 * 212 * Currently always returns zero. 213 */ 214 int blocking_notifier_chain_register(struct blocking_notifier_head *nh, 215 struct notifier_block *n) 216 { 217 int ret; 218 219 /* 220 * This code gets used during boot-up, when task switching is 221 * not yet working and interrupts must remain disabled. At 222 * such times we must not call down_write(). 223 */ 224 if (unlikely(system_state == SYSTEM_BOOTING)) 225 return notifier_chain_register(&nh->head, n); 226 227 down_write(&nh->rwsem); 228 ret = notifier_chain_register(&nh->head, n); 229 up_write(&nh->rwsem); 230 return ret; 231 } 232 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); 233 234 /** 235 * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain 236 * @nh: Pointer to head of the blocking notifier chain 237 * @n: New entry in notifier chain 238 * 239 * Adds a notifier to a blocking notifier chain, only if not already 240 * present in the chain. 241 * Must be called in process context. 242 * 243 * Currently always returns zero. 244 */ 245 int blocking_notifier_chain_cond_register(struct blocking_notifier_head *nh, 246 struct notifier_block *n) 247 { 248 int ret; 249 250 down_write(&nh->rwsem); 251 ret = notifier_chain_cond_register(&nh->head, n); 252 up_write(&nh->rwsem); 253 return ret; 254 } 255 EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register); 256 257 /** 258 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain 259 * @nh: Pointer to head of the blocking notifier chain 260 * @n: Entry to remove from notifier chain 261 * 262 * Removes a notifier from a blocking notifier chain. 263 * Must be called from process context. 264 * 265 * Returns zero on success or %-ENOENT on failure. 266 */ 267 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, 268 struct notifier_block *n) 269 { 270 int ret; 271 272 /* 273 * This code gets used during boot-up, when task switching is 274 * not yet working and interrupts must remain disabled. At 275 * such times we must not call down_write(). 276 */ 277 if (unlikely(system_state == SYSTEM_BOOTING)) 278 return notifier_chain_unregister(&nh->head, n); 279 280 down_write(&nh->rwsem); 281 ret = notifier_chain_unregister(&nh->head, n); 282 up_write(&nh->rwsem); 283 return ret; 284 } 285 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); 286 287 /** 288 * __blocking_notifier_call_chain - Call functions in a blocking notifier chain 289 * @nh: Pointer to head of the blocking notifier chain 290 * @val: Value passed unmodified to notifier function 291 * @v: Pointer passed unmodified to notifier function 292 * @nr_to_call: See comment for notifier_call_chain. 293 * @nr_calls: See comment for notifier_call_chain. 294 * 295 * Calls each function in a notifier chain in turn. The functions 296 * run in a process context, so they are allowed to block. 297 * 298 * If the return value of the notifier can be and'ed 299 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() 300 * will return immediately, with the return value of 301 * the notifier function which halted execution. 302 * Otherwise the return value is the return value 303 * of the last notifier function called. 304 */ 305 int __blocking_notifier_call_chain(struct blocking_notifier_head *nh, 306 unsigned long val, void *v, 307 int nr_to_call, int *nr_calls) 308 { 309 int ret = NOTIFY_DONE; 310 311 /* 312 * We check the head outside the lock, but if this access is 313 * racy then it does not matter what the result of the test 314 * is, we re-check the list after having taken the lock anyway: 315 */ 316 if (rcu_access_pointer(nh->head)) { 317 down_read(&nh->rwsem); 318 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, 319 nr_calls); 320 up_read(&nh->rwsem); 321 } 322 return ret; 323 } 324 EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain); 325 326 int blocking_notifier_call_chain(struct blocking_notifier_head *nh, 327 unsigned long val, void *v) 328 { 329 return __blocking_notifier_call_chain(nh, val, v, -1, NULL); 330 } 331 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); 332 333 /* 334 * Raw notifier chain routines. There is no protection; 335 * the caller must provide it. Use at your own risk! 336 */ 337 338 /** 339 * raw_notifier_chain_register - Add notifier to a raw notifier chain 340 * @nh: Pointer to head of the raw notifier chain 341 * @n: New entry in notifier chain 342 * 343 * Adds a notifier to a raw notifier chain. 344 * All locking must be provided by the caller. 345 * 346 * Currently always returns zero. 347 */ 348 int raw_notifier_chain_register(struct raw_notifier_head *nh, 349 struct notifier_block *n) 350 { 351 return notifier_chain_register(&nh->head, n); 352 } 353 EXPORT_SYMBOL_GPL(raw_notifier_chain_register); 354 355 /** 356 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain 357 * @nh: Pointer to head of the raw notifier chain 358 * @n: Entry to remove from notifier chain 359 * 360 * Removes a notifier from a raw notifier chain. 361 * All locking must be provided by the caller. 362 * 363 * Returns zero on success or %-ENOENT on failure. 364 */ 365 int raw_notifier_chain_unregister(struct raw_notifier_head *nh, 366 struct notifier_block *n) 367 { 368 return notifier_chain_unregister(&nh->head, n); 369 } 370 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); 371 372 /** 373 * __raw_notifier_call_chain - Call functions in a raw notifier chain 374 * @nh: Pointer to head of the raw notifier chain 375 * @val: Value passed unmodified to notifier function 376 * @v: Pointer passed unmodified to notifier function 377 * @nr_to_call: See comment for notifier_call_chain. 378 * @nr_calls: See comment for notifier_call_chain 379 * 380 * Calls each function in a notifier chain in turn. The functions 381 * run in an undefined context. 382 * All locking must be provided by the caller. 383 * 384 * If the return value of the notifier can be and'ed 385 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() 386 * will return immediately, with the return value of 387 * the notifier function which halted execution. 388 * Otherwise the return value is the return value 389 * of the last notifier function called. 390 */ 391 int __raw_notifier_call_chain(struct raw_notifier_head *nh, 392 unsigned long val, void *v, 393 int nr_to_call, int *nr_calls) 394 { 395 return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); 396 } 397 EXPORT_SYMBOL_GPL(__raw_notifier_call_chain); 398 399 int raw_notifier_call_chain(struct raw_notifier_head *nh, 400 unsigned long val, void *v) 401 { 402 return __raw_notifier_call_chain(nh, val, v, -1, NULL); 403 } 404 EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 405 406 #ifdef CONFIG_SRCU 407 /* 408 * SRCU notifier chain routines. Registration and unregistration 409 * use a mutex, and call_chain is synchronized by SRCU (no locks). 410 */ 411 412 /** 413 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain 414 * @nh: Pointer to head of the SRCU notifier chain 415 * @n: New entry in notifier chain 416 * 417 * Adds a notifier to an SRCU notifier chain. 418 * Must be called in process context. 419 * 420 * Currently always returns zero. 421 */ 422 int srcu_notifier_chain_register(struct srcu_notifier_head *nh, 423 struct notifier_block *n) 424 { 425 int ret; 426 427 /* 428 * This code gets used during boot-up, when task switching is 429 * not yet working and interrupts must remain disabled. At 430 * such times we must not call mutex_lock(). 431 */ 432 if (unlikely(system_state == SYSTEM_BOOTING)) 433 return notifier_chain_register(&nh->head, n); 434 435 mutex_lock(&nh->mutex); 436 ret = notifier_chain_register(&nh->head, n); 437 mutex_unlock(&nh->mutex); 438 return ret; 439 } 440 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); 441 442 /** 443 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain 444 * @nh: Pointer to head of the SRCU notifier chain 445 * @n: Entry to remove from notifier chain 446 * 447 * Removes a notifier from an SRCU notifier chain. 448 * Must be called from process context. 449 * 450 * Returns zero on success or %-ENOENT on failure. 451 */ 452 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, 453 struct notifier_block *n) 454 { 455 int ret; 456 457 /* 458 * This code gets used during boot-up, when task switching is 459 * not yet working and interrupts must remain disabled. At 460 * such times we must not call mutex_lock(). 461 */ 462 if (unlikely(system_state == SYSTEM_BOOTING)) 463 return notifier_chain_unregister(&nh->head, n); 464 465 mutex_lock(&nh->mutex); 466 ret = notifier_chain_unregister(&nh->head, n); 467 mutex_unlock(&nh->mutex); 468 synchronize_srcu(&nh->srcu); 469 return ret; 470 } 471 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); 472 473 /** 474 * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain 475 * @nh: Pointer to head of the SRCU notifier chain 476 * @val: Value passed unmodified to notifier function 477 * @v: Pointer passed unmodified to notifier function 478 * @nr_to_call: See comment for notifier_call_chain. 479 * @nr_calls: See comment for notifier_call_chain 480 * 481 * Calls each function in a notifier chain in turn. The functions 482 * run in a process context, so they are allowed to block. 483 * 484 * If the return value of the notifier can be and'ed 485 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() 486 * will return immediately, with the return value of 487 * the notifier function which halted execution. 488 * Otherwise the return value is the return value 489 * of the last notifier function called. 490 */ 491 int __srcu_notifier_call_chain(struct srcu_notifier_head *nh, 492 unsigned long val, void *v, 493 int nr_to_call, int *nr_calls) 494 { 495 int ret; 496 int idx; 497 498 idx = srcu_read_lock(&nh->srcu); 499 ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); 500 srcu_read_unlock(&nh->srcu, idx); 501 return ret; 502 } 503 EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain); 504 505 int srcu_notifier_call_chain(struct srcu_notifier_head *nh, 506 unsigned long val, void *v) 507 { 508 return __srcu_notifier_call_chain(nh, val, v, -1, NULL); 509 } 510 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); 511 512 /** 513 * srcu_init_notifier_head - Initialize an SRCU notifier head 514 * @nh: Pointer to head of the srcu notifier chain 515 * 516 * Unlike other sorts of notifier heads, SRCU notifier heads require 517 * dynamic initialization. Be sure to call this routine before 518 * calling any of the other SRCU notifier routines for this head. 519 * 520 * If an SRCU notifier head is deallocated, it must first be cleaned 521 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's 522 * per-cpu data (used by the SRCU mechanism) will leak. 523 */ 524 void srcu_init_notifier_head(struct srcu_notifier_head *nh) 525 { 526 mutex_init(&nh->mutex); 527 if (init_srcu_struct(&nh->srcu) < 0) 528 BUG(); 529 nh->head = NULL; 530 } 531 EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 532 533 #endif /* CONFIG_SRCU */ 534 535 static ATOMIC_NOTIFIER_HEAD(die_chain); 536 537 int notrace notify_die(enum die_val val, const char *str, 538 struct pt_regs *regs, long err, int trap, int sig) 539 { 540 struct die_args args = { 541 .regs = regs, 542 .str = str, 543 .err = err, 544 .trapnr = trap, 545 .signr = sig, 546 547 }; 548 RCU_LOCKDEP_WARN(!rcu_is_watching(), 549 "notify_die called but RCU thinks we're quiescent"); 550 return atomic_notifier_call_chain(&die_chain, val, &args); 551 } 552 NOKPROBE_SYMBOL(notify_die); 553 554 int register_die_notifier(struct notifier_block *nb) 555 { 556 vmalloc_sync_all(); 557 return atomic_notifier_chain_register(&die_chain, nb); 558 } 559 EXPORT_SYMBOL_GPL(register_die_notifier); 560 561 int unregister_die_notifier(struct notifier_block *nb) 562 { 563 return atomic_notifier_chain_unregister(&die_chain, nb); 564 } 565 EXPORT_SYMBOL_GPL(unregister_die_notifier); 566