1 /* 2 * linux/net/sunrpc/xprt.c 3 * 4 * This is a generic RPC call interface supporting congestion avoidance, 5 * and asynchronous calls. 6 * 7 * The interface works like this: 8 * 9 * - When a process places a call, it allocates a request slot if 10 * one is available. Otherwise, it sleeps on the backlog queue 11 * (xprt_reserve). 12 * - Next, the caller puts together the RPC message, stuffs it into 13 * the request struct, and calls xprt_transmit(). 14 * - xprt_transmit sends the message and installs the caller on the 15 * transport's wait list. At the same time, if a reply is expected, 16 * it installs a timer that is run after the packet's timeout has 17 * expired. 18 * - When a packet arrives, the data_ready handler walks the list of 19 * pending requests for that transport. If a matching XID is found, the 20 * caller is woken up, and the timer removed. 21 * - When no reply arrives within the timeout interval, the timer is 22 * fired by the kernel and runs xprt_timer(). It either adjusts the 23 * timeout values (minor timeout) or wakes up the caller with a status 24 * of -ETIMEDOUT. 25 * - When the caller receives a notification from RPC that a reply arrived, 26 * it should release the RPC slot, and process the reply. 27 * If the call timed out, it may choose to retry the operation by 28 * adjusting the initial timeout value, and simply calling rpc_call 29 * again. 30 * 31 * Support for async RPC is done through a set of RPC-specific scheduling 32 * primitives that `transparently' work for processes as well as async 33 * tasks that rely on callbacks. 34 * 35 * Copyright (C) 1995-1997, Olaf Kirch <okir@monad.swb.de> 36 * 37 * Transport switch API copyright (C) 2005, Chuck Lever <cel@netapp.com> 38 */ 39 40 #include <linux/module.h> 41 42 #include <linux/types.h> 43 #include <linux/interrupt.h> 44 #include <linux/workqueue.h> 45 #include <linux/net.h> 46 #include <linux/ktime.h> 47 48 #include <linux/sunrpc/clnt.h> 49 #include <linux/sunrpc/metrics.h> 50 #include <linux/sunrpc/bc_xprt.h> 51 #include <linux/rcupdate.h> 52 53 #include <trace/events/sunrpc.h> 54 55 #include "sunrpc.h" 56 57 /* 58 * Local variables 59 */ 60 61 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 62 # define RPCDBG_FACILITY RPCDBG_XPRT 63 #endif 64 65 /* 66 * Local functions 67 */ 68 static void xprt_init(struct rpc_xprt *xprt, struct net *net); 69 static __be32 xprt_alloc_xid(struct rpc_xprt *xprt); 70 static void xprt_connect_status(struct rpc_task *task); 71 static int __xprt_get_cong(struct rpc_xprt *, struct rpc_task *); 72 static void __xprt_put_cong(struct rpc_xprt *, struct rpc_rqst *); 73 static void xprt_destroy(struct rpc_xprt *xprt); 74 75 static DEFINE_SPINLOCK(xprt_list_lock); 76 static LIST_HEAD(xprt_list); 77 78 /** 79 * xprt_register_transport - register a transport implementation 80 * @transport: transport to register 81 * 82 * If a transport implementation is loaded as a kernel module, it can 83 * call this interface to make itself known to the RPC client. 84 * 85 * Returns: 86 * 0: transport successfully registered 87 * -EEXIST: transport already registered 88 * -EINVAL: transport module being unloaded 89 */ 90 int xprt_register_transport(struct xprt_class *transport) 91 { 92 struct xprt_class *t; 93 int result; 94 95 result = -EEXIST; 96 spin_lock(&xprt_list_lock); 97 list_for_each_entry(t, &xprt_list, list) { 98 /* don't register the same transport class twice */ 99 if (t->ident == transport->ident) 100 goto out; 101 } 102 103 list_add_tail(&transport->list, &xprt_list); 104 printk(KERN_INFO "RPC: Registered %s transport module.\n", 105 transport->name); 106 result = 0; 107 108 out: 109 spin_unlock(&xprt_list_lock); 110 return result; 111 } 112 EXPORT_SYMBOL_GPL(xprt_register_transport); 113 114 /** 115 * xprt_unregister_transport - unregister a transport implementation 116 * @transport: transport to unregister 117 * 118 * Returns: 119 * 0: transport successfully unregistered 120 * -ENOENT: transport never registered 121 */ 122 int xprt_unregister_transport(struct xprt_class *transport) 123 { 124 struct xprt_class *t; 125 int result; 126 127 result = 0; 128 spin_lock(&xprt_list_lock); 129 list_for_each_entry(t, &xprt_list, list) { 130 if (t == transport) { 131 printk(KERN_INFO 132 "RPC: Unregistered %s transport module.\n", 133 transport->name); 134 list_del_init(&transport->list); 135 goto out; 136 } 137 } 138 result = -ENOENT; 139 140 out: 141 spin_unlock(&xprt_list_lock); 142 return result; 143 } 144 EXPORT_SYMBOL_GPL(xprt_unregister_transport); 145 146 /** 147 * xprt_load_transport - load a transport implementation 148 * @transport_name: transport to load 149 * 150 * Returns: 151 * 0: transport successfully loaded 152 * -ENOENT: transport module not available 153 */ 154 int xprt_load_transport(const char *transport_name) 155 { 156 struct xprt_class *t; 157 int result; 158 159 result = 0; 160 spin_lock(&xprt_list_lock); 161 list_for_each_entry(t, &xprt_list, list) { 162 if (strcmp(t->name, transport_name) == 0) { 163 spin_unlock(&xprt_list_lock); 164 goto out; 165 } 166 } 167 spin_unlock(&xprt_list_lock); 168 result = request_module("xprt%s", transport_name); 169 out: 170 return result; 171 } 172 EXPORT_SYMBOL_GPL(xprt_load_transport); 173 174 /** 175 * xprt_reserve_xprt - serialize write access to transports 176 * @task: task that is requesting access to the transport 177 * @xprt: pointer to the target transport 178 * 179 * This prevents mixing the payload of separate requests, and prevents 180 * transport connects from colliding with writes. No congestion control 181 * is provided. 182 */ 183 int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task) 184 { 185 struct rpc_rqst *req = task->tk_rqstp; 186 int priority; 187 188 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) { 189 if (task == xprt->snd_task) 190 return 1; 191 goto out_sleep; 192 } 193 xprt->snd_task = task; 194 if (req != NULL) 195 req->rq_ntrans++; 196 197 return 1; 198 199 out_sleep: 200 dprintk("RPC: %5u failed to lock transport %p\n", 201 task->tk_pid, xprt); 202 task->tk_timeout = 0; 203 task->tk_status = -EAGAIN; 204 if (req == NULL) 205 priority = RPC_PRIORITY_LOW; 206 else if (!req->rq_ntrans) 207 priority = RPC_PRIORITY_NORMAL; 208 else 209 priority = RPC_PRIORITY_HIGH; 210 rpc_sleep_on_priority(&xprt->sending, task, NULL, priority); 211 return 0; 212 } 213 EXPORT_SYMBOL_GPL(xprt_reserve_xprt); 214 215 static void xprt_clear_locked(struct rpc_xprt *xprt) 216 { 217 xprt->snd_task = NULL; 218 if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) { 219 smp_mb__before_atomic(); 220 clear_bit(XPRT_LOCKED, &xprt->state); 221 smp_mb__after_atomic(); 222 } else 223 queue_work(xprtiod_workqueue, &xprt->task_cleanup); 224 } 225 226 /* 227 * xprt_reserve_xprt_cong - serialize write access to transports 228 * @task: task that is requesting access to the transport 229 * 230 * Same as xprt_reserve_xprt, but Van Jacobson congestion control is 231 * integrated into the decision of whether a request is allowed to be 232 * woken up and given access to the transport. 233 */ 234 int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task) 235 { 236 struct rpc_rqst *req = task->tk_rqstp; 237 int priority; 238 239 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) { 240 if (task == xprt->snd_task) 241 return 1; 242 goto out_sleep; 243 } 244 if (req == NULL) { 245 xprt->snd_task = task; 246 return 1; 247 } 248 if (__xprt_get_cong(xprt, task)) { 249 xprt->snd_task = task; 250 req->rq_ntrans++; 251 return 1; 252 } 253 xprt_clear_locked(xprt); 254 out_sleep: 255 if (req) 256 __xprt_put_cong(xprt, req); 257 dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt); 258 task->tk_timeout = 0; 259 task->tk_status = -EAGAIN; 260 if (req == NULL) 261 priority = RPC_PRIORITY_LOW; 262 else if (!req->rq_ntrans) 263 priority = RPC_PRIORITY_NORMAL; 264 else 265 priority = RPC_PRIORITY_HIGH; 266 rpc_sleep_on_priority(&xprt->sending, task, NULL, priority); 267 return 0; 268 } 269 EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong); 270 271 static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task) 272 { 273 int retval; 274 275 spin_lock_bh(&xprt->transport_lock); 276 retval = xprt->ops->reserve_xprt(xprt, task); 277 spin_unlock_bh(&xprt->transport_lock); 278 return retval; 279 } 280 281 static bool __xprt_lock_write_func(struct rpc_task *task, void *data) 282 { 283 struct rpc_xprt *xprt = data; 284 struct rpc_rqst *req; 285 286 req = task->tk_rqstp; 287 xprt->snd_task = task; 288 if (req) 289 req->rq_ntrans++; 290 return true; 291 } 292 293 static void __xprt_lock_write_next(struct rpc_xprt *xprt) 294 { 295 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) 296 return; 297 298 if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending, 299 __xprt_lock_write_func, xprt)) 300 return; 301 xprt_clear_locked(xprt); 302 } 303 304 static bool __xprt_lock_write_cong_func(struct rpc_task *task, void *data) 305 { 306 struct rpc_xprt *xprt = data; 307 struct rpc_rqst *req; 308 309 req = task->tk_rqstp; 310 if (req == NULL) { 311 xprt->snd_task = task; 312 return true; 313 } 314 if (__xprt_get_cong(xprt, task)) { 315 xprt->snd_task = task; 316 req->rq_ntrans++; 317 return true; 318 } 319 return false; 320 } 321 322 static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt) 323 { 324 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) 325 return; 326 if (RPCXPRT_CONGESTED(xprt)) 327 goto out_unlock; 328 if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending, 329 __xprt_lock_write_cong_func, xprt)) 330 return; 331 out_unlock: 332 xprt_clear_locked(xprt); 333 } 334 335 static void xprt_task_clear_bytes_sent(struct rpc_task *task) 336 { 337 if (task != NULL) { 338 struct rpc_rqst *req = task->tk_rqstp; 339 if (req != NULL) 340 req->rq_bytes_sent = 0; 341 } 342 } 343 344 /** 345 * xprt_release_xprt - allow other requests to use a transport 346 * @xprt: transport with other tasks potentially waiting 347 * @task: task that is releasing access to the transport 348 * 349 * Note that "task" can be NULL. No congestion control is provided. 350 */ 351 void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task) 352 { 353 if (xprt->snd_task == task) { 354 xprt_task_clear_bytes_sent(task); 355 xprt_clear_locked(xprt); 356 __xprt_lock_write_next(xprt); 357 } 358 } 359 EXPORT_SYMBOL_GPL(xprt_release_xprt); 360 361 /** 362 * xprt_release_xprt_cong - allow other requests to use a transport 363 * @xprt: transport with other tasks potentially waiting 364 * @task: task that is releasing access to the transport 365 * 366 * Note that "task" can be NULL. Another task is awoken to use the 367 * transport if the transport's congestion window allows it. 368 */ 369 void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task) 370 { 371 if (xprt->snd_task == task) { 372 xprt_task_clear_bytes_sent(task); 373 xprt_clear_locked(xprt); 374 __xprt_lock_write_next_cong(xprt); 375 } 376 } 377 EXPORT_SYMBOL_GPL(xprt_release_xprt_cong); 378 379 static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task) 380 { 381 spin_lock_bh(&xprt->transport_lock); 382 xprt->ops->release_xprt(xprt, task); 383 spin_unlock_bh(&xprt->transport_lock); 384 } 385 386 /* 387 * Van Jacobson congestion avoidance. Check if the congestion window 388 * overflowed. Put the task to sleep if this is the case. 389 */ 390 static int 391 __xprt_get_cong(struct rpc_xprt *xprt, struct rpc_task *task) 392 { 393 struct rpc_rqst *req = task->tk_rqstp; 394 395 if (req->rq_cong) 396 return 1; 397 dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n", 398 task->tk_pid, xprt->cong, xprt->cwnd); 399 if (RPCXPRT_CONGESTED(xprt)) 400 return 0; 401 req->rq_cong = 1; 402 xprt->cong += RPC_CWNDSCALE; 403 return 1; 404 } 405 406 /* 407 * Adjust the congestion window, and wake up the next task 408 * that has been sleeping due to congestion 409 */ 410 static void 411 __xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req) 412 { 413 if (!req->rq_cong) 414 return; 415 req->rq_cong = 0; 416 xprt->cong -= RPC_CWNDSCALE; 417 __xprt_lock_write_next_cong(xprt); 418 } 419 420 /** 421 * xprt_release_rqst_cong - housekeeping when request is complete 422 * @task: RPC request that recently completed 423 * 424 * Useful for transports that require congestion control. 425 */ 426 void xprt_release_rqst_cong(struct rpc_task *task) 427 { 428 struct rpc_rqst *req = task->tk_rqstp; 429 430 __xprt_put_cong(req->rq_xprt, req); 431 } 432 EXPORT_SYMBOL_GPL(xprt_release_rqst_cong); 433 434 /** 435 * xprt_adjust_cwnd - adjust transport congestion window 436 * @xprt: pointer to xprt 437 * @task: recently completed RPC request used to adjust window 438 * @result: result code of completed RPC request 439 * 440 * The transport code maintains an estimate on the maximum number of out- 441 * standing RPC requests, using a smoothed version of the congestion 442 * avoidance implemented in 44BSD. This is basically the Van Jacobson 443 * congestion algorithm: If a retransmit occurs, the congestion window is 444 * halved; otherwise, it is incremented by 1/cwnd when 445 * 446 * - a reply is received and 447 * - a full number of requests are outstanding and 448 * - the congestion window hasn't been updated recently. 449 */ 450 void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result) 451 { 452 struct rpc_rqst *req = task->tk_rqstp; 453 unsigned long cwnd = xprt->cwnd; 454 455 if (result >= 0 && cwnd <= xprt->cong) { 456 /* The (cwnd >> 1) term makes sure 457 * the result gets rounded properly. */ 458 cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd; 459 if (cwnd > RPC_MAXCWND(xprt)) 460 cwnd = RPC_MAXCWND(xprt); 461 __xprt_lock_write_next_cong(xprt); 462 } else if (result == -ETIMEDOUT) { 463 cwnd >>= 1; 464 if (cwnd < RPC_CWNDSCALE) 465 cwnd = RPC_CWNDSCALE; 466 } 467 dprintk("RPC: cong %ld, cwnd was %ld, now %ld\n", 468 xprt->cong, xprt->cwnd, cwnd); 469 xprt->cwnd = cwnd; 470 __xprt_put_cong(xprt, req); 471 } 472 EXPORT_SYMBOL_GPL(xprt_adjust_cwnd); 473 474 /** 475 * xprt_wake_pending_tasks - wake all tasks on a transport's pending queue 476 * @xprt: transport with waiting tasks 477 * @status: result code to plant in each task before waking it 478 * 479 */ 480 void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status) 481 { 482 if (status < 0) 483 rpc_wake_up_status(&xprt->pending, status); 484 else 485 rpc_wake_up(&xprt->pending); 486 } 487 EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks); 488 489 /** 490 * xprt_wait_for_buffer_space - wait for transport output buffer to clear 491 * @task: task to be put to sleep 492 * @action: function pointer to be executed after wait 493 * 494 * Note that we only set the timer for the case of RPC_IS_SOFT(), since 495 * we don't in general want to force a socket disconnection due to 496 * an incomplete RPC call transmission. 497 */ 498 void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action) 499 { 500 struct rpc_rqst *req = task->tk_rqstp; 501 struct rpc_xprt *xprt = req->rq_xprt; 502 503 task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0; 504 rpc_sleep_on(&xprt->pending, task, action); 505 } 506 EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space); 507 508 /** 509 * xprt_write_space - wake the task waiting for transport output buffer space 510 * @xprt: transport with waiting tasks 511 * 512 * Can be called in a soft IRQ context, so xprt_write_space never sleeps. 513 */ 514 void xprt_write_space(struct rpc_xprt *xprt) 515 { 516 spin_lock_bh(&xprt->transport_lock); 517 if (xprt->snd_task) { 518 dprintk("RPC: write space: waking waiting task on " 519 "xprt %p\n", xprt); 520 rpc_wake_up_queued_task_on_wq(xprtiod_workqueue, 521 &xprt->pending, xprt->snd_task); 522 } 523 spin_unlock_bh(&xprt->transport_lock); 524 } 525 EXPORT_SYMBOL_GPL(xprt_write_space); 526 527 /** 528 * xprt_set_retrans_timeout_def - set a request's retransmit timeout 529 * @task: task whose timeout is to be set 530 * 531 * Set a request's retransmit timeout based on the transport's 532 * default timeout parameters. Used by transports that don't adjust 533 * the retransmit timeout based on round-trip time estimation. 534 */ 535 void xprt_set_retrans_timeout_def(struct rpc_task *task) 536 { 537 task->tk_timeout = task->tk_rqstp->rq_timeout; 538 } 539 EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_def); 540 541 /** 542 * xprt_set_retrans_timeout_rtt - set a request's retransmit timeout 543 * @task: task whose timeout is to be set 544 * 545 * Set a request's retransmit timeout using the RTT estimator. 546 */ 547 void xprt_set_retrans_timeout_rtt(struct rpc_task *task) 548 { 549 int timer = task->tk_msg.rpc_proc->p_timer; 550 struct rpc_clnt *clnt = task->tk_client; 551 struct rpc_rtt *rtt = clnt->cl_rtt; 552 struct rpc_rqst *req = task->tk_rqstp; 553 unsigned long max_timeout = clnt->cl_timeout->to_maxval; 554 555 task->tk_timeout = rpc_calc_rto(rtt, timer); 556 task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries; 557 if (task->tk_timeout > max_timeout || task->tk_timeout == 0) 558 task->tk_timeout = max_timeout; 559 } 560 EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_rtt); 561 562 static void xprt_reset_majortimeo(struct rpc_rqst *req) 563 { 564 const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout; 565 566 req->rq_majortimeo = req->rq_timeout; 567 if (to->to_exponential) 568 req->rq_majortimeo <<= to->to_retries; 569 else 570 req->rq_majortimeo += to->to_increment * to->to_retries; 571 if (req->rq_majortimeo > to->to_maxval || req->rq_majortimeo == 0) 572 req->rq_majortimeo = to->to_maxval; 573 req->rq_majortimeo += jiffies; 574 } 575 576 /** 577 * xprt_adjust_timeout - adjust timeout values for next retransmit 578 * @req: RPC request containing parameters to use for the adjustment 579 * 580 */ 581 int xprt_adjust_timeout(struct rpc_rqst *req) 582 { 583 struct rpc_xprt *xprt = req->rq_xprt; 584 const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout; 585 int status = 0; 586 587 if (time_before(jiffies, req->rq_majortimeo)) { 588 if (to->to_exponential) 589 req->rq_timeout <<= 1; 590 else 591 req->rq_timeout += to->to_increment; 592 if (to->to_maxval && req->rq_timeout >= to->to_maxval) 593 req->rq_timeout = to->to_maxval; 594 req->rq_retries++; 595 } else { 596 req->rq_timeout = to->to_initval; 597 req->rq_retries = 0; 598 xprt_reset_majortimeo(req); 599 /* Reset the RTT counters == "slow start" */ 600 spin_lock_bh(&xprt->transport_lock); 601 rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval); 602 spin_unlock_bh(&xprt->transport_lock); 603 status = -ETIMEDOUT; 604 } 605 606 if (req->rq_timeout == 0) { 607 printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0!\n"); 608 req->rq_timeout = 5 * HZ; 609 } 610 return status; 611 } 612 613 static void xprt_autoclose(struct work_struct *work) 614 { 615 struct rpc_xprt *xprt = 616 container_of(work, struct rpc_xprt, task_cleanup); 617 618 clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 619 xprt->ops->close(xprt); 620 xprt_release_write(xprt, NULL); 621 wake_up_bit(&xprt->state, XPRT_LOCKED); 622 } 623 624 /** 625 * xprt_disconnect_done - mark a transport as disconnected 626 * @xprt: transport to flag for disconnect 627 * 628 */ 629 void xprt_disconnect_done(struct rpc_xprt *xprt) 630 { 631 dprintk("RPC: disconnected transport %p\n", xprt); 632 spin_lock_bh(&xprt->transport_lock); 633 xprt_clear_connected(xprt); 634 xprt_wake_pending_tasks(xprt, -EAGAIN); 635 spin_unlock_bh(&xprt->transport_lock); 636 } 637 EXPORT_SYMBOL_GPL(xprt_disconnect_done); 638 639 /** 640 * xprt_force_disconnect - force a transport to disconnect 641 * @xprt: transport to disconnect 642 * 643 */ 644 void xprt_force_disconnect(struct rpc_xprt *xprt) 645 { 646 /* Don't race with the test_bit() in xprt_clear_locked() */ 647 spin_lock_bh(&xprt->transport_lock); 648 set_bit(XPRT_CLOSE_WAIT, &xprt->state); 649 /* Try to schedule an autoclose RPC call */ 650 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0) 651 queue_work(xprtiod_workqueue, &xprt->task_cleanup); 652 xprt_wake_pending_tasks(xprt, -EAGAIN); 653 spin_unlock_bh(&xprt->transport_lock); 654 } 655 EXPORT_SYMBOL_GPL(xprt_force_disconnect); 656 657 /** 658 * xprt_conditional_disconnect - force a transport to disconnect 659 * @xprt: transport to disconnect 660 * @cookie: 'connection cookie' 661 * 662 * This attempts to break the connection if and only if 'cookie' matches 663 * the current transport 'connection cookie'. It ensures that we don't 664 * try to break the connection more than once when we need to retransmit 665 * a batch of RPC requests. 666 * 667 */ 668 void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie) 669 { 670 /* Don't race with the test_bit() in xprt_clear_locked() */ 671 spin_lock_bh(&xprt->transport_lock); 672 if (cookie != xprt->connect_cookie) 673 goto out; 674 if (test_bit(XPRT_CLOSING, &xprt->state)) 675 goto out; 676 set_bit(XPRT_CLOSE_WAIT, &xprt->state); 677 /* Try to schedule an autoclose RPC call */ 678 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0) 679 queue_work(xprtiod_workqueue, &xprt->task_cleanup); 680 xprt_wake_pending_tasks(xprt, -EAGAIN); 681 out: 682 spin_unlock_bh(&xprt->transport_lock); 683 } 684 685 static bool 686 xprt_has_timer(const struct rpc_xprt *xprt) 687 { 688 return xprt->idle_timeout != 0; 689 } 690 691 static void 692 xprt_schedule_autodisconnect(struct rpc_xprt *xprt) 693 __must_hold(&xprt->transport_lock) 694 { 695 if (list_empty(&xprt->recv) && xprt_has_timer(xprt)) 696 mod_timer(&xprt->timer, xprt->last_used + xprt->idle_timeout); 697 } 698 699 static void 700 xprt_init_autodisconnect(struct timer_list *t) 701 { 702 struct rpc_xprt *xprt = from_timer(xprt, t, timer); 703 704 spin_lock(&xprt->transport_lock); 705 if (!list_empty(&xprt->recv)) 706 goto out_abort; 707 /* Reset xprt->last_used to avoid connect/autodisconnect cycling */ 708 xprt->last_used = jiffies; 709 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) 710 goto out_abort; 711 spin_unlock(&xprt->transport_lock); 712 queue_work(xprtiod_workqueue, &xprt->task_cleanup); 713 return; 714 out_abort: 715 spin_unlock(&xprt->transport_lock); 716 } 717 718 bool xprt_lock_connect(struct rpc_xprt *xprt, 719 struct rpc_task *task, 720 void *cookie) 721 { 722 bool ret = false; 723 724 spin_lock_bh(&xprt->transport_lock); 725 if (!test_bit(XPRT_LOCKED, &xprt->state)) 726 goto out; 727 if (xprt->snd_task != task) 728 goto out; 729 xprt_task_clear_bytes_sent(task); 730 xprt->snd_task = cookie; 731 ret = true; 732 out: 733 spin_unlock_bh(&xprt->transport_lock); 734 return ret; 735 } 736 737 void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie) 738 { 739 spin_lock_bh(&xprt->transport_lock); 740 if (xprt->snd_task != cookie) 741 goto out; 742 if (!test_bit(XPRT_LOCKED, &xprt->state)) 743 goto out; 744 xprt->snd_task =NULL; 745 xprt->ops->release_xprt(xprt, NULL); 746 xprt_schedule_autodisconnect(xprt); 747 out: 748 spin_unlock_bh(&xprt->transport_lock); 749 wake_up_bit(&xprt->state, XPRT_LOCKED); 750 } 751 752 /** 753 * xprt_connect - schedule a transport connect operation 754 * @task: RPC task that is requesting the connect 755 * 756 */ 757 void xprt_connect(struct rpc_task *task) 758 { 759 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt; 760 761 dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid, 762 xprt, (xprt_connected(xprt) ? "is" : "is not")); 763 764 if (!xprt_bound(xprt)) { 765 task->tk_status = -EAGAIN; 766 return; 767 } 768 if (!xprt_lock_write(xprt, task)) 769 return; 770 771 if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) 772 xprt->ops->close(xprt); 773 774 if (!xprt_connected(xprt)) { 775 task->tk_rqstp->rq_bytes_sent = 0; 776 task->tk_timeout = task->tk_rqstp->rq_timeout; 777 task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie; 778 rpc_sleep_on(&xprt->pending, task, xprt_connect_status); 779 780 if (test_bit(XPRT_CLOSING, &xprt->state)) 781 return; 782 if (xprt_test_and_set_connecting(xprt)) 783 return; 784 xprt->stat.connect_start = jiffies; 785 xprt->ops->connect(xprt, task); 786 } 787 xprt_release_write(xprt, task); 788 } 789 790 static void xprt_connect_status(struct rpc_task *task) 791 { 792 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt; 793 794 if (task->tk_status == 0) { 795 xprt->stat.connect_count++; 796 xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start; 797 dprintk("RPC: %5u xprt_connect_status: connection established\n", 798 task->tk_pid); 799 return; 800 } 801 802 switch (task->tk_status) { 803 case -ECONNREFUSED: 804 case -ECONNRESET: 805 case -ECONNABORTED: 806 case -ENETUNREACH: 807 case -EHOSTUNREACH: 808 case -EPIPE: 809 case -EAGAIN: 810 dprintk("RPC: %5u xprt_connect_status: retrying\n", task->tk_pid); 811 break; 812 case -ETIMEDOUT: 813 dprintk("RPC: %5u xprt_connect_status: connect attempt timed " 814 "out\n", task->tk_pid); 815 break; 816 default: 817 dprintk("RPC: %5u xprt_connect_status: error %d connecting to " 818 "server %s\n", task->tk_pid, -task->tk_status, 819 xprt->servername); 820 task->tk_status = -EIO; 821 } 822 } 823 824 /** 825 * xprt_lookup_rqst - find an RPC request corresponding to an XID 826 * @xprt: transport on which the original request was transmitted 827 * @xid: RPC XID of incoming reply 828 * 829 * Caller holds xprt->queue_lock. 830 */ 831 struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid) 832 { 833 struct rpc_rqst *entry; 834 835 list_for_each_entry(entry, &xprt->recv, rq_list) 836 if (entry->rq_xid == xid) { 837 trace_xprt_lookup_rqst(xprt, xid, 0); 838 entry->rq_rtt = ktime_sub(ktime_get(), entry->rq_xtime); 839 return entry; 840 } 841 842 dprintk("RPC: xprt_lookup_rqst did not find xid %08x\n", 843 ntohl(xid)); 844 trace_xprt_lookup_rqst(xprt, xid, -ENOENT); 845 xprt->stat.bad_xids++; 846 return NULL; 847 } 848 EXPORT_SYMBOL_GPL(xprt_lookup_rqst); 849 850 static bool 851 xprt_is_pinned_rqst(struct rpc_rqst *req) 852 { 853 return atomic_read(&req->rq_pin) != 0; 854 } 855 856 /** 857 * xprt_pin_rqst - Pin a request on the transport receive list 858 * @req: Request to pin 859 * 860 * Caller must ensure this is atomic with the call to xprt_lookup_rqst() 861 * so should be holding the xprt receive lock. 862 */ 863 void xprt_pin_rqst(struct rpc_rqst *req) 864 { 865 atomic_inc(&req->rq_pin); 866 } 867 EXPORT_SYMBOL_GPL(xprt_pin_rqst); 868 869 /** 870 * xprt_unpin_rqst - Unpin a request on the transport receive list 871 * @req: Request to pin 872 * 873 * Caller should be holding the xprt receive lock. 874 */ 875 void xprt_unpin_rqst(struct rpc_rqst *req) 876 { 877 if (!test_bit(RPC_TASK_MSG_PIN_WAIT, &req->rq_task->tk_runstate)) { 878 atomic_dec(&req->rq_pin); 879 return; 880 } 881 if (atomic_dec_and_test(&req->rq_pin)) 882 wake_up_var(&req->rq_pin); 883 } 884 EXPORT_SYMBOL_GPL(xprt_unpin_rqst); 885 886 static void xprt_wait_on_pinned_rqst(struct rpc_rqst *req) 887 { 888 wait_var_event(&req->rq_pin, !xprt_is_pinned_rqst(req)); 889 } 890 891 static bool 892 xprt_request_data_received(struct rpc_task *task) 893 { 894 return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) && 895 READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) != 0; 896 } 897 898 static bool 899 xprt_request_need_enqueue_receive(struct rpc_task *task, struct rpc_rqst *req) 900 { 901 return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) && 902 READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) == 0; 903 } 904 905 /** 906 * xprt_request_enqueue_receive - Add an request to the receive queue 907 * @task: RPC task 908 * 909 */ 910 void 911 xprt_request_enqueue_receive(struct rpc_task *task) 912 { 913 struct rpc_rqst *req = task->tk_rqstp; 914 struct rpc_xprt *xprt = req->rq_xprt; 915 916 if (!xprt_request_need_enqueue_receive(task, req)) 917 return; 918 spin_lock(&xprt->queue_lock); 919 920 /* Update the softirq receive buffer */ 921 memcpy(&req->rq_private_buf, &req->rq_rcv_buf, 922 sizeof(req->rq_private_buf)); 923 924 /* Add request to the receive list */ 925 list_add_tail(&req->rq_list, &xprt->recv); 926 set_bit(RPC_TASK_NEED_RECV, &task->tk_runstate); 927 spin_unlock(&xprt->queue_lock); 928 929 xprt_reset_majortimeo(req); 930 /* Turn off autodisconnect */ 931 del_singleshot_timer_sync(&xprt->timer); 932 } 933 934 /** 935 * xprt_request_dequeue_receive_locked - Remove a request from the receive queue 936 * @task: RPC task 937 * 938 * Caller must hold xprt->queue_lock. 939 */ 940 static void 941 xprt_request_dequeue_receive_locked(struct rpc_task *task) 942 { 943 if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) 944 list_del(&task->tk_rqstp->rq_list); 945 } 946 947 /** 948 * xprt_update_rtt - Update RPC RTT statistics 949 * @task: RPC request that recently completed 950 * 951 * Caller holds xprt->queue_lock. 952 */ 953 void xprt_update_rtt(struct rpc_task *task) 954 { 955 struct rpc_rqst *req = task->tk_rqstp; 956 struct rpc_rtt *rtt = task->tk_client->cl_rtt; 957 unsigned int timer = task->tk_msg.rpc_proc->p_timer; 958 long m = usecs_to_jiffies(ktime_to_us(req->rq_rtt)); 959 960 if (timer) { 961 if (req->rq_ntrans == 1) 962 rpc_update_rtt(rtt, timer, m); 963 rpc_set_timeo(rtt, timer, req->rq_ntrans - 1); 964 } 965 } 966 EXPORT_SYMBOL_GPL(xprt_update_rtt); 967 968 /** 969 * xprt_complete_rqst - called when reply processing is complete 970 * @task: RPC request that recently completed 971 * @copied: actual number of bytes received from the transport 972 * 973 * Caller holds xprt->queue_lock. 974 */ 975 void xprt_complete_rqst(struct rpc_task *task, int copied) 976 { 977 struct rpc_rqst *req = task->tk_rqstp; 978 struct rpc_xprt *xprt = req->rq_xprt; 979 980 dprintk("RPC: %5u xid %08x complete (%d bytes received)\n", 981 task->tk_pid, ntohl(req->rq_xid), copied); 982 trace_xprt_complete_rqst(xprt, req->rq_xid, copied); 983 984 xprt->stat.recvs++; 985 986 req->rq_private_buf.len = copied; 987 /* Ensure all writes are done before we update */ 988 /* req->rq_reply_bytes_recvd */ 989 smp_wmb(); 990 req->rq_reply_bytes_recvd = copied; 991 xprt_request_dequeue_receive_locked(task); 992 rpc_wake_up_queued_task(&xprt->pending, task); 993 } 994 EXPORT_SYMBOL_GPL(xprt_complete_rqst); 995 996 static void xprt_timer(struct rpc_task *task) 997 { 998 struct rpc_rqst *req = task->tk_rqstp; 999 struct rpc_xprt *xprt = req->rq_xprt; 1000 1001 if (task->tk_status != -ETIMEDOUT) 1002 return; 1003 1004 trace_xprt_timer(xprt, req->rq_xid, task->tk_status); 1005 if (!req->rq_reply_bytes_recvd) { 1006 if (xprt->ops->timer) 1007 xprt->ops->timer(xprt, task); 1008 } else 1009 task->tk_status = 0; 1010 } 1011 1012 /** 1013 * xprt_prepare_transmit - reserve the transport before sending a request 1014 * @task: RPC task about to send a request 1015 * 1016 */ 1017 bool xprt_prepare_transmit(struct rpc_task *task) 1018 { 1019 struct rpc_rqst *req = task->tk_rqstp; 1020 struct rpc_xprt *xprt = req->rq_xprt; 1021 bool ret = false; 1022 1023 dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid); 1024 1025 spin_lock_bh(&xprt->transport_lock); 1026 if (!req->rq_bytes_sent) { 1027 if (req->rq_reply_bytes_recvd) { 1028 task->tk_status = req->rq_reply_bytes_recvd; 1029 goto out_unlock; 1030 } 1031 if ((task->tk_flags & RPC_TASK_NO_RETRANS_TIMEOUT) 1032 && xprt_connected(xprt) 1033 && req->rq_connect_cookie == xprt->connect_cookie) { 1034 xprt->ops->set_retrans_timeout(task); 1035 rpc_sleep_on(&xprt->pending, task, xprt_timer); 1036 goto out_unlock; 1037 } 1038 } 1039 if (!xprt->ops->reserve_xprt(xprt, task)) { 1040 task->tk_status = -EAGAIN; 1041 goto out_unlock; 1042 } 1043 ret = true; 1044 out_unlock: 1045 spin_unlock_bh(&xprt->transport_lock); 1046 return ret; 1047 } 1048 1049 void xprt_end_transmit(struct rpc_task *task) 1050 { 1051 xprt_release_write(task->tk_rqstp->rq_xprt, task); 1052 } 1053 1054 /** 1055 * xprt_transmit - send an RPC request on a transport 1056 * @task: controlling RPC task 1057 * 1058 * We have to copy the iovec because sendmsg fiddles with its contents. 1059 */ 1060 void xprt_transmit(struct rpc_task *task) 1061 { 1062 struct rpc_rqst *req = task->tk_rqstp; 1063 struct rpc_xprt *xprt = req->rq_xprt; 1064 unsigned int connect_cookie; 1065 int status; 1066 1067 dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen); 1068 1069 if (!req->rq_bytes_sent) { 1070 if (xprt_request_data_received(task)) 1071 return; 1072 /* Verify that our message lies in the RPCSEC_GSS window */ 1073 if (rpcauth_xmit_need_reencode(task)) { 1074 task->tk_status = -EBADMSG; 1075 return; 1076 } 1077 } 1078 1079 connect_cookie = xprt->connect_cookie; 1080 status = xprt->ops->send_request(task); 1081 trace_xprt_transmit(xprt, req->rq_xid, status); 1082 if (status != 0) { 1083 task->tk_status = status; 1084 return; 1085 } 1086 1087 xprt_inject_disconnect(xprt); 1088 1089 dprintk("RPC: %5u xmit complete\n", task->tk_pid); 1090 clear_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate); 1091 task->tk_flags |= RPC_TASK_SENT; 1092 spin_lock_bh(&xprt->transport_lock); 1093 1094 xprt->ops->set_retrans_timeout(task); 1095 1096 xprt->stat.sends++; 1097 xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs; 1098 xprt->stat.bklog_u += xprt->backlog.qlen; 1099 xprt->stat.sending_u += xprt->sending.qlen; 1100 xprt->stat.pending_u += xprt->pending.qlen; 1101 spin_unlock_bh(&xprt->transport_lock); 1102 1103 req->rq_connect_cookie = connect_cookie; 1104 if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) { 1105 /* 1106 * Sleep on the pending queue if we're expecting a reply. 1107 * The spinlock ensures atomicity between the test of 1108 * req->rq_reply_bytes_recvd, and the call to rpc_sleep_on(). 1109 */ 1110 spin_lock(&xprt->queue_lock); 1111 if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) { 1112 rpc_sleep_on(&xprt->pending, task, xprt_timer); 1113 /* Wake up immediately if the connection was dropped */ 1114 if (!xprt_connected(xprt)) 1115 rpc_wake_up_queued_task_set_status(&xprt->pending, 1116 task, -ENOTCONN); 1117 } 1118 spin_unlock(&xprt->queue_lock); 1119 } 1120 } 1121 1122 static void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task) 1123 { 1124 set_bit(XPRT_CONGESTED, &xprt->state); 1125 rpc_sleep_on(&xprt->backlog, task, NULL); 1126 } 1127 1128 static void xprt_wake_up_backlog(struct rpc_xprt *xprt) 1129 { 1130 if (rpc_wake_up_next(&xprt->backlog) == NULL) 1131 clear_bit(XPRT_CONGESTED, &xprt->state); 1132 } 1133 1134 static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task) 1135 { 1136 bool ret = false; 1137 1138 if (!test_bit(XPRT_CONGESTED, &xprt->state)) 1139 goto out; 1140 spin_lock(&xprt->reserve_lock); 1141 if (test_bit(XPRT_CONGESTED, &xprt->state)) { 1142 rpc_sleep_on(&xprt->backlog, task, NULL); 1143 ret = true; 1144 } 1145 spin_unlock(&xprt->reserve_lock); 1146 out: 1147 return ret; 1148 } 1149 1150 static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt) 1151 { 1152 struct rpc_rqst *req = ERR_PTR(-EAGAIN); 1153 1154 if (xprt->num_reqs >= xprt->max_reqs) 1155 goto out; 1156 ++xprt->num_reqs; 1157 spin_unlock(&xprt->reserve_lock); 1158 req = kzalloc(sizeof(struct rpc_rqst), GFP_NOFS); 1159 spin_lock(&xprt->reserve_lock); 1160 if (req != NULL) 1161 goto out; 1162 --xprt->num_reqs; 1163 req = ERR_PTR(-ENOMEM); 1164 out: 1165 return req; 1166 } 1167 1168 static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req) 1169 { 1170 if (xprt->num_reqs > xprt->min_reqs) { 1171 --xprt->num_reqs; 1172 kfree(req); 1173 return true; 1174 } 1175 return false; 1176 } 1177 1178 void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) 1179 { 1180 struct rpc_rqst *req; 1181 1182 spin_lock(&xprt->reserve_lock); 1183 if (!list_empty(&xprt->free)) { 1184 req = list_entry(xprt->free.next, struct rpc_rqst, rq_list); 1185 list_del(&req->rq_list); 1186 goto out_init_req; 1187 } 1188 req = xprt_dynamic_alloc_slot(xprt); 1189 if (!IS_ERR(req)) 1190 goto out_init_req; 1191 switch (PTR_ERR(req)) { 1192 case -ENOMEM: 1193 dprintk("RPC: dynamic allocation of request slot " 1194 "failed! Retrying\n"); 1195 task->tk_status = -ENOMEM; 1196 break; 1197 case -EAGAIN: 1198 xprt_add_backlog(xprt, task); 1199 dprintk("RPC: waiting for request slot\n"); 1200 /* fall through */ 1201 default: 1202 task->tk_status = -EAGAIN; 1203 } 1204 spin_unlock(&xprt->reserve_lock); 1205 return; 1206 out_init_req: 1207 xprt->stat.max_slots = max_t(unsigned int, xprt->stat.max_slots, 1208 xprt->num_reqs); 1209 spin_unlock(&xprt->reserve_lock); 1210 1211 task->tk_status = 0; 1212 task->tk_rqstp = req; 1213 } 1214 EXPORT_SYMBOL_GPL(xprt_alloc_slot); 1215 1216 void xprt_lock_and_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) 1217 { 1218 /* Note: grabbing the xprt_lock_write() ensures that we throttle 1219 * new slot allocation if the transport is congested (i.e. when 1220 * reconnecting a stream transport or when out of socket write 1221 * buffer space). 1222 */ 1223 if (xprt_lock_write(xprt, task)) { 1224 xprt_alloc_slot(xprt, task); 1225 xprt_release_write(xprt, task); 1226 } 1227 } 1228 EXPORT_SYMBOL_GPL(xprt_lock_and_alloc_slot); 1229 1230 void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req) 1231 { 1232 spin_lock(&xprt->reserve_lock); 1233 if (!xprt_dynamic_free_slot(xprt, req)) { 1234 memset(req, 0, sizeof(*req)); /* mark unused */ 1235 list_add(&req->rq_list, &xprt->free); 1236 } 1237 xprt_wake_up_backlog(xprt); 1238 spin_unlock(&xprt->reserve_lock); 1239 } 1240 EXPORT_SYMBOL_GPL(xprt_free_slot); 1241 1242 static void xprt_free_all_slots(struct rpc_xprt *xprt) 1243 { 1244 struct rpc_rqst *req; 1245 while (!list_empty(&xprt->free)) { 1246 req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list); 1247 list_del(&req->rq_list); 1248 kfree(req); 1249 } 1250 } 1251 1252 struct rpc_xprt *xprt_alloc(struct net *net, size_t size, 1253 unsigned int num_prealloc, 1254 unsigned int max_alloc) 1255 { 1256 struct rpc_xprt *xprt; 1257 struct rpc_rqst *req; 1258 int i; 1259 1260 xprt = kzalloc(size, GFP_KERNEL); 1261 if (xprt == NULL) 1262 goto out; 1263 1264 xprt_init(xprt, net); 1265 1266 for (i = 0; i < num_prealloc; i++) { 1267 req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL); 1268 if (!req) 1269 goto out_free; 1270 list_add(&req->rq_list, &xprt->free); 1271 } 1272 if (max_alloc > num_prealloc) 1273 xprt->max_reqs = max_alloc; 1274 else 1275 xprt->max_reqs = num_prealloc; 1276 xprt->min_reqs = num_prealloc; 1277 xprt->num_reqs = num_prealloc; 1278 1279 return xprt; 1280 1281 out_free: 1282 xprt_free(xprt); 1283 out: 1284 return NULL; 1285 } 1286 EXPORT_SYMBOL_GPL(xprt_alloc); 1287 1288 void xprt_free(struct rpc_xprt *xprt) 1289 { 1290 put_net(xprt->xprt_net); 1291 xprt_free_all_slots(xprt); 1292 kfree_rcu(xprt, rcu); 1293 } 1294 EXPORT_SYMBOL_GPL(xprt_free); 1295 1296 static __be32 1297 xprt_alloc_xid(struct rpc_xprt *xprt) 1298 { 1299 __be32 xid; 1300 1301 spin_lock(&xprt->reserve_lock); 1302 xid = (__force __be32)xprt->xid++; 1303 spin_unlock(&xprt->reserve_lock); 1304 return xid; 1305 } 1306 1307 static void 1308 xprt_init_xid(struct rpc_xprt *xprt) 1309 { 1310 xprt->xid = prandom_u32(); 1311 } 1312 1313 static void 1314 xprt_request_init(struct rpc_task *task) 1315 { 1316 struct rpc_xprt *xprt = task->tk_xprt; 1317 struct rpc_rqst *req = task->tk_rqstp; 1318 1319 req->rq_timeout = task->tk_client->cl_timeout->to_initval; 1320 req->rq_task = task; 1321 req->rq_xprt = xprt; 1322 req->rq_buffer = NULL; 1323 req->rq_xid = xprt_alloc_xid(xprt); 1324 req->rq_connect_cookie = xprt->connect_cookie - 1; 1325 req->rq_bytes_sent = 0; 1326 req->rq_snd_buf.len = 0; 1327 req->rq_snd_buf.buflen = 0; 1328 req->rq_rcv_buf.len = 0; 1329 req->rq_rcv_buf.buflen = 0; 1330 req->rq_release_snd_buf = NULL; 1331 xprt_reset_majortimeo(req); 1332 dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid, 1333 req, ntohl(req->rq_xid)); 1334 } 1335 1336 static void 1337 xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task) 1338 { 1339 xprt->ops->alloc_slot(xprt, task); 1340 if (task->tk_rqstp != NULL) 1341 xprt_request_init(task); 1342 } 1343 1344 /** 1345 * xprt_reserve - allocate an RPC request slot 1346 * @task: RPC task requesting a slot allocation 1347 * 1348 * If the transport is marked as being congested, or if no more 1349 * slots are available, place the task on the transport's 1350 * backlog queue. 1351 */ 1352 void xprt_reserve(struct rpc_task *task) 1353 { 1354 struct rpc_xprt *xprt = task->tk_xprt; 1355 1356 task->tk_status = 0; 1357 if (task->tk_rqstp != NULL) 1358 return; 1359 1360 task->tk_timeout = 0; 1361 task->tk_status = -EAGAIN; 1362 if (!xprt_throttle_congested(xprt, task)) 1363 xprt_do_reserve(xprt, task); 1364 } 1365 1366 /** 1367 * xprt_retry_reserve - allocate an RPC request slot 1368 * @task: RPC task requesting a slot allocation 1369 * 1370 * If no more slots are available, place the task on the transport's 1371 * backlog queue. 1372 * Note that the only difference with xprt_reserve is that we now 1373 * ignore the value of the XPRT_CONGESTED flag. 1374 */ 1375 void xprt_retry_reserve(struct rpc_task *task) 1376 { 1377 struct rpc_xprt *xprt = task->tk_xprt; 1378 1379 task->tk_status = 0; 1380 if (task->tk_rqstp != NULL) 1381 return; 1382 1383 task->tk_timeout = 0; 1384 task->tk_status = -EAGAIN; 1385 xprt_do_reserve(xprt, task); 1386 } 1387 1388 static void 1389 xprt_request_dequeue_all(struct rpc_task *task, struct rpc_rqst *req) 1390 { 1391 struct rpc_xprt *xprt = req->rq_xprt; 1392 1393 if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) || 1394 xprt_is_pinned_rqst(req)) { 1395 spin_lock(&xprt->queue_lock); 1396 xprt_request_dequeue_receive_locked(task); 1397 while (xprt_is_pinned_rqst(req)) { 1398 set_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate); 1399 spin_unlock(&xprt->queue_lock); 1400 xprt_wait_on_pinned_rqst(req); 1401 spin_lock(&xprt->queue_lock); 1402 clear_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate); 1403 } 1404 spin_unlock(&xprt->queue_lock); 1405 } 1406 } 1407 1408 /** 1409 * xprt_release - release an RPC request slot 1410 * @task: task which is finished with the slot 1411 * 1412 */ 1413 void xprt_release(struct rpc_task *task) 1414 { 1415 struct rpc_xprt *xprt; 1416 struct rpc_rqst *req = task->tk_rqstp; 1417 1418 if (req == NULL) { 1419 if (task->tk_client) { 1420 xprt = task->tk_xprt; 1421 if (xprt->snd_task == task) 1422 xprt_release_write(xprt, task); 1423 } 1424 return; 1425 } 1426 1427 xprt = req->rq_xprt; 1428 if (task->tk_ops->rpc_count_stats != NULL) 1429 task->tk_ops->rpc_count_stats(task, task->tk_calldata); 1430 else if (task->tk_client) 1431 rpc_count_iostats(task, task->tk_client->cl_metrics); 1432 xprt_request_dequeue_all(task, req); 1433 spin_lock_bh(&xprt->transport_lock); 1434 xprt->ops->release_xprt(xprt, task); 1435 if (xprt->ops->release_request) 1436 xprt->ops->release_request(task); 1437 xprt->last_used = jiffies; 1438 xprt_schedule_autodisconnect(xprt); 1439 spin_unlock_bh(&xprt->transport_lock); 1440 if (req->rq_buffer) 1441 xprt->ops->buf_free(task); 1442 xprt_inject_disconnect(xprt); 1443 if (req->rq_cred != NULL) 1444 put_rpccred(req->rq_cred); 1445 task->tk_rqstp = NULL; 1446 if (req->rq_release_snd_buf) 1447 req->rq_release_snd_buf(req); 1448 1449 dprintk("RPC: %5u release request %p\n", task->tk_pid, req); 1450 if (likely(!bc_prealloc(req))) 1451 xprt->ops->free_slot(xprt, req); 1452 else 1453 xprt_free_bc_request(req); 1454 } 1455 1456 static void xprt_init(struct rpc_xprt *xprt, struct net *net) 1457 { 1458 kref_init(&xprt->kref); 1459 1460 spin_lock_init(&xprt->transport_lock); 1461 spin_lock_init(&xprt->reserve_lock); 1462 spin_lock_init(&xprt->queue_lock); 1463 1464 INIT_LIST_HEAD(&xprt->free); 1465 INIT_LIST_HEAD(&xprt->recv); 1466 #if defined(CONFIG_SUNRPC_BACKCHANNEL) 1467 spin_lock_init(&xprt->bc_pa_lock); 1468 INIT_LIST_HEAD(&xprt->bc_pa_list); 1469 #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 1470 INIT_LIST_HEAD(&xprt->xprt_switch); 1471 1472 xprt->last_used = jiffies; 1473 xprt->cwnd = RPC_INITCWND; 1474 xprt->bind_index = 0; 1475 1476 rpc_init_wait_queue(&xprt->binding, "xprt_binding"); 1477 rpc_init_wait_queue(&xprt->pending, "xprt_pending"); 1478 rpc_init_priority_wait_queue(&xprt->sending, "xprt_sending"); 1479 rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog"); 1480 1481 xprt_init_xid(xprt); 1482 1483 xprt->xprt_net = get_net(net); 1484 } 1485 1486 /** 1487 * xprt_create_transport - create an RPC transport 1488 * @args: rpc transport creation arguments 1489 * 1490 */ 1491 struct rpc_xprt *xprt_create_transport(struct xprt_create *args) 1492 { 1493 struct rpc_xprt *xprt; 1494 struct xprt_class *t; 1495 1496 spin_lock(&xprt_list_lock); 1497 list_for_each_entry(t, &xprt_list, list) { 1498 if (t->ident == args->ident) { 1499 spin_unlock(&xprt_list_lock); 1500 goto found; 1501 } 1502 } 1503 spin_unlock(&xprt_list_lock); 1504 dprintk("RPC: transport (%d) not supported\n", args->ident); 1505 return ERR_PTR(-EIO); 1506 1507 found: 1508 xprt = t->setup(args); 1509 if (IS_ERR(xprt)) { 1510 dprintk("RPC: xprt_create_transport: failed, %ld\n", 1511 -PTR_ERR(xprt)); 1512 goto out; 1513 } 1514 if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT) 1515 xprt->idle_timeout = 0; 1516 INIT_WORK(&xprt->task_cleanup, xprt_autoclose); 1517 if (xprt_has_timer(xprt)) 1518 timer_setup(&xprt->timer, xprt_init_autodisconnect, 0); 1519 else 1520 timer_setup(&xprt->timer, NULL, 0); 1521 1522 if (strlen(args->servername) > RPC_MAXNETNAMELEN) { 1523 xprt_destroy(xprt); 1524 return ERR_PTR(-EINVAL); 1525 } 1526 xprt->servername = kstrdup(args->servername, GFP_KERNEL); 1527 if (xprt->servername == NULL) { 1528 xprt_destroy(xprt); 1529 return ERR_PTR(-ENOMEM); 1530 } 1531 1532 rpc_xprt_debugfs_register(xprt); 1533 1534 dprintk("RPC: created transport %p with %u slots\n", xprt, 1535 xprt->max_reqs); 1536 out: 1537 return xprt; 1538 } 1539 1540 static void xprt_destroy_cb(struct work_struct *work) 1541 { 1542 struct rpc_xprt *xprt = 1543 container_of(work, struct rpc_xprt, task_cleanup); 1544 1545 rpc_xprt_debugfs_unregister(xprt); 1546 rpc_destroy_wait_queue(&xprt->binding); 1547 rpc_destroy_wait_queue(&xprt->pending); 1548 rpc_destroy_wait_queue(&xprt->sending); 1549 rpc_destroy_wait_queue(&xprt->backlog); 1550 kfree(xprt->servername); 1551 /* 1552 * Tear down transport state and free the rpc_xprt 1553 */ 1554 xprt->ops->destroy(xprt); 1555 } 1556 1557 /** 1558 * xprt_destroy - destroy an RPC transport, killing off all requests. 1559 * @xprt: transport to destroy 1560 * 1561 */ 1562 static void xprt_destroy(struct rpc_xprt *xprt) 1563 { 1564 dprintk("RPC: destroying transport %p\n", xprt); 1565 1566 /* 1567 * Exclude transport connect/disconnect handlers and autoclose 1568 */ 1569 wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE); 1570 1571 del_timer_sync(&xprt->timer); 1572 1573 /* 1574 * Destroy sockets etc from the system workqueue so they can 1575 * safely flush receive work running on rpciod. 1576 */ 1577 INIT_WORK(&xprt->task_cleanup, xprt_destroy_cb); 1578 schedule_work(&xprt->task_cleanup); 1579 } 1580 1581 static void xprt_destroy_kref(struct kref *kref) 1582 { 1583 xprt_destroy(container_of(kref, struct rpc_xprt, kref)); 1584 } 1585 1586 /** 1587 * xprt_get - return a reference to an RPC transport. 1588 * @xprt: pointer to the transport 1589 * 1590 */ 1591 struct rpc_xprt *xprt_get(struct rpc_xprt *xprt) 1592 { 1593 if (xprt != NULL && kref_get_unless_zero(&xprt->kref)) 1594 return xprt; 1595 return NULL; 1596 } 1597 EXPORT_SYMBOL_GPL(xprt_get); 1598 1599 /** 1600 * xprt_put - release a reference to an RPC transport. 1601 * @xprt: pointer to the transport 1602 * 1603 */ 1604 void xprt_put(struct rpc_xprt *xprt) 1605 { 1606 if (xprt != NULL) 1607 kref_put(&xprt->kref, xprt_destroy_kref); 1608 } 1609 EXPORT_SYMBOL_GPL(xprt_put); 1610