1 /* 2 drbd_state.c 3 4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg. 5 6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH. 7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>. 8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. 9 10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev 11 from Logicworks, Inc. for making SDP replication support possible. 12 13 drbd is free software; you can redistribute it and/or modify 14 it under the terms of the GNU General Public License as published by 15 the Free Software Foundation; either version 2, or (at your option) 16 any later version. 17 18 drbd is distributed in the hope that it will be useful, 19 but WITHOUT ANY WARRANTY; without even the implied warranty of 20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 GNU General Public License for more details. 22 23 You should have received a copy of the GNU General Public License 24 along with drbd; see the file COPYING. If not, write to 25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 26 */ 27 28 #include <linux/drbd_limits.h> 29 #include "drbd_int.h" 30 #include "drbd_protocol.h" 31 #include "drbd_req.h" 32 #include "drbd_state_change.h" 33 34 struct after_state_chg_work { 35 struct drbd_work w; 36 struct drbd_device *device; 37 union drbd_state os; 38 union drbd_state ns; 39 enum chg_state_flags flags; 40 struct completion *done; 41 struct drbd_state_change *state_change; 42 }; 43 44 enum sanitize_state_warnings { 45 NO_WARNING, 46 ABORTED_ONLINE_VERIFY, 47 ABORTED_RESYNC, 48 CONNECTION_LOST_NEGOTIATING, 49 IMPLICITLY_UPGRADED_DISK, 50 IMPLICITLY_UPGRADED_PDSK, 51 }; 52 53 static void count_objects(struct drbd_resource *resource, 54 unsigned int *n_devices, 55 unsigned int *n_connections) 56 { 57 struct drbd_device *device; 58 struct drbd_connection *connection; 59 int vnr; 60 61 *n_devices = 0; 62 *n_connections = 0; 63 64 idr_for_each_entry(&resource->devices, device, vnr) 65 (*n_devices)++; 66 for_each_connection(connection, resource) 67 (*n_connections)++; 68 } 69 70 static struct drbd_state_change *alloc_state_change(unsigned int n_devices, unsigned int n_connections, gfp_t gfp) 71 { 72 struct drbd_state_change *state_change; 73 unsigned int size, n; 74 75 size = sizeof(struct drbd_state_change) + 76 n_devices * sizeof(struct drbd_device_state_change) + 77 n_connections * sizeof(struct drbd_connection_state_change) + 78 n_devices * n_connections * sizeof(struct drbd_peer_device_state_change); 79 state_change = kmalloc(size, gfp); 80 if (!state_change) 81 return NULL; 82 state_change->n_devices = n_devices; 83 state_change->n_connections = n_connections; 84 state_change->devices = (void *)(state_change + 1); 85 state_change->connections = (void *)&state_change->devices[n_devices]; 86 state_change->peer_devices = (void *)&state_change->connections[n_connections]; 87 state_change->resource->resource = NULL; 88 for (n = 0; n < n_devices; n++) 89 state_change->devices[n].device = NULL; 90 for (n = 0; n < n_connections; n++) 91 state_change->connections[n].connection = NULL; 92 return state_change; 93 } 94 95 struct drbd_state_change *remember_old_state(struct drbd_resource *resource, gfp_t gfp) 96 { 97 struct drbd_state_change *state_change; 98 struct drbd_device *device; 99 unsigned int n_devices; 100 struct drbd_connection *connection; 101 unsigned int n_connections; 102 int vnr; 103 104 struct drbd_device_state_change *device_state_change; 105 struct drbd_peer_device_state_change *peer_device_state_change; 106 struct drbd_connection_state_change *connection_state_change; 107 108 /* Caller holds req_lock spinlock. 109 * No state, no device IDR, no connections lists can change. */ 110 count_objects(resource, &n_devices, &n_connections); 111 state_change = alloc_state_change(n_devices, n_connections, gfp); 112 if (!state_change) 113 return NULL; 114 115 kref_get(&resource->kref); 116 state_change->resource->resource = resource; 117 state_change->resource->role[OLD] = 118 conn_highest_role(first_connection(resource)); 119 state_change->resource->susp[OLD] = resource->susp; 120 state_change->resource->susp_nod[OLD] = resource->susp_nod; 121 state_change->resource->susp_fen[OLD] = resource->susp_fen; 122 123 connection_state_change = state_change->connections; 124 for_each_connection(connection, resource) { 125 kref_get(&connection->kref); 126 connection_state_change->connection = connection; 127 connection_state_change->cstate[OLD] = 128 connection->cstate; 129 connection_state_change->peer_role[OLD] = 130 conn_highest_peer(connection); 131 connection_state_change++; 132 } 133 134 device_state_change = state_change->devices; 135 peer_device_state_change = state_change->peer_devices; 136 idr_for_each_entry(&resource->devices, device, vnr) { 137 kref_get(&device->kref); 138 device_state_change->device = device; 139 device_state_change->disk_state[OLD] = device->state.disk; 140 141 /* The peer_devices for each device have to be enumerated in 142 the order of the connections. We may not use for_each_peer_device() here. */ 143 for_each_connection(connection, resource) { 144 struct drbd_peer_device *peer_device; 145 146 peer_device = conn_peer_device(connection, device->vnr); 147 peer_device_state_change->peer_device = peer_device; 148 peer_device_state_change->disk_state[OLD] = 149 device->state.pdsk; 150 peer_device_state_change->repl_state[OLD] = 151 max_t(enum drbd_conns, 152 C_WF_REPORT_PARAMS, device->state.conn); 153 peer_device_state_change->resync_susp_user[OLD] = 154 device->state.user_isp; 155 peer_device_state_change->resync_susp_peer[OLD] = 156 device->state.peer_isp; 157 peer_device_state_change->resync_susp_dependency[OLD] = 158 device->state.aftr_isp; 159 peer_device_state_change++; 160 } 161 device_state_change++; 162 } 163 164 return state_change; 165 } 166 167 static void remember_new_state(struct drbd_state_change *state_change) 168 { 169 struct drbd_resource_state_change *resource_state_change; 170 struct drbd_resource *resource; 171 unsigned int n; 172 173 if (!state_change) 174 return; 175 176 resource_state_change = &state_change->resource[0]; 177 resource = resource_state_change->resource; 178 179 resource_state_change->role[NEW] = 180 conn_highest_role(first_connection(resource)); 181 resource_state_change->susp[NEW] = resource->susp; 182 resource_state_change->susp_nod[NEW] = resource->susp_nod; 183 resource_state_change->susp_fen[NEW] = resource->susp_fen; 184 185 for (n = 0; n < state_change->n_devices; n++) { 186 struct drbd_device_state_change *device_state_change = 187 &state_change->devices[n]; 188 struct drbd_device *device = device_state_change->device; 189 190 device_state_change->disk_state[NEW] = device->state.disk; 191 } 192 193 for (n = 0; n < state_change->n_connections; n++) { 194 struct drbd_connection_state_change *connection_state_change = 195 &state_change->connections[n]; 196 struct drbd_connection *connection = 197 connection_state_change->connection; 198 199 connection_state_change->cstate[NEW] = connection->cstate; 200 connection_state_change->peer_role[NEW] = 201 conn_highest_peer(connection); 202 } 203 204 for (n = 0; n < state_change->n_devices * state_change->n_connections; n++) { 205 struct drbd_peer_device_state_change *peer_device_state_change = 206 &state_change->peer_devices[n]; 207 struct drbd_device *device = 208 peer_device_state_change->peer_device->device; 209 union drbd_dev_state state = device->state; 210 211 peer_device_state_change->disk_state[NEW] = state.pdsk; 212 peer_device_state_change->repl_state[NEW] = 213 max_t(enum drbd_conns, C_WF_REPORT_PARAMS, state.conn); 214 peer_device_state_change->resync_susp_user[NEW] = 215 state.user_isp; 216 peer_device_state_change->resync_susp_peer[NEW] = 217 state.peer_isp; 218 peer_device_state_change->resync_susp_dependency[NEW] = 219 state.aftr_isp; 220 } 221 } 222 223 void copy_old_to_new_state_change(struct drbd_state_change *state_change) 224 { 225 struct drbd_resource_state_change *resource_state_change = &state_change->resource[0]; 226 unsigned int n_device, n_connection, n_peer_device, n_peer_devices; 227 228 #define OLD_TO_NEW(x) \ 229 (x[NEW] = x[OLD]) 230 231 OLD_TO_NEW(resource_state_change->role); 232 OLD_TO_NEW(resource_state_change->susp); 233 OLD_TO_NEW(resource_state_change->susp_nod); 234 OLD_TO_NEW(resource_state_change->susp_fen); 235 236 for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) { 237 struct drbd_connection_state_change *connection_state_change = 238 &state_change->connections[n_connection]; 239 240 OLD_TO_NEW(connection_state_change->peer_role); 241 OLD_TO_NEW(connection_state_change->cstate); 242 } 243 244 for (n_device = 0; n_device < state_change->n_devices; n_device++) { 245 struct drbd_device_state_change *device_state_change = 246 &state_change->devices[n_device]; 247 248 OLD_TO_NEW(device_state_change->disk_state); 249 } 250 251 n_peer_devices = state_change->n_devices * state_change->n_connections; 252 for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) { 253 struct drbd_peer_device_state_change *p = 254 &state_change->peer_devices[n_peer_device]; 255 256 OLD_TO_NEW(p->disk_state); 257 OLD_TO_NEW(p->repl_state); 258 OLD_TO_NEW(p->resync_susp_user); 259 OLD_TO_NEW(p->resync_susp_peer); 260 OLD_TO_NEW(p->resync_susp_dependency); 261 } 262 263 #undef OLD_TO_NEW 264 } 265 266 void forget_state_change(struct drbd_state_change *state_change) 267 { 268 unsigned int n; 269 270 if (!state_change) 271 return; 272 273 if (state_change->resource->resource) 274 kref_put(&state_change->resource->resource->kref, drbd_destroy_resource); 275 for (n = 0; n < state_change->n_devices; n++) { 276 struct drbd_device *device = state_change->devices[n].device; 277 278 if (device) 279 kref_put(&device->kref, drbd_destroy_device); 280 } 281 for (n = 0; n < state_change->n_connections; n++) { 282 struct drbd_connection *connection = 283 state_change->connections[n].connection; 284 285 if (connection) 286 kref_put(&connection->kref, drbd_destroy_connection); 287 } 288 kfree(state_change); 289 } 290 291 static int w_after_state_ch(struct drbd_work *w, int unused); 292 static void after_state_ch(struct drbd_device *device, union drbd_state os, 293 union drbd_state ns, enum chg_state_flags flags, 294 struct drbd_state_change *); 295 static enum drbd_state_rv is_valid_state(struct drbd_device *, union drbd_state); 296 static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state, struct drbd_connection *); 297 static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns); 298 static union drbd_state sanitize_state(struct drbd_device *device, union drbd_state os, 299 union drbd_state ns, enum sanitize_state_warnings *warn); 300 301 static inline bool is_susp(union drbd_state s) 302 { 303 return s.susp || s.susp_nod || s.susp_fen; 304 } 305 306 bool conn_all_vols_unconf(struct drbd_connection *connection) 307 { 308 struct drbd_peer_device *peer_device; 309 bool rv = true; 310 int vnr; 311 312 rcu_read_lock(); 313 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 314 struct drbd_device *device = peer_device->device; 315 if (device->state.disk != D_DISKLESS || 316 device->state.conn != C_STANDALONE || 317 device->state.role != R_SECONDARY) { 318 rv = false; 319 break; 320 } 321 } 322 rcu_read_unlock(); 323 324 return rv; 325 } 326 327 /* Unfortunately the states where not correctly ordered, when 328 they where defined. therefore can not use max_t() here. */ 329 static enum drbd_role max_role(enum drbd_role role1, enum drbd_role role2) 330 { 331 if (role1 == R_PRIMARY || role2 == R_PRIMARY) 332 return R_PRIMARY; 333 if (role1 == R_SECONDARY || role2 == R_SECONDARY) 334 return R_SECONDARY; 335 return R_UNKNOWN; 336 } 337 338 static enum drbd_role min_role(enum drbd_role role1, enum drbd_role role2) 339 { 340 if (role1 == R_UNKNOWN || role2 == R_UNKNOWN) 341 return R_UNKNOWN; 342 if (role1 == R_SECONDARY || role2 == R_SECONDARY) 343 return R_SECONDARY; 344 return R_PRIMARY; 345 } 346 347 enum drbd_role conn_highest_role(struct drbd_connection *connection) 348 { 349 enum drbd_role role = R_SECONDARY; 350 struct drbd_peer_device *peer_device; 351 int vnr; 352 353 rcu_read_lock(); 354 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 355 struct drbd_device *device = peer_device->device; 356 role = max_role(role, device->state.role); 357 } 358 rcu_read_unlock(); 359 360 return role; 361 } 362 363 enum drbd_role conn_highest_peer(struct drbd_connection *connection) 364 { 365 enum drbd_role peer = R_UNKNOWN; 366 struct drbd_peer_device *peer_device; 367 int vnr; 368 369 rcu_read_lock(); 370 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 371 struct drbd_device *device = peer_device->device; 372 peer = max_role(peer, device->state.peer); 373 } 374 rcu_read_unlock(); 375 376 return peer; 377 } 378 379 enum drbd_disk_state conn_highest_disk(struct drbd_connection *connection) 380 { 381 enum drbd_disk_state disk_state = D_DISKLESS; 382 struct drbd_peer_device *peer_device; 383 int vnr; 384 385 rcu_read_lock(); 386 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 387 struct drbd_device *device = peer_device->device; 388 disk_state = max_t(enum drbd_disk_state, disk_state, device->state.disk); 389 } 390 rcu_read_unlock(); 391 392 return disk_state; 393 } 394 395 enum drbd_disk_state conn_lowest_disk(struct drbd_connection *connection) 396 { 397 enum drbd_disk_state disk_state = D_MASK; 398 struct drbd_peer_device *peer_device; 399 int vnr; 400 401 rcu_read_lock(); 402 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 403 struct drbd_device *device = peer_device->device; 404 disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk); 405 } 406 rcu_read_unlock(); 407 408 return disk_state; 409 } 410 411 enum drbd_disk_state conn_highest_pdsk(struct drbd_connection *connection) 412 { 413 enum drbd_disk_state disk_state = D_DISKLESS; 414 struct drbd_peer_device *peer_device; 415 int vnr; 416 417 rcu_read_lock(); 418 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 419 struct drbd_device *device = peer_device->device; 420 disk_state = max_t(enum drbd_disk_state, disk_state, device->state.pdsk); 421 } 422 rcu_read_unlock(); 423 424 return disk_state; 425 } 426 427 enum drbd_conns conn_lowest_conn(struct drbd_connection *connection) 428 { 429 enum drbd_conns conn = C_MASK; 430 struct drbd_peer_device *peer_device; 431 int vnr; 432 433 rcu_read_lock(); 434 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 435 struct drbd_device *device = peer_device->device; 436 conn = min_t(enum drbd_conns, conn, device->state.conn); 437 } 438 rcu_read_unlock(); 439 440 return conn; 441 } 442 443 static bool no_peer_wf_report_params(struct drbd_connection *connection) 444 { 445 struct drbd_peer_device *peer_device; 446 int vnr; 447 bool rv = true; 448 449 rcu_read_lock(); 450 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 451 if (peer_device->device->state.conn == C_WF_REPORT_PARAMS) { 452 rv = false; 453 break; 454 } 455 rcu_read_unlock(); 456 457 return rv; 458 } 459 460 static void wake_up_all_devices(struct drbd_connection *connection) 461 { 462 struct drbd_peer_device *peer_device; 463 int vnr; 464 465 rcu_read_lock(); 466 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 467 wake_up(&peer_device->device->state_wait); 468 rcu_read_unlock(); 469 470 } 471 472 473 /** 474 * cl_wide_st_chg() - true if the state change is a cluster wide one 475 * @device: DRBD device. 476 * @os: old (current) state. 477 * @ns: new (wanted) state. 478 */ 479 static int cl_wide_st_chg(struct drbd_device *device, 480 union drbd_state os, union drbd_state ns) 481 { 482 return (os.conn >= C_CONNECTED && ns.conn >= C_CONNECTED && 483 ((os.role != R_PRIMARY && ns.role == R_PRIMARY) || 484 (os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) || 485 (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S) || 486 (os.disk != D_FAILED && ns.disk == D_FAILED))) || 487 (os.conn >= C_CONNECTED && ns.conn == C_DISCONNECTING) || 488 (os.conn == C_CONNECTED && ns.conn == C_VERIFY_S) || 489 (os.conn == C_CONNECTED && ns.conn == C_WF_REPORT_PARAMS); 490 } 491 492 static union drbd_state 493 apply_mask_val(union drbd_state os, union drbd_state mask, union drbd_state val) 494 { 495 union drbd_state ns; 496 ns.i = (os.i & ~mask.i) | val.i; 497 return ns; 498 } 499 500 enum drbd_state_rv 501 drbd_change_state(struct drbd_device *device, enum chg_state_flags f, 502 union drbd_state mask, union drbd_state val) 503 { 504 unsigned long flags; 505 union drbd_state ns; 506 enum drbd_state_rv rv; 507 508 spin_lock_irqsave(&device->resource->req_lock, flags); 509 ns = apply_mask_val(drbd_read_state(device), mask, val); 510 rv = _drbd_set_state(device, ns, f, NULL); 511 spin_unlock_irqrestore(&device->resource->req_lock, flags); 512 513 return rv; 514 } 515 516 /** 517 * drbd_force_state() - Impose a change which happens outside our control on our state 518 * @device: DRBD device. 519 * @mask: mask of state bits to change. 520 * @val: value of new state bits. 521 */ 522 void drbd_force_state(struct drbd_device *device, 523 union drbd_state mask, union drbd_state val) 524 { 525 drbd_change_state(device, CS_HARD, mask, val); 526 } 527 528 static enum drbd_state_rv 529 _req_st_cond(struct drbd_device *device, union drbd_state mask, 530 union drbd_state val) 531 { 532 union drbd_state os, ns; 533 unsigned long flags; 534 enum drbd_state_rv rv; 535 536 if (test_and_clear_bit(CL_ST_CHG_SUCCESS, &device->flags)) 537 return SS_CW_SUCCESS; 538 539 if (test_and_clear_bit(CL_ST_CHG_FAIL, &device->flags)) 540 return SS_CW_FAILED_BY_PEER; 541 542 spin_lock_irqsave(&device->resource->req_lock, flags); 543 os = drbd_read_state(device); 544 ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL); 545 rv = is_valid_transition(os, ns); 546 if (rv >= SS_SUCCESS) 547 rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */ 548 549 if (!cl_wide_st_chg(device, os, ns)) 550 rv = SS_CW_NO_NEED; 551 if (rv == SS_UNKNOWN_ERROR) { 552 rv = is_valid_state(device, ns); 553 if (rv >= SS_SUCCESS) { 554 rv = is_valid_soft_transition(os, ns, first_peer_device(device)->connection); 555 if (rv >= SS_SUCCESS) 556 rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */ 557 } 558 } 559 spin_unlock_irqrestore(&device->resource->req_lock, flags); 560 561 return rv; 562 } 563 564 /** 565 * drbd_req_state() - Perform an eventually cluster wide state change 566 * @device: DRBD device. 567 * @mask: mask of state bits to change. 568 * @val: value of new state bits. 569 * @f: flags 570 * 571 * Should not be called directly, use drbd_request_state() or 572 * _drbd_request_state(). 573 */ 574 static enum drbd_state_rv 575 drbd_req_state(struct drbd_device *device, union drbd_state mask, 576 union drbd_state val, enum chg_state_flags f) 577 { 578 struct completion done; 579 unsigned long flags; 580 union drbd_state os, ns; 581 enum drbd_state_rv rv; 582 void *buffer = NULL; 583 584 init_completion(&done); 585 586 if (f & CS_SERIALIZE) 587 mutex_lock(device->state_mutex); 588 if (f & CS_INHIBIT_MD_IO) 589 buffer = drbd_md_get_buffer(device, __func__); 590 591 spin_lock_irqsave(&device->resource->req_lock, flags); 592 os = drbd_read_state(device); 593 ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL); 594 rv = is_valid_transition(os, ns); 595 if (rv < SS_SUCCESS) { 596 spin_unlock_irqrestore(&device->resource->req_lock, flags); 597 goto abort; 598 } 599 600 if (cl_wide_st_chg(device, os, ns)) { 601 rv = is_valid_state(device, ns); 602 if (rv == SS_SUCCESS) 603 rv = is_valid_soft_transition(os, ns, first_peer_device(device)->connection); 604 spin_unlock_irqrestore(&device->resource->req_lock, flags); 605 606 if (rv < SS_SUCCESS) { 607 if (f & CS_VERBOSE) 608 print_st_err(device, os, ns, rv); 609 goto abort; 610 } 611 612 if (drbd_send_state_req(first_peer_device(device), mask, val)) { 613 rv = SS_CW_FAILED_BY_PEER; 614 if (f & CS_VERBOSE) 615 print_st_err(device, os, ns, rv); 616 goto abort; 617 } 618 619 wait_event(device->state_wait, 620 (rv = _req_st_cond(device, mask, val))); 621 622 if (rv < SS_SUCCESS) { 623 if (f & CS_VERBOSE) 624 print_st_err(device, os, ns, rv); 625 goto abort; 626 } 627 spin_lock_irqsave(&device->resource->req_lock, flags); 628 ns = apply_mask_val(drbd_read_state(device), mask, val); 629 rv = _drbd_set_state(device, ns, f, &done); 630 } else { 631 rv = _drbd_set_state(device, ns, f, &done); 632 } 633 634 spin_unlock_irqrestore(&device->resource->req_lock, flags); 635 636 if (f & CS_WAIT_COMPLETE && rv == SS_SUCCESS) { 637 D_ASSERT(device, current != first_peer_device(device)->connection->worker.task); 638 wait_for_completion(&done); 639 } 640 641 abort: 642 if (buffer) 643 drbd_md_put_buffer(device); 644 if (f & CS_SERIALIZE) 645 mutex_unlock(device->state_mutex); 646 647 return rv; 648 } 649 650 /** 651 * _drbd_request_state() - Request a state change (with flags) 652 * @device: DRBD device. 653 * @mask: mask of state bits to change. 654 * @val: value of new state bits. 655 * @f: flags 656 * 657 * Cousin of drbd_request_state(), useful with the CS_WAIT_COMPLETE 658 * flag, or when logging of failed state change requests is not desired. 659 */ 660 enum drbd_state_rv 661 _drbd_request_state(struct drbd_device *device, union drbd_state mask, 662 union drbd_state val, enum chg_state_flags f) 663 { 664 enum drbd_state_rv rv; 665 666 wait_event(device->state_wait, 667 (rv = drbd_req_state(device, mask, val, f)) != SS_IN_TRANSIENT_STATE); 668 669 return rv; 670 } 671 672 /* 673 * We grab drbd_md_get_buffer(), because we don't want to "fail" the disk while 674 * there is IO in-flight: the transition into D_FAILED for detach purposes 675 * may get misinterpreted as actual IO error in a confused endio function. 676 * 677 * We wrap it all into wait_event(), to retry in case the drbd_req_state() 678 * returns SS_IN_TRANSIENT_STATE. 679 * 680 * To avoid potential deadlock with e.g. the receiver thread trying to grab 681 * drbd_md_get_buffer() while trying to get out of the "transient state", we 682 * need to grab and release the meta data buffer inside of that wait_event loop. 683 */ 684 static enum drbd_state_rv 685 request_detach(struct drbd_device *device) 686 { 687 return drbd_req_state(device, NS(disk, D_FAILED), 688 CS_VERBOSE | CS_ORDERED | CS_INHIBIT_MD_IO); 689 } 690 691 int drbd_request_detach_interruptible(struct drbd_device *device) 692 { 693 int ret, rv; 694 695 drbd_suspend_io(device); /* so no-one is stuck in drbd_al_begin_io */ 696 wait_event_interruptible(device->state_wait, 697 (rv = request_detach(device)) != SS_IN_TRANSIENT_STATE); 698 drbd_resume_io(device); 699 700 ret = wait_event_interruptible(device->misc_wait, 701 device->state.disk != D_FAILED); 702 703 if (rv == SS_IS_DISKLESS) 704 rv = SS_NOTHING_TO_DO; 705 if (ret) 706 rv = ERR_INTR; 707 708 return rv; 709 } 710 711 enum drbd_state_rv 712 _drbd_request_state_holding_state_mutex(struct drbd_device *device, union drbd_state mask, 713 union drbd_state val, enum chg_state_flags f) 714 { 715 enum drbd_state_rv rv; 716 717 BUG_ON(f & CS_SERIALIZE); 718 719 wait_event_cmd(device->state_wait, 720 (rv = drbd_req_state(device, mask, val, f)) != SS_IN_TRANSIENT_STATE, 721 mutex_unlock(device->state_mutex), 722 mutex_lock(device->state_mutex)); 723 724 return rv; 725 } 726 727 static void print_st(struct drbd_device *device, const char *name, union drbd_state ns) 728 { 729 drbd_err(device, " %s = { cs:%s ro:%s/%s ds:%s/%s %c%c%c%c%c%c }\n", 730 name, 731 drbd_conn_str(ns.conn), 732 drbd_role_str(ns.role), 733 drbd_role_str(ns.peer), 734 drbd_disk_str(ns.disk), 735 drbd_disk_str(ns.pdsk), 736 is_susp(ns) ? 's' : 'r', 737 ns.aftr_isp ? 'a' : '-', 738 ns.peer_isp ? 'p' : '-', 739 ns.user_isp ? 'u' : '-', 740 ns.susp_fen ? 'F' : '-', 741 ns.susp_nod ? 'N' : '-' 742 ); 743 } 744 745 void print_st_err(struct drbd_device *device, union drbd_state os, 746 union drbd_state ns, enum drbd_state_rv err) 747 { 748 if (err == SS_IN_TRANSIENT_STATE) 749 return; 750 drbd_err(device, "State change failed: %s\n", drbd_set_st_err_str(err)); 751 print_st(device, " state", os); 752 print_st(device, "wanted", ns); 753 } 754 755 static long print_state_change(char *pb, union drbd_state os, union drbd_state ns, 756 enum chg_state_flags flags) 757 { 758 char *pbp; 759 pbp = pb; 760 *pbp = 0; 761 762 if (ns.role != os.role && flags & CS_DC_ROLE) 763 pbp += sprintf(pbp, "role( %s -> %s ) ", 764 drbd_role_str(os.role), 765 drbd_role_str(ns.role)); 766 if (ns.peer != os.peer && flags & CS_DC_PEER) 767 pbp += sprintf(pbp, "peer( %s -> %s ) ", 768 drbd_role_str(os.peer), 769 drbd_role_str(ns.peer)); 770 if (ns.conn != os.conn && flags & CS_DC_CONN) 771 pbp += sprintf(pbp, "conn( %s -> %s ) ", 772 drbd_conn_str(os.conn), 773 drbd_conn_str(ns.conn)); 774 if (ns.disk != os.disk && flags & CS_DC_DISK) 775 pbp += sprintf(pbp, "disk( %s -> %s ) ", 776 drbd_disk_str(os.disk), 777 drbd_disk_str(ns.disk)); 778 if (ns.pdsk != os.pdsk && flags & CS_DC_PDSK) 779 pbp += sprintf(pbp, "pdsk( %s -> %s ) ", 780 drbd_disk_str(os.pdsk), 781 drbd_disk_str(ns.pdsk)); 782 783 return pbp - pb; 784 } 785 786 static void drbd_pr_state_change(struct drbd_device *device, union drbd_state os, union drbd_state ns, 787 enum chg_state_flags flags) 788 { 789 char pb[300]; 790 char *pbp = pb; 791 792 pbp += print_state_change(pbp, os, ns, flags ^ CS_DC_MASK); 793 794 if (ns.aftr_isp != os.aftr_isp) 795 pbp += sprintf(pbp, "aftr_isp( %d -> %d ) ", 796 os.aftr_isp, 797 ns.aftr_isp); 798 if (ns.peer_isp != os.peer_isp) 799 pbp += sprintf(pbp, "peer_isp( %d -> %d ) ", 800 os.peer_isp, 801 ns.peer_isp); 802 if (ns.user_isp != os.user_isp) 803 pbp += sprintf(pbp, "user_isp( %d -> %d ) ", 804 os.user_isp, 805 ns.user_isp); 806 807 if (pbp != pb) 808 drbd_info(device, "%s\n", pb); 809 } 810 811 static void conn_pr_state_change(struct drbd_connection *connection, union drbd_state os, union drbd_state ns, 812 enum chg_state_flags flags) 813 { 814 char pb[300]; 815 char *pbp = pb; 816 817 pbp += print_state_change(pbp, os, ns, flags); 818 819 if (is_susp(ns) != is_susp(os) && flags & CS_DC_SUSP) 820 pbp += sprintf(pbp, "susp( %d -> %d ) ", 821 is_susp(os), 822 is_susp(ns)); 823 824 if (pbp != pb) 825 drbd_info(connection, "%s\n", pb); 826 } 827 828 829 /** 830 * is_valid_state() - Returns an SS_ error code if ns is not valid 831 * @device: DRBD device. 832 * @ns: State to consider. 833 */ 834 static enum drbd_state_rv 835 is_valid_state(struct drbd_device *device, union drbd_state ns) 836 { 837 /* See drbd_state_sw_errors in drbd_strings.c */ 838 839 enum drbd_fencing_p fp; 840 enum drbd_state_rv rv = SS_SUCCESS; 841 struct net_conf *nc; 842 843 rcu_read_lock(); 844 fp = FP_DONT_CARE; 845 if (get_ldev(device)) { 846 fp = rcu_dereference(device->ldev->disk_conf)->fencing; 847 put_ldev(device); 848 } 849 850 nc = rcu_dereference(first_peer_device(device)->connection->net_conf); 851 if (nc) { 852 if (!nc->two_primaries && ns.role == R_PRIMARY) { 853 if (ns.peer == R_PRIMARY) 854 rv = SS_TWO_PRIMARIES; 855 else if (conn_highest_peer(first_peer_device(device)->connection) == R_PRIMARY) 856 rv = SS_O_VOL_PEER_PRI; 857 } 858 } 859 860 if (rv <= 0) 861 goto out; /* already found a reason to abort */ 862 else if (ns.role == R_SECONDARY && device->open_cnt) 863 rv = SS_DEVICE_IN_USE; 864 865 else if (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.disk < D_UP_TO_DATE) 866 rv = SS_NO_UP_TO_DATE_DISK; 867 868 else if (fp >= FP_RESOURCE && 869 ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk >= D_UNKNOWN) 870 rv = SS_PRIMARY_NOP; 871 872 else if (ns.role == R_PRIMARY && ns.disk <= D_INCONSISTENT && ns.pdsk <= D_INCONSISTENT) 873 rv = SS_NO_UP_TO_DATE_DISK; 874 875 else if (ns.conn > C_CONNECTED && ns.disk < D_INCONSISTENT) 876 rv = SS_NO_LOCAL_DISK; 877 878 else if (ns.conn > C_CONNECTED && ns.pdsk < D_INCONSISTENT) 879 rv = SS_NO_REMOTE_DISK; 880 881 else if (ns.conn > C_CONNECTED && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) 882 rv = SS_NO_UP_TO_DATE_DISK; 883 884 else if ((ns.conn == C_CONNECTED || 885 ns.conn == C_WF_BITMAP_S || 886 ns.conn == C_SYNC_SOURCE || 887 ns.conn == C_PAUSED_SYNC_S) && 888 ns.disk == D_OUTDATED) 889 rv = SS_CONNECTED_OUTDATES; 890 891 else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && 892 (nc->verify_alg[0] == 0)) 893 rv = SS_NO_VERIFY_ALG; 894 895 else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && 896 first_peer_device(device)->connection->agreed_pro_version < 88) 897 rv = SS_NOT_SUPPORTED; 898 899 else if (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) 900 rv = SS_NO_UP_TO_DATE_DISK; 901 902 else if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) && 903 ns.pdsk == D_UNKNOWN) 904 rv = SS_NEED_CONNECTION; 905 906 else if (ns.conn >= C_CONNECTED && ns.pdsk == D_UNKNOWN) 907 rv = SS_CONNECTED_OUTDATES; 908 909 out: 910 rcu_read_unlock(); 911 912 return rv; 913 } 914 915 /** 916 * is_valid_soft_transition() - Returns an SS_ error code if the state transition is not possible 917 * This function limits state transitions that may be declined by DRBD. I.e. 918 * user requests (aka soft transitions). 919 * @device: DRBD device. 920 * @ns: new state. 921 * @os: old state. 922 */ 923 static enum drbd_state_rv 924 is_valid_soft_transition(union drbd_state os, union drbd_state ns, struct drbd_connection *connection) 925 { 926 enum drbd_state_rv rv = SS_SUCCESS; 927 928 if ((ns.conn == C_STARTING_SYNC_T || ns.conn == C_STARTING_SYNC_S) && 929 os.conn > C_CONNECTED) 930 rv = SS_RESYNC_RUNNING; 931 932 if (ns.conn == C_DISCONNECTING && os.conn == C_STANDALONE) 933 rv = SS_ALREADY_STANDALONE; 934 935 if (ns.disk > D_ATTACHING && os.disk == D_DISKLESS) 936 rv = SS_IS_DISKLESS; 937 938 if (ns.conn == C_WF_CONNECTION && os.conn < C_UNCONNECTED) 939 rv = SS_NO_NET_CONFIG; 940 941 if (ns.disk == D_OUTDATED && os.disk < D_OUTDATED && os.disk != D_ATTACHING) 942 rv = SS_LOWER_THAN_OUTDATED; 943 944 if (ns.conn == C_DISCONNECTING && os.conn == C_UNCONNECTED) 945 rv = SS_IN_TRANSIENT_STATE; 946 947 /* While establishing a connection only allow cstate to change. 948 Delay/refuse role changes, detach attach etc... (they do not touch cstate) */ 949 if (test_bit(STATE_SENT, &connection->flags) && 950 !((ns.conn == C_WF_REPORT_PARAMS && os.conn == C_WF_CONNECTION) || 951 (ns.conn >= C_CONNECTED && os.conn == C_WF_REPORT_PARAMS))) 952 rv = SS_IN_TRANSIENT_STATE; 953 954 /* Do not promote during resync handshake triggered by "force primary". 955 * This is a hack. It should really be rejected by the peer during the 956 * cluster wide state change request. */ 957 if (os.role != R_PRIMARY && ns.role == R_PRIMARY 958 && ns.pdsk == D_UP_TO_DATE 959 && ns.disk != D_UP_TO_DATE && ns.disk != D_DISKLESS 960 && (ns.conn <= C_WF_SYNC_UUID || ns.conn != os.conn)) 961 rv = SS_IN_TRANSIENT_STATE; 962 963 if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED) 964 rv = SS_NEED_CONNECTION; 965 966 if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && 967 ns.conn != os.conn && os.conn > C_CONNECTED) 968 rv = SS_RESYNC_RUNNING; 969 970 if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) && 971 os.conn < C_CONNECTED) 972 rv = SS_NEED_CONNECTION; 973 974 if ((ns.conn == C_SYNC_TARGET || ns.conn == C_SYNC_SOURCE) 975 && os.conn < C_WF_REPORT_PARAMS) 976 rv = SS_NEED_CONNECTION; /* No NetworkFailure -> SyncTarget etc... */ 977 978 if (ns.conn == C_DISCONNECTING && ns.pdsk == D_OUTDATED && 979 os.conn < C_CONNECTED && os.pdsk > D_OUTDATED) 980 rv = SS_OUTDATE_WO_CONN; 981 982 return rv; 983 } 984 985 static enum drbd_state_rv 986 is_valid_conn_transition(enum drbd_conns oc, enum drbd_conns nc) 987 { 988 /* no change -> nothing to do, at least for the connection part */ 989 if (oc == nc) 990 return SS_NOTHING_TO_DO; 991 992 /* disconnect of an unconfigured connection does not make sense */ 993 if (oc == C_STANDALONE && nc == C_DISCONNECTING) 994 return SS_ALREADY_STANDALONE; 995 996 /* from C_STANDALONE, we start with C_UNCONNECTED */ 997 if (oc == C_STANDALONE && nc != C_UNCONNECTED) 998 return SS_NEED_CONNECTION; 999 1000 /* When establishing a connection we need to go through WF_REPORT_PARAMS! 1001 Necessary to do the right thing upon invalidate-remote on a disconnected resource */ 1002 if (oc < C_WF_REPORT_PARAMS && nc >= C_CONNECTED) 1003 return SS_NEED_CONNECTION; 1004 1005 /* After a network error only C_UNCONNECTED or C_DISCONNECTING may follow. */ 1006 if (oc >= C_TIMEOUT && oc <= C_TEAR_DOWN && nc != C_UNCONNECTED && nc != C_DISCONNECTING) 1007 return SS_IN_TRANSIENT_STATE; 1008 1009 /* After C_DISCONNECTING only C_STANDALONE may follow */ 1010 if (oc == C_DISCONNECTING && nc != C_STANDALONE) 1011 return SS_IN_TRANSIENT_STATE; 1012 1013 return SS_SUCCESS; 1014 } 1015 1016 1017 /** 1018 * is_valid_transition() - Returns an SS_ error code if the state transition is not possible 1019 * This limits hard state transitions. Hard state transitions are facts there are 1020 * imposed on DRBD by the environment. E.g. disk broke or network broke down. 1021 * But those hard state transitions are still not allowed to do everything. 1022 * @ns: new state. 1023 * @os: old state. 1024 */ 1025 static enum drbd_state_rv 1026 is_valid_transition(union drbd_state os, union drbd_state ns) 1027 { 1028 enum drbd_state_rv rv; 1029 1030 rv = is_valid_conn_transition(os.conn, ns.conn); 1031 1032 /* we cannot fail (again) if we already detached */ 1033 if (ns.disk == D_FAILED && os.disk == D_DISKLESS) 1034 rv = SS_IS_DISKLESS; 1035 1036 return rv; 1037 } 1038 1039 static void print_sanitize_warnings(struct drbd_device *device, enum sanitize_state_warnings warn) 1040 { 1041 static const char *msg_table[] = { 1042 [NO_WARNING] = "", 1043 [ABORTED_ONLINE_VERIFY] = "Online-verify aborted.", 1044 [ABORTED_RESYNC] = "Resync aborted.", 1045 [CONNECTION_LOST_NEGOTIATING] = "Connection lost while negotiating, no data!", 1046 [IMPLICITLY_UPGRADED_DISK] = "Implicitly upgraded disk", 1047 [IMPLICITLY_UPGRADED_PDSK] = "Implicitly upgraded pdsk", 1048 }; 1049 1050 if (warn != NO_WARNING) 1051 drbd_warn(device, "%s\n", msg_table[warn]); 1052 } 1053 1054 /** 1055 * sanitize_state() - Resolves implicitly necessary additional changes to a state transition 1056 * @device: DRBD device. 1057 * @os: old state. 1058 * @ns: new state. 1059 * @warn_sync_abort: 1060 * 1061 * When we loose connection, we have to set the state of the peers disk (pdsk) 1062 * to D_UNKNOWN. This rule and many more along those lines are in this function. 1063 */ 1064 static union drbd_state sanitize_state(struct drbd_device *device, union drbd_state os, 1065 union drbd_state ns, enum sanitize_state_warnings *warn) 1066 { 1067 enum drbd_fencing_p fp; 1068 enum drbd_disk_state disk_min, disk_max, pdsk_min, pdsk_max; 1069 1070 if (warn) 1071 *warn = NO_WARNING; 1072 1073 fp = FP_DONT_CARE; 1074 if (get_ldev(device)) { 1075 rcu_read_lock(); 1076 fp = rcu_dereference(device->ldev->disk_conf)->fencing; 1077 rcu_read_unlock(); 1078 put_ldev(device); 1079 } 1080 1081 /* Implications from connection to peer and peer_isp */ 1082 if (ns.conn < C_CONNECTED) { 1083 ns.peer_isp = 0; 1084 ns.peer = R_UNKNOWN; 1085 if (ns.pdsk > D_UNKNOWN || ns.pdsk < D_INCONSISTENT) 1086 ns.pdsk = D_UNKNOWN; 1087 } 1088 1089 /* Clear the aftr_isp when becoming unconfigured */ 1090 if (ns.conn == C_STANDALONE && ns.disk == D_DISKLESS && ns.role == R_SECONDARY) 1091 ns.aftr_isp = 0; 1092 1093 /* An implication of the disk states onto the connection state */ 1094 /* Abort resync if a disk fails/detaches */ 1095 if (ns.conn > C_CONNECTED && (ns.disk <= D_FAILED || ns.pdsk <= D_FAILED)) { 1096 if (warn) 1097 *warn = ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T ? 1098 ABORTED_ONLINE_VERIFY : ABORTED_RESYNC; 1099 ns.conn = C_CONNECTED; 1100 } 1101 1102 /* Connection breaks down before we finished "Negotiating" */ 1103 if (ns.conn < C_CONNECTED && ns.disk == D_NEGOTIATING && 1104 get_ldev_if_state(device, D_NEGOTIATING)) { 1105 if (device->ed_uuid == device->ldev->md.uuid[UI_CURRENT]) { 1106 ns.disk = device->new_state_tmp.disk; 1107 ns.pdsk = device->new_state_tmp.pdsk; 1108 } else { 1109 if (warn) 1110 *warn = CONNECTION_LOST_NEGOTIATING; 1111 ns.disk = D_DISKLESS; 1112 ns.pdsk = D_UNKNOWN; 1113 } 1114 put_ldev(device); 1115 } 1116 1117 /* D_CONSISTENT and D_OUTDATED vanish when we get connected */ 1118 if (ns.conn >= C_CONNECTED && ns.conn < C_AHEAD) { 1119 if (ns.disk == D_CONSISTENT || ns.disk == D_OUTDATED) 1120 ns.disk = D_UP_TO_DATE; 1121 if (ns.pdsk == D_CONSISTENT || ns.pdsk == D_OUTDATED) 1122 ns.pdsk = D_UP_TO_DATE; 1123 } 1124 1125 /* Implications of the connection state on the disk states */ 1126 disk_min = D_DISKLESS; 1127 disk_max = D_UP_TO_DATE; 1128 pdsk_min = D_INCONSISTENT; 1129 pdsk_max = D_UNKNOWN; 1130 switch ((enum drbd_conns)ns.conn) { 1131 case C_WF_BITMAP_T: 1132 case C_PAUSED_SYNC_T: 1133 case C_STARTING_SYNC_T: 1134 case C_WF_SYNC_UUID: 1135 case C_BEHIND: 1136 disk_min = D_INCONSISTENT; 1137 disk_max = D_OUTDATED; 1138 pdsk_min = D_UP_TO_DATE; 1139 pdsk_max = D_UP_TO_DATE; 1140 break; 1141 case C_VERIFY_S: 1142 case C_VERIFY_T: 1143 disk_min = D_UP_TO_DATE; 1144 disk_max = D_UP_TO_DATE; 1145 pdsk_min = D_UP_TO_DATE; 1146 pdsk_max = D_UP_TO_DATE; 1147 break; 1148 case C_CONNECTED: 1149 disk_min = D_DISKLESS; 1150 disk_max = D_UP_TO_DATE; 1151 pdsk_min = D_DISKLESS; 1152 pdsk_max = D_UP_TO_DATE; 1153 break; 1154 case C_WF_BITMAP_S: 1155 case C_PAUSED_SYNC_S: 1156 case C_STARTING_SYNC_S: 1157 case C_AHEAD: 1158 disk_min = D_UP_TO_DATE; 1159 disk_max = D_UP_TO_DATE; 1160 pdsk_min = D_INCONSISTENT; 1161 pdsk_max = D_CONSISTENT; /* D_OUTDATED would be nice. But explicit outdate necessary*/ 1162 break; 1163 case C_SYNC_TARGET: 1164 disk_min = D_INCONSISTENT; 1165 disk_max = D_INCONSISTENT; 1166 pdsk_min = D_UP_TO_DATE; 1167 pdsk_max = D_UP_TO_DATE; 1168 break; 1169 case C_SYNC_SOURCE: 1170 disk_min = D_UP_TO_DATE; 1171 disk_max = D_UP_TO_DATE; 1172 pdsk_min = D_INCONSISTENT; 1173 pdsk_max = D_INCONSISTENT; 1174 break; 1175 case C_STANDALONE: 1176 case C_DISCONNECTING: 1177 case C_UNCONNECTED: 1178 case C_TIMEOUT: 1179 case C_BROKEN_PIPE: 1180 case C_NETWORK_FAILURE: 1181 case C_PROTOCOL_ERROR: 1182 case C_TEAR_DOWN: 1183 case C_WF_CONNECTION: 1184 case C_WF_REPORT_PARAMS: 1185 case C_MASK: 1186 break; 1187 } 1188 if (ns.disk > disk_max) 1189 ns.disk = disk_max; 1190 1191 if (ns.disk < disk_min) { 1192 if (warn) 1193 *warn = IMPLICITLY_UPGRADED_DISK; 1194 ns.disk = disk_min; 1195 } 1196 if (ns.pdsk > pdsk_max) 1197 ns.pdsk = pdsk_max; 1198 1199 if (ns.pdsk < pdsk_min) { 1200 if (warn) 1201 *warn = IMPLICITLY_UPGRADED_PDSK; 1202 ns.pdsk = pdsk_min; 1203 } 1204 1205 if (fp == FP_STONITH && 1206 (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk > D_OUTDATED) && 1207 !(os.role == R_PRIMARY && os.conn < C_CONNECTED && os.pdsk > D_OUTDATED)) 1208 ns.susp_fen = 1; /* Suspend IO while fence-peer handler runs (peer lost) */ 1209 1210 if (device->resource->res_opts.on_no_data == OND_SUSPEND_IO && 1211 (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE) && 1212 !(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE)) 1213 ns.susp_nod = 1; /* Suspend IO while no data available (no accessible data available) */ 1214 1215 if (ns.aftr_isp || ns.peer_isp || ns.user_isp) { 1216 if (ns.conn == C_SYNC_SOURCE) 1217 ns.conn = C_PAUSED_SYNC_S; 1218 if (ns.conn == C_SYNC_TARGET) 1219 ns.conn = C_PAUSED_SYNC_T; 1220 } else { 1221 if (ns.conn == C_PAUSED_SYNC_S) 1222 ns.conn = C_SYNC_SOURCE; 1223 if (ns.conn == C_PAUSED_SYNC_T) 1224 ns.conn = C_SYNC_TARGET; 1225 } 1226 1227 return ns; 1228 } 1229 1230 void drbd_resume_al(struct drbd_device *device) 1231 { 1232 if (test_and_clear_bit(AL_SUSPENDED, &device->flags)) 1233 drbd_info(device, "Resumed AL updates\n"); 1234 } 1235 1236 /* helper for _drbd_set_state */ 1237 static void set_ov_position(struct drbd_device *device, enum drbd_conns cs) 1238 { 1239 if (first_peer_device(device)->connection->agreed_pro_version < 90) 1240 device->ov_start_sector = 0; 1241 device->rs_total = drbd_bm_bits(device); 1242 device->ov_position = 0; 1243 if (cs == C_VERIFY_T) { 1244 /* starting online verify from an arbitrary position 1245 * does not fit well into the existing protocol. 1246 * on C_VERIFY_T, we initialize ov_left and friends 1247 * implicitly in receive_DataRequest once the 1248 * first P_OV_REQUEST is received */ 1249 device->ov_start_sector = ~(sector_t)0; 1250 } else { 1251 unsigned long bit = BM_SECT_TO_BIT(device->ov_start_sector); 1252 if (bit >= device->rs_total) { 1253 device->ov_start_sector = 1254 BM_BIT_TO_SECT(device->rs_total - 1); 1255 device->rs_total = 1; 1256 } else 1257 device->rs_total -= bit; 1258 device->ov_position = device->ov_start_sector; 1259 } 1260 device->ov_left = device->rs_total; 1261 } 1262 1263 /** 1264 * _drbd_set_state() - Set a new DRBD state 1265 * @device: DRBD device. 1266 * @ns: new state. 1267 * @flags: Flags 1268 * @done: Optional completion, that will get completed after the after_state_ch() finished 1269 * 1270 * Caller needs to hold req_lock. Do not call directly. 1271 */ 1272 enum drbd_state_rv 1273 _drbd_set_state(struct drbd_device *device, union drbd_state ns, 1274 enum chg_state_flags flags, struct completion *done) 1275 { 1276 struct drbd_peer_device *peer_device = first_peer_device(device); 1277 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; 1278 union drbd_state os; 1279 enum drbd_state_rv rv = SS_SUCCESS; 1280 enum sanitize_state_warnings ssw; 1281 struct after_state_chg_work *ascw; 1282 struct drbd_state_change *state_change; 1283 1284 os = drbd_read_state(device); 1285 1286 ns = sanitize_state(device, os, ns, &ssw); 1287 if (ns.i == os.i) 1288 return SS_NOTHING_TO_DO; 1289 1290 rv = is_valid_transition(os, ns); 1291 if (rv < SS_SUCCESS) 1292 return rv; 1293 1294 if (!(flags & CS_HARD)) { 1295 /* pre-state-change checks ; only look at ns */ 1296 /* See drbd_state_sw_errors in drbd_strings.c */ 1297 1298 rv = is_valid_state(device, ns); 1299 if (rv < SS_SUCCESS) { 1300 /* If the old state was illegal as well, then let 1301 this happen...*/ 1302 1303 if (is_valid_state(device, os) == rv) 1304 rv = is_valid_soft_transition(os, ns, connection); 1305 } else 1306 rv = is_valid_soft_transition(os, ns, connection); 1307 } 1308 1309 if (rv < SS_SUCCESS) { 1310 if (flags & CS_VERBOSE) 1311 print_st_err(device, os, ns, rv); 1312 return rv; 1313 } 1314 1315 print_sanitize_warnings(device, ssw); 1316 1317 drbd_pr_state_change(device, os, ns, flags); 1318 1319 /* Display changes to the susp* flags that where caused by the call to 1320 sanitize_state(). Only display it here if we where not called from 1321 _conn_request_state() */ 1322 if (!(flags & CS_DC_SUSP)) 1323 conn_pr_state_change(connection, os, ns, 1324 (flags & ~CS_DC_MASK) | CS_DC_SUSP); 1325 1326 /* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference 1327 * on the ldev here, to be sure the transition -> D_DISKLESS resp. 1328 * drbd_ldev_destroy() won't happen before our corresponding 1329 * after_state_ch works run, where we put_ldev again. */ 1330 if ((os.disk != D_FAILED && ns.disk == D_FAILED) || 1331 (os.disk != D_DISKLESS && ns.disk == D_DISKLESS)) 1332 atomic_inc(&device->local_cnt); 1333 1334 if (!is_sync_state(os.conn) && is_sync_state(ns.conn)) 1335 clear_bit(RS_DONE, &device->flags); 1336 1337 /* FIXME: Have any flags been set earlier in this function already? */ 1338 state_change = remember_old_state(device->resource, GFP_ATOMIC); 1339 1340 /* changes to local_cnt and device flags should be visible before 1341 * changes to state, which again should be visible before anything else 1342 * depending on that change happens. */ 1343 smp_wmb(); 1344 device->state.i = ns.i; 1345 device->resource->susp = ns.susp; 1346 device->resource->susp_nod = ns.susp_nod; 1347 device->resource->susp_fen = ns.susp_fen; 1348 smp_wmb(); 1349 1350 remember_new_state(state_change); 1351 1352 /* put replicated vs not-replicated requests in seperate epochs */ 1353 if (drbd_should_do_remote((union drbd_dev_state)os.i) != 1354 drbd_should_do_remote((union drbd_dev_state)ns.i)) 1355 start_new_tl_epoch(connection); 1356 1357 if (os.disk == D_ATTACHING && ns.disk >= D_NEGOTIATING) 1358 drbd_print_uuids(device, "attached to UUIDs"); 1359 1360 /* Wake up role changes, that were delayed because of connection establishing */ 1361 if (os.conn == C_WF_REPORT_PARAMS && ns.conn != C_WF_REPORT_PARAMS && 1362 no_peer_wf_report_params(connection)) { 1363 clear_bit(STATE_SENT, &connection->flags); 1364 wake_up_all_devices(connection); 1365 } 1366 1367 wake_up(&device->misc_wait); 1368 wake_up(&device->state_wait); 1369 wake_up(&connection->ping_wait); 1370 1371 /* Aborted verify run, or we reached the stop sector. 1372 * Log the last position, unless end-of-device. */ 1373 if ((os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) && 1374 ns.conn <= C_CONNECTED) { 1375 device->ov_start_sector = 1376 BM_BIT_TO_SECT(drbd_bm_bits(device) - device->ov_left); 1377 if (device->ov_left) 1378 drbd_info(device, "Online Verify reached sector %llu\n", 1379 (unsigned long long)device->ov_start_sector); 1380 } 1381 1382 if ((os.conn == C_PAUSED_SYNC_T || os.conn == C_PAUSED_SYNC_S) && 1383 (ns.conn == C_SYNC_TARGET || ns.conn == C_SYNC_SOURCE)) { 1384 drbd_info(device, "Syncer continues.\n"); 1385 device->rs_paused += (long)jiffies 1386 -(long)device->rs_mark_time[device->rs_last_mark]; 1387 if (ns.conn == C_SYNC_TARGET) 1388 mod_timer(&device->resync_timer, jiffies); 1389 } 1390 1391 if ((os.conn == C_SYNC_TARGET || os.conn == C_SYNC_SOURCE) && 1392 (ns.conn == C_PAUSED_SYNC_T || ns.conn == C_PAUSED_SYNC_S)) { 1393 drbd_info(device, "Resync suspended\n"); 1394 device->rs_mark_time[device->rs_last_mark] = jiffies; 1395 } 1396 1397 if (os.conn == C_CONNECTED && 1398 (ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T)) { 1399 unsigned long now = jiffies; 1400 int i; 1401 1402 set_ov_position(device, ns.conn); 1403 device->rs_start = now; 1404 device->rs_last_sect_ev = 0; 1405 device->ov_last_oos_size = 0; 1406 device->ov_last_oos_start = 0; 1407 1408 for (i = 0; i < DRBD_SYNC_MARKS; i++) { 1409 device->rs_mark_left[i] = device->ov_left; 1410 device->rs_mark_time[i] = now; 1411 } 1412 1413 drbd_rs_controller_reset(device); 1414 1415 if (ns.conn == C_VERIFY_S) { 1416 drbd_info(device, "Starting Online Verify from sector %llu\n", 1417 (unsigned long long)device->ov_position); 1418 mod_timer(&device->resync_timer, jiffies); 1419 } 1420 } 1421 1422 if (get_ldev(device)) { 1423 u32 mdf = device->ldev->md.flags & ~(MDF_CONSISTENT|MDF_PRIMARY_IND| 1424 MDF_CONNECTED_IND|MDF_WAS_UP_TO_DATE| 1425 MDF_PEER_OUT_DATED|MDF_CRASHED_PRIMARY); 1426 1427 mdf &= ~MDF_AL_CLEAN; 1428 if (test_bit(CRASHED_PRIMARY, &device->flags)) 1429 mdf |= MDF_CRASHED_PRIMARY; 1430 if (device->state.role == R_PRIMARY || 1431 (device->state.pdsk < D_INCONSISTENT && device->state.peer == R_PRIMARY)) 1432 mdf |= MDF_PRIMARY_IND; 1433 if (device->state.conn > C_WF_REPORT_PARAMS) 1434 mdf |= MDF_CONNECTED_IND; 1435 if (device->state.disk > D_INCONSISTENT) 1436 mdf |= MDF_CONSISTENT; 1437 if (device->state.disk > D_OUTDATED) 1438 mdf |= MDF_WAS_UP_TO_DATE; 1439 if (device->state.pdsk <= D_OUTDATED && device->state.pdsk >= D_INCONSISTENT) 1440 mdf |= MDF_PEER_OUT_DATED; 1441 if (mdf != device->ldev->md.flags) { 1442 device->ldev->md.flags = mdf; 1443 drbd_md_mark_dirty(device); 1444 } 1445 if (os.disk < D_CONSISTENT && ns.disk >= D_CONSISTENT) 1446 drbd_set_ed_uuid(device, device->ldev->md.uuid[UI_CURRENT]); 1447 put_ldev(device); 1448 } 1449 1450 /* Peer was forced D_UP_TO_DATE & R_PRIMARY, consider to resync */ 1451 if (os.disk == D_INCONSISTENT && os.pdsk == D_INCONSISTENT && 1452 os.peer == R_SECONDARY && ns.peer == R_PRIMARY) 1453 set_bit(CONSIDER_RESYNC, &device->flags); 1454 1455 /* Receiver should clean up itself */ 1456 if (os.conn != C_DISCONNECTING && ns.conn == C_DISCONNECTING) 1457 drbd_thread_stop_nowait(&connection->receiver); 1458 1459 /* Now the receiver finished cleaning up itself, it should die */ 1460 if (os.conn != C_STANDALONE && ns.conn == C_STANDALONE) 1461 drbd_thread_stop_nowait(&connection->receiver); 1462 1463 /* Upon network failure, we need to restart the receiver. */ 1464 if (os.conn > C_WF_CONNECTION && 1465 ns.conn <= C_TEAR_DOWN && ns.conn >= C_TIMEOUT) 1466 drbd_thread_restart_nowait(&connection->receiver); 1467 1468 /* Resume AL writing if we get a connection */ 1469 if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED) { 1470 drbd_resume_al(device); 1471 connection->connect_cnt++; 1472 } 1473 1474 /* remember last attach time so request_timer_fn() won't 1475 * kill newly established sessions while we are still trying to thaw 1476 * previously frozen IO */ 1477 if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) && 1478 ns.disk > D_NEGOTIATING) 1479 device->last_reattach_jif = jiffies; 1480 1481 ascw = kmalloc(sizeof(*ascw), GFP_ATOMIC); 1482 if (ascw) { 1483 ascw->os = os; 1484 ascw->ns = ns; 1485 ascw->flags = flags; 1486 ascw->w.cb = w_after_state_ch; 1487 ascw->device = device; 1488 ascw->done = done; 1489 ascw->state_change = state_change; 1490 drbd_queue_work(&connection->sender_work, 1491 &ascw->w); 1492 } else { 1493 drbd_err(device, "Could not kmalloc an ascw\n"); 1494 } 1495 1496 return rv; 1497 } 1498 1499 static int w_after_state_ch(struct drbd_work *w, int unused) 1500 { 1501 struct after_state_chg_work *ascw = 1502 container_of(w, struct after_state_chg_work, w); 1503 struct drbd_device *device = ascw->device; 1504 1505 after_state_ch(device, ascw->os, ascw->ns, ascw->flags, ascw->state_change); 1506 forget_state_change(ascw->state_change); 1507 if (ascw->flags & CS_WAIT_COMPLETE) 1508 complete(ascw->done); 1509 kfree(ascw); 1510 1511 return 0; 1512 } 1513 1514 static void abw_start_sync(struct drbd_device *device, int rv) 1515 { 1516 if (rv) { 1517 drbd_err(device, "Writing the bitmap failed not starting resync.\n"); 1518 _drbd_request_state(device, NS(conn, C_CONNECTED), CS_VERBOSE); 1519 return; 1520 } 1521 1522 switch (device->state.conn) { 1523 case C_STARTING_SYNC_T: 1524 _drbd_request_state(device, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE); 1525 break; 1526 case C_STARTING_SYNC_S: 1527 drbd_start_resync(device, C_SYNC_SOURCE); 1528 break; 1529 } 1530 } 1531 1532 int drbd_bitmap_io_from_worker(struct drbd_device *device, 1533 int (*io_fn)(struct drbd_device *), 1534 char *why, enum bm_flag flags) 1535 { 1536 int rv; 1537 1538 D_ASSERT(device, current == first_peer_device(device)->connection->worker.task); 1539 1540 /* open coded non-blocking drbd_suspend_io(device); */ 1541 atomic_inc(&device->suspend_cnt); 1542 1543 drbd_bm_lock(device, why, flags); 1544 rv = io_fn(device); 1545 drbd_bm_unlock(device); 1546 1547 drbd_resume_io(device); 1548 1549 return rv; 1550 } 1551 1552 void notify_resource_state_change(struct sk_buff *skb, 1553 unsigned int seq, 1554 struct drbd_resource_state_change *resource_state_change, 1555 enum drbd_notification_type type) 1556 { 1557 struct drbd_resource *resource = resource_state_change->resource; 1558 struct resource_info resource_info = { 1559 .res_role = resource_state_change->role[NEW], 1560 .res_susp = resource_state_change->susp[NEW], 1561 .res_susp_nod = resource_state_change->susp_nod[NEW], 1562 .res_susp_fen = resource_state_change->susp_fen[NEW], 1563 }; 1564 1565 notify_resource_state(skb, seq, resource, &resource_info, type); 1566 } 1567 1568 void notify_connection_state_change(struct sk_buff *skb, 1569 unsigned int seq, 1570 struct drbd_connection_state_change *connection_state_change, 1571 enum drbd_notification_type type) 1572 { 1573 struct drbd_connection *connection = connection_state_change->connection; 1574 struct connection_info connection_info = { 1575 .conn_connection_state = connection_state_change->cstate[NEW], 1576 .conn_role = connection_state_change->peer_role[NEW], 1577 }; 1578 1579 notify_connection_state(skb, seq, connection, &connection_info, type); 1580 } 1581 1582 void notify_device_state_change(struct sk_buff *skb, 1583 unsigned int seq, 1584 struct drbd_device_state_change *device_state_change, 1585 enum drbd_notification_type type) 1586 { 1587 struct drbd_device *device = device_state_change->device; 1588 struct device_info device_info = { 1589 .dev_disk_state = device_state_change->disk_state[NEW], 1590 }; 1591 1592 notify_device_state(skb, seq, device, &device_info, type); 1593 } 1594 1595 void notify_peer_device_state_change(struct sk_buff *skb, 1596 unsigned int seq, 1597 struct drbd_peer_device_state_change *p, 1598 enum drbd_notification_type type) 1599 { 1600 struct drbd_peer_device *peer_device = p->peer_device; 1601 struct peer_device_info peer_device_info = { 1602 .peer_repl_state = p->repl_state[NEW], 1603 .peer_disk_state = p->disk_state[NEW], 1604 .peer_resync_susp_user = p->resync_susp_user[NEW], 1605 .peer_resync_susp_peer = p->resync_susp_peer[NEW], 1606 .peer_resync_susp_dependency = p->resync_susp_dependency[NEW], 1607 }; 1608 1609 notify_peer_device_state(skb, seq, peer_device, &peer_device_info, type); 1610 } 1611 1612 static void broadcast_state_change(struct drbd_state_change *state_change) 1613 { 1614 struct drbd_resource_state_change *resource_state_change = &state_change->resource[0]; 1615 bool resource_state_has_changed; 1616 unsigned int n_device, n_connection, n_peer_device, n_peer_devices; 1617 void (*last_func)(struct sk_buff *, unsigned int, void *, 1618 enum drbd_notification_type) = NULL; 1619 void *uninitialized_var(last_arg); 1620 1621 #define HAS_CHANGED(state) ((state)[OLD] != (state)[NEW]) 1622 #define FINAL_STATE_CHANGE(type) \ 1623 ({ if (last_func) \ 1624 last_func(NULL, 0, last_arg, type); \ 1625 }) 1626 #define REMEMBER_STATE_CHANGE(func, arg, type) \ 1627 ({ FINAL_STATE_CHANGE(type | NOTIFY_CONTINUES); \ 1628 last_func = (typeof(last_func))func; \ 1629 last_arg = arg; \ 1630 }) 1631 1632 mutex_lock(¬ification_mutex); 1633 1634 resource_state_has_changed = 1635 HAS_CHANGED(resource_state_change->role) || 1636 HAS_CHANGED(resource_state_change->susp) || 1637 HAS_CHANGED(resource_state_change->susp_nod) || 1638 HAS_CHANGED(resource_state_change->susp_fen); 1639 1640 if (resource_state_has_changed) 1641 REMEMBER_STATE_CHANGE(notify_resource_state_change, 1642 resource_state_change, NOTIFY_CHANGE); 1643 1644 for (n_connection = 0; n_connection < state_change->n_connections; n_connection++) { 1645 struct drbd_connection_state_change *connection_state_change = 1646 &state_change->connections[n_connection]; 1647 1648 if (HAS_CHANGED(connection_state_change->peer_role) || 1649 HAS_CHANGED(connection_state_change->cstate)) 1650 REMEMBER_STATE_CHANGE(notify_connection_state_change, 1651 connection_state_change, NOTIFY_CHANGE); 1652 } 1653 1654 for (n_device = 0; n_device < state_change->n_devices; n_device++) { 1655 struct drbd_device_state_change *device_state_change = 1656 &state_change->devices[n_device]; 1657 1658 if (HAS_CHANGED(device_state_change->disk_state)) 1659 REMEMBER_STATE_CHANGE(notify_device_state_change, 1660 device_state_change, NOTIFY_CHANGE); 1661 } 1662 1663 n_peer_devices = state_change->n_devices * state_change->n_connections; 1664 for (n_peer_device = 0; n_peer_device < n_peer_devices; n_peer_device++) { 1665 struct drbd_peer_device_state_change *p = 1666 &state_change->peer_devices[n_peer_device]; 1667 1668 if (HAS_CHANGED(p->disk_state) || 1669 HAS_CHANGED(p->repl_state) || 1670 HAS_CHANGED(p->resync_susp_user) || 1671 HAS_CHANGED(p->resync_susp_peer) || 1672 HAS_CHANGED(p->resync_susp_dependency)) 1673 REMEMBER_STATE_CHANGE(notify_peer_device_state_change, 1674 p, NOTIFY_CHANGE); 1675 } 1676 1677 FINAL_STATE_CHANGE(NOTIFY_CHANGE); 1678 mutex_unlock(¬ification_mutex); 1679 1680 #undef HAS_CHANGED 1681 #undef FINAL_STATE_CHANGE 1682 #undef REMEMBER_STATE_CHANGE 1683 } 1684 1685 /* takes old and new peer disk state */ 1686 static bool lost_contact_to_peer_data(enum drbd_disk_state os, enum drbd_disk_state ns) 1687 { 1688 if ((os >= D_INCONSISTENT && os != D_UNKNOWN && os != D_OUTDATED) 1689 && (ns < D_INCONSISTENT || ns == D_UNKNOWN || ns == D_OUTDATED)) 1690 return true; 1691 1692 /* Scenario, starting with normal operation 1693 * Connected Primary/Secondary UpToDate/UpToDate 1694 * NetworkFailure Primary/Unknown UpToDate/DUnknown (frozen) 1695 * ... 1696 * Connected Primary/Secondary UpToDate/Diskless (resumed; needs to bump uuid!) 1697 */ 1698 if (os == D_UNKNOWN 1699 && (ns == D_DISKLESS || ns == D_FAILED || ns == D_OUTDATED)) 1700 return true; 1701 1702 return false; 1703 } 1704 1705 /** 1706 * after_state_ch() - Perform after state change actions that may sleep 1707 * @device: DRBD device. 1708 * @os: old state. 1709 * @ns: new state. 1710 * @flags: Flags 1711 */ 1712 static void after_state_ch(struct drbd_device *device, union drbd_state os, 1713 union drbd_state ns, enum chg_state_flags flags, 1714 struct drbd_state_change *state_change) 1715 { 1716 struct drbd_resource *resource = device->resource; 1717 struct drbd_peer_device *peer_device = first_peer_device(device); 1718 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; 1719 struct sib_info sib; 1720 1721 broadcast_state_change(state_change); 1722 1723 sib.sib_reason = SIB_STATE_CHANGE; 1724 sib.os = os; 1725 sib.ns = ns; 1726 1727 if ((os.disk != D_UP_TO_DATE || os.pdsk != D_UP_TO_DATE) 1728 && (ns.disk == D_UP_TO_DATE && ns.pdsk == D_UP_TO_DATE)) { 1729 clear_bit(CRASHED_PRIMARY, &device->flags); 1730 if (device->p_uuid) 1731 device->p_uuid[UI_FLAGS] &= ~((u64)2); 1732 } 1733 1734 /* Inform userspace about the change... */ 1735 drbd_bcast_event(device, &sib); 1736 1737 if (!(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE) && 1738 (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE)) 1739 drbd_khelper(device, "pri-on-incon-degr"); 1740 1741 /* Here we have the actions that are performed after a 1742 state change. This function might sleep */ 1743 1744 if (ns.susp_nod) { 1745 enum drbd_req_event what = NOTHING; 1746 1747 spin_lock_irq(&device->resource->req_lock); 1748 if (os.conn < C_CONNECTED && conn_lowest_conn(connection) >= C_CONNECTED) 1749 what = RESEND; 1750 1751 if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) && 1752 conn_lowest_disk(connection) == D_UP_TO_DATE) 1753 what = RESTART_FROZEN_DISK_IO; 1754 1755 if (resource->susp_nod && what != NOTHING) { 1756 _tl_restart(connection, what); 1757 _conn_request_state(connection, 1758 (union drbd_state) { { .susp_nod = 1 } }, 1759 (union drbd_state) { { .susp_nod = 0 } }, 1760 CS_VERBOSE); 1761 } 1762 spin_unlock_irq(&device->resource->req_lock); 1763 } 1764 1765 if (ns.susp_fen) { 1766 spin_lock_irq(&device->resource->req_lock); 1767 if (resource->susp_fen && conn_lowest_conn(connection) >= C_CONNECTED) { 1768 /* case2: The connection was established again: */ 1769 struct drbd_peer_device *peer_device; 1770 int vnr; 1771 1772 rcu_read_lock(); 1773 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 1774 clear_bit(NEW_CUR_UUID, &peer_device->device->flags); 1775 rcu_read_unlock(); 1776 1777 /* We should actively create a new uuid, _before_ 1778 * we resume/resent, if the peer is diskless 1779 * (recovery from a multiple error scenario). 1780 * Currently, this happens with a slight delay 1781 * below when checking lost_contact_to_peer_data() ... 1782 */ 1783 _tl_restart(connection, RESEND); 1784 _conn_request_state(connection, 1785 (union drbd_state) { { .susp_fen = 1 } }, 1786 (union drbd_state) { { .susp_fen = 0 } }, 1787 CS_VERBOSE); 1788 } 1789 spin_unlock_irq(&device->resource->req_lock); 1790 } 1791 1792 /* Became sync source. With protocol >= 96, we still need to send out 1793 * the sync uuid now. Need to do that before any drbd_send_state, or 1794 * the other side may go "paused sync" before receiving the sync uuids, 1795 * which is unexpected. */ 1796 if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) && 1797 (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) && 1798 connection->agreed_pro_version >= 96 && get_ldev(device)) { 1799 drbd_gen_and_send_sync_uuid(peer_device); 1800 put_ldev(device); 1801 } 1802 1803 /* Do not change the order of the if above and the two below... */ 1804 if (os.pdsk == D_DISKLESS && 1805 ns.pdsk > D_DISKLESS && ns.pdsk != D_UNKNOWN) { /* attach on the peer */ 1806 /* we probably will start a resync soon. 1807 * make sure those things are properly reset. */ 1808 device->rs_total = 0; 1809 device->rs_failed = 0; 1810 atomic_set(&device->rs_pending_cnt, 0); 1811 drbd_rs_cancel_all(device); 1812 1813 drbd_send_uuids(peer_device); 1814 drbd_send_state(peer_device, ns); 1815 } 1816 /* No point in queuing send_bitmap if we don't have a connection 1817 * anymore, so check also the _current_ state, not only the new state 1818 * at the time this work was queued. */ 1819 if (os.conn != C_WF_BITMAP_S && ns.conn == C_WF_BITMAP_S && 1820 device->state.conn == C_WF_BITMAP_S) 1821 drbd_queue_bitmap_io(device, &drbd_send_bitmap, NULL, 1822 "send_bitmap (WFBitMapS)", 1823 BM_LOCKED_TEST_ALLOWED); 1824 1825 /* Lost contact to peer's copy of the data */ 1826 if (lost_contact_to_peer_data(os.pdsk, ns.pdsk)) { 1827 if (get_ldev(device)) { 1828 if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) && 1829 device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) { 1830 if (drbd_suspended(device)) { 1831 set_bit(NEW_CUR_UUID, &device->flags); 1832 } else { 1833 drbd_uuid_new_current(device); 1834 drbd_send_uuids(peer_device); 1835 } 1836 } 1837 put_ldev(device); 1838 } 1839 } 1840 1841 if (ns.pdsk < D_INCONSISTENT && get_ldev(device)) { 1842 if (os.peer != R_PRIMARY && ns.peer == R_PRIMARY && 1843 device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) { 1844 drbd_uuid_new_current(device); 1845 drbd_send_uuids(peer_device); 1846 } 1847 /* D_DISKLESS Peer becomes secondary */ 1848 if (os.peer == R_PRIMARY && ns.peer == R_SECONDARY) 1849 /* We may still be Primary ourselves. 1850 * No harm done if the bitmap still changes, 1851 * redirtied pages will follow later. */ 1852 drbd_bitmap_io_from_worker(device, &drbd_bm_write, 1853 "demote diskless peer", BM_LOCKED_SET_ALLOWED); 1854 put_ldev(device); 1855 } 1856 1857 /* Write out all changed bits on demote. 1858 * Though, no need to da that just yet 1859 * if there is a resync going on still */ 1860 if (os.role == R_PRIMARY && ns.role == R_SECONDARY && 1861 device->state.conn <= C_CONNECTED && get_ldev(device)) { 1862 /* No changes to the bitmap expected this time, so assert that, 1863 * even though no harm was done if it did change. */ 1864 drbd_bitmap_io_from_worker(device, &drbd_bm_write, 1865 "demote", BM_LOCKED_TEST_ALLOWED); 1866 put_ldev(device); 1867 } 1868 1869 /* Last part of the attaching process ... */ 1870 if (ns.conn >= C_CONNECTED && 1871 os.disk == D_ATTACHING && ns.disk == D_NEGOTIATING) { 1872 drbd_send_sizes(peer_device, 0, 0); /* to start sync... */ 1873 drbd_send_uuids(peer_device); 1874 drbd_send_state(peer_device, ns); 1875 } 1876 1877 /* We want to pause/continue resync, tell peer. */ 1878 if (ns.conn >= C_CONNECTED && 1879 ((os.aftr_isp != ns.aftr_isp) || 1880 (os.user_isp != ns.user_isp))) 1881 drbd_send_state(peer_device, ns); 1882 1883 /* In case one of the isp bits got set, suspend other devices. */ 1884 if ((!os.aftr_isp && !os.peer_isp && !os.user_isp) && 1885 (ns.aftr_isp || ns.peer_isp || ns.user_isp)) 1886 suspend_other_sg(device); 1887 1888 /* Make sure the peer gets informed about eventual state 1889 changes (ISP bits) while we were in WFReportParams. */ 1890 if (os.conn == C_WF_REPORT_PARAMS && ns.conn >= C_CONNECTED) 1891 drbd_send_state(peer_device, ns); 1892 1893 if (os.conn != C_AHEAD && ns.conn == C_AHEAD) 1894 drbd_send_state(peer_device, ns); 1895 1896 /* We are in the progress to start a full sync... */ 1897 if ((os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) || 1898 (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S)) 1899 /* no other bitmap changes expected during this phase */ 1900 drbd_queue_bitmap_io(device, 1901 &drbd_bmio_set_n_write, &abw_start_sync, 1902 "set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED); 1903 1904 /* first half of local IO error, failure to attach, 1905 * or administrative detach */ 1906 if (os.disk != D_FAILED && ns.disk == D_FAILED) { 1907 enum drbd_io_error_p eh = EP_PASS_ON; 1908 int was_io_error = 0; 1909 /* corresponding get_ldev was in _drbd_set_state, to serialize 1910 * our cleanup here with the transition to D_DISKLESS. 1911 * But is is still not save to dreference ldev here, since 1912 * we might come from an failed Attach before ldev was set. */ 1913 if (device->ldev) { 1914 rcu_read_lock(); 1915 eh = rcu_dereference(device->ldev->disk_conf)->on_io_error; 1916 rcu_read_unlock(); 1917 1918 was_io_error = test_and_clear_bit(WAS_IO_ERROR, &device->flags); 1919 1920 /* Intentionally call this handler first, before drbd_send_state(). 1921 * See: 2932204 drbd: call local-io-error handler early 1922 * People may chose to hard-reset the box from this handler. 1923 * It is useful if this looks like a "regular node crash". */ 1924 if (was_io_error && eh == EP_CALL_HELPER) 1925 drbd_khelper(device, "local-io-error"); 1926 1927 /* Immediately allow completion of all application IO, 1928 * that waits for completion from the local disk, 1929 * if this was a force-detach due to disk_timeout 1930 * or administrator request (drbdsetup detach --force). 1931 * Do NOT abort otherwise. 1932 * Aborting local requests may cause serious problems, 1933 * if requests are completed to upper layers already, 1934 * and then later the already submitted local bio completes. 1935 * This can cause DMA into former bio pages that meanwhile 1936 * have been re-used for other things. 1937 * So aborting local requests may cause crashes, 1938 * or even worse, silent data corruption. 1939 */ 1940 if (test_and_clear_bit(FORCE_DETACH, &device->flags)) 1941 tl_abort_disk_io(device); 1942 1943 /* current state still has to be D_FAILED, 1944 * there is only one way out: to D_DISKLESS, 1945 * and that may only happen after our put_ldev below. */ 1946 if (device->state.disk != D_FAILED) 1947 drbd_err(device, 1948 "ASSERT FAILED: disk is %s during detach\n", 1949 drbd_disk_str(device->state.disk)); 1950 1951 if (ns.conn >= C_CONNECTED) 1952 drbd_send_state(peer_device, ns); 1953 1954 drbd_rs_cancel_all(device); 1955 1956 /* In case we want to get something to stable storage still, 1957 * this may be the last chance. 1958 * Following put_ldev may transition to D_DISKLESS. */ 1959 drbd_md_sync(device); 1960 } 1961 put_ldev(device); 1962 } 1963 1964 /* second half of local IO error, failure to attach, 1965 * or administrative detach, 1966 * after local_cnt references have reached zero again */ 1967 if (os.disk != D_DISKLESS && ns.disk == D_DISKLESS) { 1968 /* We must still be diskless, 1969 * re-attach has to be serialized with this! */ 1970 if (device->state.disk != D_DISKLESS) 1971 drbd_err(device, 1972 "ASSERT FAILED: disk is %s while going diskless\n", 1973 drbd_disk_str(device->state.disk)); 1974 1975 if (ns.conn >= C_CONNECTED) 1976 drbd_send_state(peer_device, ns); 1977 /* corresponding get_ldev in __drbd_set_state 1978 * this may finally trigger drbd_ldev_destroy. */ 1979 put_ldev(device); 1980 } 1981 1982 /* Notify peer that I had a local IO error, and did not detached.. */ 1983 if (os.disk == D_UP_TO_DATE && ns.disk == D_INCONSISTENT && ns.conn >= C_CONNECTED) 1984 drbd_send_state(peer_device, ns); 1985 1986 /* Disks got bigger while they were detached */ 1987 if (ns.disk > D_NEGOTIATING && ns.pdsk > D_NEGOTIATING && 1988 test_and_clear_bit(RESYNC_AFTER_NEG, &device->flags)) { 1989 if (ns.conn == C_CONNECTED) 1990 resync_after_online_grow(device); 1991 } 1992 1993 /* A resync finished or aborted, wake paused devices... */ 1994 if ((os.conn > C_CONNECTED && ns.conn <= C_CONNECTED) || 1995 (os.peer_isp && !ns.peer_isp) || 1996 (os.user_isp && !ns.user_isp)) 1997 resume_next_sg(device); 1998 1999 /* sync target done with resync. Explicitly notify peer, even though 2000 * it should (at least for non-empty resyncs) already know itself. */ 2001 if (os.disk < D_UP_TO_DATE && os.conn >= C_SYNC_SOURCE && ns.conn == C_CONNECTED) 2002 drbd_send_state(peer_device, ns); 2003 2004 /* Verify finished, or reached stop sector. Peer did not know about 2005 * the stop sector, and we may even have changed the stop sector during 2006 * verify to interrupt/stop early. Send the new state. */ 2007 if (os.conn == C_VERIFY_S && ns.conn == C_CONNECTED 2008 && verify_can_do_stop_sector(device)) 2009 drbd_send_state(peer_device, ns); 2010 2011 /* This triggers bitmap writeout of potentially still unwritten pages 2012 * if the resync finished cleanly, or aborted because of peer disk 2013 * failure, or on transition from resync back to AHEAD/BEHIND. 2014 * 2015 * Connection loss is handled in drbd_disconnected() by the receiver. 2016 * 2017 * For resync aborted because of local disk failure, we cannot do 2018 * any bitmap writeout anymore. 2019 * 2020 * No harm done if some bits change during this phase. 2021 */ 2022 if ((os.conn > C_CONNECTED && os.conn < C_AHEAD) && 2023 (ns.conn == C_CONNECTED || ns.conn >= C_AHEAD) && get_ldev(device)) { 2024 drbd_queue_bitmap_io(device, &drbd_bm_write_copy_pages, NULL, 2025 "write from resync_finished", BM_LOCKED_CHANGE_ALLOWED); 2026 put_ldev(device); 2027 } 2028 2029 if (ns.disk == D_DISKLESS && 2030 ns.conn == C_STANDALONE && 2031 ns.role == R_SECONDARY) { 2032 if (os.aftr_isp != ns.aftr_isp) 2033 resume_next_sg(device); 2034 } 2035 2036 drbd_md_sync(device); 2037 } 2038 2039 struct after_conn_state_chg_work { 2040 struct drbd_work w; 2041 enum drbd_conns oc; 2042 union drbd_state ns_min; 2043 union drbd_state ns_max; /* new, max state, over all devices */ 2044 enum chg_state_flags flags; 2045 struct drbd_connection *connection; 2046 struct drbd_state_change *state_change; 2047 }; 2048 2049 static int w_after_conn_state_ch(struct drbd_work *w, int unused) 2050 { 2051 struct after_conn_state_chg_work *acscw = 2052 container_of(w, struct after_conn_state_chg_work, w); 2053 struct drbd_connection *connection = acscw->connection; 2054 enum drbd_conns oc = acscw->oc; 2055 union drbd_state ns_max = acscw->ns_max; 2056 struct drbd_peer_device *peer_device; 2057 int vnr; 2058 2059 broadcast_state_change(acscw->state_change); 2060 forget_state_change(acscw->state_change); 2061 kfree(acscw); 2062 2063 /* Upon network configuration, we need to start the receiver */ 2064 if (oc == C_STANDALONE && ns_max.conn == C_UNCONNECTED) 2065 drbd_thread_start(&connection->receiver); 2066 2067 if (oc == C_DISCONNECTING && ns_max.conn == C_STANDALONE) { 2068 struct net_conf *old_conf; 2069 2070 mutex_lock(¬ification_mutex); 2071 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) 2072 notify_peer_device_state(NULL, 0, peer_device, NULL, 2073 NOTIFY_DESTROY | NOTIFY_CONTINUES); 2074 notify_connection_state(NULL, 0, connection, NULL, NOTIFY_DESTROY); 2075 mutex_unlock(¬ification_mutex); 2076 2077 mutex_lock(&connection->resource->conf_update); 2078 old_conf = connection->net_conf; 2079 connection->my_addr_len = 0; 2080 connection->peer_addr_len = 0; 2081 RCU_INIT_POINTER(connection->net_conf, NULL); 2082 conn_free_crypto(connection); 2083 mutex_unlock(&connection->resource->conf_update); 2084 2085 synchronize_rcu(); 2086 kfree(old_conf); 2087 } 2088 2089 if (ns_max.susp_fen) { 2090 /* case1: The outdate peer handler is successful: */ 2091 if (ns_max.pdsk <= D_OUTDATED) { 2092 rcu_read_lock(); 2093 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2094 struct drbd_device *device = peer_device->device; 2095 if (test_bit(NEW_CUR_UUID, &device->flags)) { 2096 drbd_uuid_new_current(device); 2097 clear_bit(NEW_CUR_UUID, &device->flags); 2098 } 2099 } 2100 rcu_read_unlock(); 2101 spin_lock_irq(&connection->resource->req_lock); 2102 _tl_restart(connection, CONNECTION_LOST_WHILE_PENDING); 2103 _conn_request_state(connection, 2104 (union drbd_state) { { .susp_fen = 1 } }, 2105 (union drbd_state) { { .susp_fen = 0 } }, 2106 CS_VERBOSE); 2107 spin_unlock_irq(&connection->resource->req_lock); 2108 } 2109 } 2110 conn_md_sync(connection); 2111 kref_put(&connection->kref, drbd_destroy_connection); 2112 2113 return 0; 2114 } 2115 2116 static void conn_old_common_state(struct drbd_connection *connection, union drbd_state *pcs, enum chg_state_flags *pf) 2117 { 2118 enum chg_state_flags flags = ~0; 2119 struct drbd_peer_device *peer_device; 2120 int vnr, first_vol = 1; 2121 union drbd_dev_state os, cs = { 2122 { .role = R_SECONDARY, 2123 .peer = R_UNKNOWN, 2124 .conn = connection->cstate, 2125 .disk = D_DISKLESS, 2126 .pdsk = D_UNKNOWN, 2127 } }; 2128 2129 rcu_read_lock(); 2130 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2131 struct drbd_device *device = peer_device->device; 2132 os = device->state; 2133 2134 if (first_vol) { 2135 cs = os; 2136 first_vol = 0; 2137 continue; 2138 } 2139 2140 if (cs.role != os.role) 2141 flags &= ~CS_DC_ROLE; 2142 2143 if (cs.peer != os.peer) 2144 flags &= ~CS_DC_PEER; 2145 2146 if (cs.conn != os.conn) 2147 flags &= ~CS_DC_CONN; 2148 2149 if (cs.disk != os.disk) 2150 flags &= ~CS_DC_DISK; 2151 2152 if (cs.pdsk != os.pdsk) 2153 flags &= ~CS_DC_PDSK; 2154 } 2155 rcu_read_unlock(); 2156 2157 *pf |= CS_DC_MASK; 2158 *pf &= flags; 2159 (*pcs).i = cs.i; 2160 } 2161 2162 static enum drbd_state_rv 2163 conn_is_valid_transition(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2164 enum chg_state_flags flags) 2165 { 2166 enum drbd_state_rv rv = SS_SUCCESS; 2167 union drbd_state ns, os; 2168 struct drbd_peer_device *peer_device; 2169 int vnr; 2170 2171 rcu_read_lock(); 2172 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2173 struct drbd_device *device = peer_device->device; 2174 os = drbd_read_state(device); 2175 ns = sanitize_state(device, os, apply_mask_val(os, mask, val), NULL); 2176 2177 if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED) 2178 ns.disk = os.disk; 2179 2180 if (ns.i == os.i) 2181 continue; 2182 2183 rv = is_valid_transition(os, ns); 2184 2185 if (rv >= SS_SUCCESS && !(flags & CS_HARD)) { 2186 rv = is_valid_state(device, ns); 2187 if (rv < SS_SUCCESS) { 2188 if (is_valid_state(device, os) == rv) 2189 rv = is_valid_soft_transition(os, ns, connection); 2190 } else 2191 rv = is_valid_soft_transition(os, ns, connection); 2192 } 2193 2194 if (rv < SS_SUCCESS) { 2195 if (flags & CS_VERBOSE) 2196 print_st_err(device, os, ns, rv); 2197 break; 2198 } 2199 } 2200 rcu_read_unlock(); 2201 2202 return rv; 2203 } 2204 2205 static void 2206 conn_set_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2207 union drbd_state *pns_min, union drbd_state *pns_max, enum chg_state_flags flags) 2208 { 2209 union drbd_state ns, os, ns_max = { }; 2210 union drbd_state ns_min = { 2211 { .role = R_MASK, 2212 .peer = R_MASK, 2213 .conn = val.conn, 2214 .disk = D_MASK, 2215 .pdsk = D_MASK 2216 } }; 2217 struct drbd_peer_device *peer_device; 2218 enum drbd_state_rv rv; 2219 int vnr, number_of_volumes = 0; 2220 2221 if (mask.conn == C_MASK) { 2222 /* remember last connect time so request_timer_fn() won't 2223 * kill newly established sessions while we are still trying to thaw 2224 * previously frozen IO */ 2225 if (connection->cstate != C_WF_REPORT_PARAMS && val.conn == C_WF_REPORT_PARAMS) 2226 connection->last_reconnect_jif = jiffies; 2227 2228 connection->cstate = val.conn; 2229 } 2230 2231 rcu_read_lock(); 2232 idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2233 struct drbd_device *device = peer_device->device; 2234 number_of_volumes++; 2235 os = drbd_read_state(device); 2236 ns = apply_mask_val(os, mask, val); 2237 ns = sanitize_state(device, os, ns, NULL); 2238 2239 if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED) 2240 ns.disk = os.disk; 2241 2242 rv = _drbd_set_state(device, ns, flags, NULL); 2243 BUG_ON(rv < SS_SUCCESS); 2244 ns.i = device->state.i; 2245 ns_max.role = max_role(ns.role, ns_max.role); 2246 ns_max.peer = max_role(ns.peer, ns_max.peer); 2247 ns_max.conn = max_t(enum drbd_conns, ns.conn, ns_max.conn); 2248 ns_max.disk = max_t(enum drbd_disk_state, ns.disk, ns_max.disk); 2249 ns_max.pdsk = max_t(enum drbd_disk_state, ns.pdsk, ns_max.pdsk); 2250 2251 ns_min.role = min_role(ns.role, ns_min.role); 2252 ns_min.peer = min_role(ns.peer, ns_min.peer); 2253 ns_min.conn = min_t(enum drbd_conns, ns.conn, ns_min.conn); 2254 ns_min.disk = min_t(enum drbd_disk_state, ns.disk, ns_min.disk); 2255 ns_min.pdsk = min_t(enum drbd_disk_state, ns.pdsk, ns_min.pdsk); 2256 } 2257 rcu_read_unlock(); 2258 2259 if (number_of_volumes == 0) { 2260 ns_min = ns_max = (union drbd_state) { { 2261 .role = R_SECONDARY, 2262 .peer = R_UNKNOWN, 2263 .conn = val.conn, 2264 .disk = D_DISKLESS, 2265 .pdsk = D_UNKNOWN 2266 } }; 2267 } 2268 2269 ns_min.susp = ns_max.susp = connection->resource->susp; 2270 ns_min.susp_nod = ns_max.susp_nod = connection->resource->susp_nod; 2271 ns_min.susp_fen = ns_max.susp_fen = connection->resource->susp_fen; 2272 2273 *pns_min = ns_min; 2274 *pns_max = ns_max; 2275 } 2276 2277 static enum drbd_state_rv 2278 _conn_rq_cond(struct drbd_connection *connection, union drbd_state mask, union drbd_state val) 2279 { 2280 enum drbd_state_rv err, rv = SS_UNKNOWN_ERROR; /* continue waiting */; 2281 2282 if (test_and_clear_bit(CONN_WD_ST_CHG_OKAY, &connection->flags)) 2283 rv = SS_CW_SUCCESS; 2284 2285 if (test_and_clear_bit(CONN_WD_ST_CHG_FAIL, &connection->flags)) 2286 rv = SS_CW_FAILED_BY_PEER; 2287 2288 err = conn_is_valid_transition(connection, mask, val, 0); 2289 if (err == SS_SUCCESS && connection->cstate == C_WF_REPORT_PARAMS) 2290 return rv; 2291 2292 return err; 2293 } 2294 2295 enum drbd_state_rv 2296 _conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2297 enum chg_state_flags flags) 2298 { 2299 enum drbd_state_rv rv = SS_SUCCESS; 2300 struct after_conn_state_chg_work *acscw; 2301 enum drbd_conns oc = connection->cstate; 2302 union drbd_state ns_max, ns_min, os; 2303 bool have_mutex = false; 2304 struct drbd_state_change *state_change; 2305 2306 if (mask.conn) { 2307 rv = is_valid_conn_transition(oc, val.conn); 2308 if (rv < SS_SUCCESS) 2309 goto abort; 2310 } 2311 2312 rv = conn_is_valid_transition(connection, mask, val, flags); 2313 if (rv < SS_SUCCESS) 2314 goto abort; 2315 2316 if (oc == C_WF_REPORT_PARAMS && val.conn == C_DISCONNECTING && 2317 !(flags & (CS_LOCAL_ONLY | CS_HARD))) { 2318 2319 /* This will be a cluster-wide state change. 2320 * Need to give up the spinlock, grab the mutex, 2321 * then send the state change request, ... */ 2322 spin_unlock_irq(&connection->resource->req_lock); 2323 mutex_lock(&connection->cstate_mutex); 2324 have_mutex = true; 2325 2326 set_bit(CONN_WD_ST_CHG_REQ, &connection->flags); 2327 if (conn_send_state_req(connection, mask, val)) { 2328 /* sending failed. */ 2329 clear_bit(CONN_WD_ST_CHG_REQ, &connection->flags); 2330 rv = SS_CW_FAILED_BY_PEER; 2331 /* need to re-aquire the spin lock, though */ 2332 goto abort_unlocked; 2333 } 2334 2335 if (val.conn == C_DISCONNECTING) 2336 set_bit(DISCONNECT_SENT, &connection->flags); 2337 2338 /* ... and re-aquire the spinlock. 2339 * If _conn_rq_cond() returned >= SS_SUCCESS, we must call 2340 * conn_set_state() within the same spinlock. */ 2341 spin_lock_irq(&connection->resource->req_lock); 2342 wait_event_lock_irq(connection->ping_wait, 2343 (rv = _conn_rq_cond(connection, mask, val)), 2344 connection->resource->req_lock); 2345 clear_bit(CONN_WD_ST_CHG_REQ, &connection->flags); 2346 if (rv < SS_SUCCESS) 2347 goto abort; 2348 } 2349 2350 state_change = remember_old_state(connection->resource, GFP_ATOMIC); 2351 conn_old_common_state(connection, &os, &flags); 2352 flags |= CS_DC_SUSP; 2353 conn_set_state(connection, mask, val, &ns_min, &ns_max, flags); 2354 conn_pr_state_change(connection, os, ns_max, flags); 2355 remember_new_state(state_change); 2356 2357 acscw = kmalloc(sizeof(*acscw), GFP_ATOMIC); 2358 if (acscw) { 2359 acscw->oc = os.conn; 2360 acscw->ns_min = ns_min; 2361 acscw->ns_max = ns_max; 2362 acscw->flags = flags; 2363 acscw->w.cb = w_after_conn_state_ch; 2364 kref_get(&connection->kref); 2365 acscw->connection = connection; 2366 acscw->state_change = state_change; 2367 drbd_queue_work(&connection->sender_work, &acscw->w); 2368 } else { 2369 drbd_err(connection, "Could not kmalloc an acscw\n"); 2370 } 2371 2372 abort: 2373 if (have_mutex) { 2374 /* mutex_unlock() "... must not be used in interrupt context.", 2375 * so give up the spinlock, then re-aquire it */ 2376 spin_unlock_irq(&connection->resource->req_lock); 2377 abort_unlocked: 2378 mutex_unlock(&connection->cstate_mutex); 2379 spin_lock_irq(&connection->resource->req_lock); 2380 } 2381 if (rv < SS_SUCCESS && flags & CS_VERBOSE) { 2382 drbd_err(connection, "State change failed: %s\n", drbd_set_st_err_str(rv)); 2383 drbd_err(connection, " mask = 0x%x val = 0x%x\n", mask.i, val.i); 2384 drbd_err(connection, " old_conn:%s wanted_conn:%s\n", drbd_conn_str(oc), drbd_conn_str(val.conn)); 2385 } 2386 return rv; 2387 } 2388 2389 enum drbd_state_rv 2390 conn_request_state(struct drbd_connection *connection, union drbd_state mask, union drbd_state val, 2391 enum chg_state_flags flags) 2392 { 2393 enum drbd_state_rv rv; 2394 2395 spin_lock_irq(&connection->resource->req_lock); 2396 rv = _conn_request_state(connection, mask, val, flags); 2397 spin_unlock_irq(&connection->resource->req_lock); 2398 2399 return rv; 2400 } 2401