1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * stack_o2cb.c 4 * 5 * Code which interfaces ocfs2 with the o2cb stack. 6 * 7 * Copyright (C) 2007 Oracle. All rights reserved. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/crc32.h> 12 #include <linux/slab.h> 13 #include <linux/module.h> 14 15 /* Needed for AOP_TRUNCATED_PAGE in mlog_errno() */ 16 #include <linux/fs.h> 17 18 #include "cluster/masklog.h" 19 #include "cluster/nodemanager.h" 20 #include "cluster/heartbeat.h" 21 #include "cluster/tcp.h" 22 23 #include "stackglue.h" 24 25 struct o2dlm_private { 26 struct dlm_eviction_cb op_eviction_cb; 27 }; 28 29 static struct ocfs2_stack_plugin o2cb_stack; 30 31 /* These should be identical */ 32 #if (DLM_LOCK_IV != LKM_IVMODE) 33 # error Lock modes do not match 34 #endif 35 #if (DLM_LOCK_NL != LKM_NLMODE) 36 # error Lock modes do not match 37 #endif 38 #if (DLM_LOCK_CR != LKM_CRMODE) 39 # error Lock modes do not match 40 #endif 41 #if (DLM_LOCK_CW != LKM_CWMODE) 42 # error Lock modes do not match 43 #endif 44 #if (DLM_LOCK_PR != LKM_PRMODE) 45 # error Lock modes do not match 46 #endif 47 #if (DLM_LOCK_PW != LKM_PWMODE) 48 # error Lock modes do not match 49 #endif 50 #if (DLM_LOCK_EX != LKM_EXMODE) 51 # error Lock modes do not match 52 #endif 53 static inline int mode_to_o2dlm(int mode) 54 { 55 BUG_ON(mode > LKM_MAXMODE); 56 57 return mode; 58 } 59 60 static int flags_to_o2dlm(u32 flags) 61 { 62 int o2dlm_flags = 0; 63 64 if (flags & DLM_LKF_NOQUEUE) 65 o2dlm_flags |= LKM_NOQUEUE; 66 if (flags & DLM_LKF_CANCEL) 67 o2dlm_flags |= LKM_CANCEL; 68 if (flags & DLM_LKF_CONVERT) 69 o2dlm_flags |= LKM_CONVERT; 70 if (flags & DLM_LKF_VALBLK) 71 o2dlm_flags |= LKM_VALBLK; 72 if (flags & DLM_LKF_IVVALBLK) 73 o2dlm_flags |= LKM_INVVALBLK; 74 if (flags & DLM_LKF_ORPHAN) 75 o2dlm_flags |= LKM_ORPHAN; 76 if (flags & DLM_LKF_FORCEUNLOCK) 77 o2dlm_flags |= LKM_FORCE; 78 if (flags & DLM_LKF_TIMEOUT) 79 o2dlm_flags |= LKM_TIMEOUT; 80 if (flags & DLM_LKF_LOCAL) 81 o2dlm_flags |= LKM_LOCAL; 82 83 return o2dlm_flags; 84 } 85 86 /* 87 * Map an o2dlm status to standard errno values. 88 * 89 * o2dlm only uses a handful of these, and returns even fewer to the 90 * caller. Still, we try to assign sane values to each error. 91 * 92 * The following value pairs have special meanings to dlmglue, thus 93 * the right hand side needs to stay unique - never duplicate the 94 * mapping elsewhere in the table! 95 * 96 * DLM_NORMAL: 0 97 * DLM_NOTQUEUED: -EAGAIN 98 * DLM_CANCELGRANT: -EBUSY 99 * DLM_CANCEL: -DLM_ECANCEL 100 */ 101 /* Keep in sync with dlmapi.h */ 102 static int status_map[] = { 103 [DLM_NORMAL] = 0, /* Success */ 104 [DLM_GRANTED] = -EINVAL, 105 [DLM_DENIED] = -EACCES, 106 [DLM_DENIED_NOLOCKS] = -EACCES, 107 [DLM_WORKING] = -EACCES, 108 [DLM_BLOCKED] = -EINVAL, 109 [DLM_BLOCKED_ORPHAN] = -EINVAL, 110 [DLM_DENIED_GRACE_PERIOD] = -EACCES, 111 [DLM_SYSERR] = -ENOMEM, /* It is what it is */ 112 [DLM_NOSUPPORT] = -EPROTO, 113 [DLM_CANCELGRANT] = -EBUSY, /* Cancel after grant */ 114 [DLM_IVLOCKID] = -EINVAL, 115 [DLM_SYNC] = -EINVAL, 116 [DLM_BADTYPE] = -EINVAL, 117 [DLM_BADRESOURCE] = -EINVAL, 118 [DLM_MAXHANDLES] = -ENOMEM, 119 [DLM_NOCLINFO] = -EINVAL, 120 [DLM_NOLOCKMGR] = -EINVAL, 121 [DLM_NOPURGED] = -EINVAL, 122 [DLM_BADARGS] = -EINVAL, 123 [DLM_VOID] = -EINVAL, 124 [DLM_NOTQUEUED] = -EAGAIN, /* Trylock failed */ 125 [DLM_IVBUFLEN] = -EINVAL, 126 [DLM_CVTUNGRANT] = -EPERM, 127 [DLM_BADPARAM] = -EINVAL, 128 [DLM_VALNOTVALID] = -EINVAL, 129 [DLM_REJECTED] = -EPERM, 130 [DLM_ABORT] = -EINVAL, 131 [DLM_CANCEL] = -DLM_ECANCEL, /* Successful cancel */ 132 [DLM_IVRESHANDLE] = -EINVAL, 133 [DLM_DEADLOCK] = -EDEADLK, 134 [DLM_DENIED_NOASTS] = -EINVAL, 135 [DLM_FORWARD] = -EINVAL, 136 [DLM_TIMEOUT] = -ETIMEDOUT, 137 [DLM_IVGROUPID] = -EINVAL, 138 [DLM_VERS_CONFLICT] = -EOPNOTSUPP, 139 [DLM_BAD_DEVICE_PATH] = -ENOENT, 140 [DLM_NO_DEVICE_PERMISSION] = -EPERM, 141 [DLM_NO_CONTROL_DEVICE] = -ENOENT, 142 [DLM_RECOVERING] = -ENOTCONN, 143 [DLM_MIGRATING] = -ERESTART, 144 [DLM_MAXSTATS] = -EINVAL, 145 }; 146 147 static int dlm_status_to_errno(enum dlm_status status) 148 { 149 BUG_ON(status < 0 || status >= ARRAY_SIZE(status_map)); 150 151 return status_map[status]; 152 } 153 154 static void o2dlm_lock_ast_wrapper(void *astarg) 155 { 156 struct ocfs2_dlm_lksb *lksb = astarg; 157 158 lksb->lksb_conn->cc_proto->lp_lock_ast(lksb); 159 } 160 161 static void o2dlm_blocking_ast_wrapper(void *astarg, int level) 162 { 163 struct ocfs2_dlm_lksb *lksb = astarg; 164 165 lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level); 166 } 167 168 static void o2dlm_unlock_ast_wrapper(void *astarg, enum dlm_status status) 169 { 170 struct ocfs2_dlm_lksb *lksb = astarg; 171 int error = dlm_status_to_errno(status); 172 173 /* 174 * In o2dlm, you can get both the lock_ast() for the lock being 175 * granted and the unlock_ast() for the CANCEL failing. A 176 * successful cancel sends DLM_NORMAL here. If the 177 * lock grant happened before the cancel arrived, you get 178 * DLM_CANCELGRANT. 179 * 180 * There's no need for the double-ast. If we see DLM_CANCELGRANT, 181 * we just ignore it. We expect the lock_ast() to handle the 182 * granted lock. 183 */ 184 if (status == DLM_CANCELGRANT) 185 return; 186 187 lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, error); 188 } 189 190 static int o2cb_dlm_lock(struct ocfs2_cluster_connection *conn, 191 int mode, 192 struct ocfs2_dlm_lksb *lksb, 193 u32 flags, 194 void *name, 195 unsigned int namelen) 196 { 197 enum dlm_status status; 198 int o2dlm_mode = mode_to_o2dlm(mode); 199 int o2dlm_flags = flags_to_o2dlm(flags); 200 int ret; 201 202 status = dlmlock(conn->cc_lockspace, o2dlm_mode, &lksb->lksb_o2dlm, 203 o2dlm_flags, name, namelen, 204 o2dlm_lock_ast_wrapper, lksb, 205 o2dlm_blocking_ast_wrapper); 206 ret = dlm_status_to_errno(status); 207 return ret; 208 } 209 210 static int o2cb_dlm_unlock(struct ocfs2_cluster_connection *conn, 211 struct ocfs2_dlm_lksb *lksb, 212 u32 flags) 213 { 214 enum dlm_status status; 215 int o2dlm_flags = flags_to_o2dlm(flags); 216 int ret; 217 218 status = dlmunlock(conn->cc_lockspace, &lksb->lksb_o2dlm, 219 o2dlm_flags, o2dlm_unlock_ast_wrapper, lksb); 220 ret = dlm_status_to_errno(status); 221 return ret; 222 } 223 224 static int o2cb_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) 225 { 226 return dlm_status_to_errno(lksb->lksb_o2dlm.status); 227 } 228 229 /* 230 * o2dlm aways has a "valid" LVB. If the dlm loses track of the LVB 231 * contents, it will zero out the LVB. Thus the caller can always trust 232 * the contents. 233 */ 234 static int o2cb_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) 235 { 236 return 1; 237 } 238 239 static void *o2cb_dlm_lvb(struct ocfs2_dlm_lksb *lksb) 240 { 241 return (void *)(lksb->lksb_o2dlm.lvb); 242 } 243 244 static void o2cb_dump_lksb(struct ocfs2_dlm_lksb *lksb) 245 { 246 dlm_print_one_lock(lksb->lksb_o2dlm.lockid); 247 } 248 249 /* 250 * Check if this node is heartbeating and is connected to all other 251 * heartbeating nodes. 252 */ 253 static int o2cb_cluster_check(void) 254 { 255 u8 node_num; 256 int i; 257 unsigned long hbmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; 258 unsigned long netmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; 259 260 node_num = o2nm_this_node(); 261 if (node_num == O2NM_MAX_NODES) { 262 printk(KERN_ERR "o2cb: This node has not been configured.\n"); 263 return -EINVAL; 264 } 265 266 /* 267 * o2dlm expects o2net sockets to be created. If not, then 268 * dlm_join_domain() fails with a stack of errors which are both cryptic 269 * and incomplete. The idea here is to detect upfront whether we have 270 * managed to connect to all nodes or not. If not, then list the nodes 271 * to allow the user to check the configuration (incorrect IP, firewall, 272 * etc.) Yes, this is racy. But its not the end of the world. 273 */ 274 #define O2CB_MAP_STABILIZE_COUNT 60 275 for (i = 0; i < O2CB_MAP_STABILIZE_COUNT; ++i) { 276 o2hb_fill_node_map(hbmap, O2NM_MAX_NODES); 277 if (!test_bit(node_num, hbmap)) { 278 printk(KERN_ERR "o2cb: %s heartbeat has not been " 279 "started.\n", (o2hb_global_heartbeat_active() ? 280 "Global" : "Local")); 281 return -EINVAL; 282 } 283 o2net_fill_node_map(netmap, O2NM_MAX_NODES); 284 /* Force set the current node to allow easy compare */ 285 set_bit(node_num, netmap); 286 if (bitmap_equal(hbmap, netmap, O2NM_MAX_NODES)) 287 return 0; 288 if (i < O2CB_MAP_STABILIZE_COUNT - 1) 289 msleep(1000); 290 } 291 292 printk(KERN_ERR "o2cb: This node could not connect to nodes:"); 293 i = -1; 294 while ((i = find_next_bit(hbmap, O2NM_MAX_NODES, 295 i + 1)) < O2NM_MAX_NODES) { 296 if (!test_bit(i, netmap)) 297 printk(" %u", i); 298 } 299 printk(".\n"); 300 301 return -ENOTCONN; 302 } 303 304 /* 305 * Called from the dlm when it's about to evict a node. This is how the 306 * classic stack signals node death. 307 */ 308 static void o2dlm_eviction_cb(int node_num, void *data) 309 { 310 struct ocfs2_cluster_connection *conn = data; 311 312 printk(KERN_NOTICE "o2cb: o2dlm has evicted node %d from domain %.*s\n", 313 node_num, conn->cc_namelen, conn->cc_name); 314 315 conn->cc_recovery_handler(node_num, conn->cc_recovery_data); 316 } 317 318 static int o2cb_cluster_connect(struct ocfs2_cluster_connection *conn) 319 { 320 int rc = 0; 321 u32 dlm_key; 322 struct dlm_ctxt *dlm; 323 struct o2dlm_private *priv; 324 struct dlm_protocol_version fs_version; 325 326 BUG_ON(conn == NULL); 327 BUG_ON(conn->cc_proto == NULL); 328 329 /* Ensure cluster stack is up and all nodes are connected */ 330 rc = o2cb_cluster_check(); 331 if (rc) { 332 printk(KERN_ERR "o2cb: Cluster check failed. Fix errors " 333 "before retrying.\n"); 334 goto out; 335 } 336 337 priv = kzalloc(sizeof(struct o2dlm_private), GFP_KERNEL); 338 if (!priv) { 339 rc = -ENOMEM; 340 goto out_free; 341 } 342 343 /* This just fills the structure in. It is safe to pass conn. */ 344 dlm_setup_eviction_cb(&priv->op_eviction_cb, o2dlm_eviction_cb, 345 conn); 346 347 conn->cc_private = priv; 348 349 /* used by the dlm code to make message headers unique, each 350 * node in this domain must agree on this. */ 351 dlm_key = crc32_le(0, conn->cc_name, conn->cc_namelen); 352 fs_version.pv_major = conn->cc_version.pv_major; 353 fs_version.pv_minor = conn->cc_version.pv_minor; 354 355 dlm = dlm_register_domain(conn->cc_name, dlm_key, &fs_version); 356 if (IS_ERR(dlm)) { 357 rc = PTR_ERR(dlm); 358 mlog_errno(rc); 359 goto out_free; 360 } 361 362 conn->cc_version.pv_major = fs_version.pv_major; 363 conn->cc_version.pv_minor = fs_version.pv_minor; 364 conn->cc_lockspace = dlm; 365 366 dlm_register_eviction_cb(dlm, &priv->op_eviction_cb); 367 368 out_free: 369 if (rc) 370 kfree(conn->cc_private); 371 372 out: 373 return rc; 374 } 375 376 static int o2cb_cluster_disconnect(struct ocfs2_cluster_connection *conn) 377 { 378 struct dlm_ctxt *dlm = conn->cc_lockspace; 379 struct o2dlm_private *priv = conn->cc_private; 380 381 dlm_unregister_eviction_cb(&priv->op_eviction_cb); 382 conn->cc_private = NULL; 383 kfree(priv); 384 385 dlm_unregister_domain(dlm); 386 conn->cc_lockspace = NULL; 387 388 return 0; 389 } 390 391 static int o2cb_cluster_this_node(struct ocfs2_cluster_connection *conn, 392 unsigned int *node) 393 { 394 int node_num; 395 396 node_num = o2nm_this_node(); 397 if (node_num == O2NM_INVALID_NODE_NUM) 398 return -ENOENT; 399 400 if (node_num >= O2NM_MAX_NODES) 401 return -EOVERFLOW; 402 403 *node = node_num; 404 return 0; 405 } 406 407 static struct ocfs2_stack_operations o2cb_stack_ops = { 408 .connect = o2cb_cluster_connect, 409 .disconnect = o2cb_cluster_disconnect, 410 .this_node = o2cb_cluster_this_node, 411 .dlm_lock = o2cb_dlm_lock, 412 .dlm_unlock = o2cb_dlm_unlock, 413 .lock_status = o2cb_dlm_lock_status, 414 .lvb_valid = o2cb_dlm_lvb_valid, 415 .lock_lvb = o2cb_dlm_lvb, 416 .dump_lksb = o2cb_dump_lksb, 417 }; 418 419 static struct ocfs2_stack_plugin o2cb_stack = { 420 .sp_name = "o2cb", 421 .sp_ops = &o2cb_stack_ops, 422 .sp_owner = THIS_MODULE, 423 }; 424 425 static int __init o2cb_stack_init(void) 426 { 427 return ocfs2_stack_glue_register(&o2cb_stack); 428 } 429 430 static void __exit o2cb_stack_exit(void) 431 { 432 ocfs2_stack_glue_unregister(&o2cb_stack); 433 } 434 435 MODULE_AUTHOR("Oracle"); 436 MODULE_DESCRIPTION("ocfs2 driver for the classic o2cb stack"); 437 MODULE_LICENSE("GPL"); 438 module_init(o2cb_stack_init); 439 module_exit(o2cb_stack_exit); 440