1 // SPDX-License-Identifier: GPL-2.0-only 2 /* -*- mode: c; c-basic-offset: 8; -*- 3 * vim: noexpandtab sw=8 ts=8 sts=0: 4 * 5 * stack_o2cb.c 6 * 7 * Code which interfaces ocfs2 with the o2cb stack. 8 * 9 * Copyright (C) 2007 Oracle. All rights reserved. 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/crc32.h> 14 #include <linux/slab.h> 15 #include <linux/module.h> 16 17 /* Needed for AOP_TRUNCATED_PAGE in mlog_errno() */ 18 #include <linux/fs.h> 19 20 #include "cluster/masklog.h" 21 #include "cluster/nodemanager.h" 22 #include "cluster/heartbeat.h" 23 #include "cluster/tcp.h" 24 25 #include "stackglue.h" 26 27 struct o2dlm_private { 28 struct dlm_eviction_cb op_eviction_cb; 29 }; 30 31 static struct ocfs2_stack_plugin o2cb_stack; 32 33 /* These should be identical */ 34 #if (DLM_LOCK_IV != LKM_IVMODE) 35 # error Lock modes do not match 36 #endif 37 #if (DLM_LOCK_NL != LKM_NLMODE) 38 # error Lock modes do not match 39 #endif 40 #if (DLM_LOCK_CR != LKM_CRMODE) 41 # error Lock modes do not match 42 #endif 43 #if (DLM_LOCK_CW != LKM_CWMODE) 44 # error Lock modes do not match 45 #endif 46 #if (DLM_LOCK_PR != LKM_PRMODE) 47 # error Lock modes do not match 48 #endif 49 #if (DLM_LOCK_PW != LKM_PWMODE) 50 # error Lock modes do not match 51 #endif 52 #if (DLM_LOCK_EX != LKM_EXMODE) 53 # error Lock modes do not match 54 #endif 55 static inline int mode_to_o2dlm(int mode) 56 { 57 BUG_ON(mode > LKM_MAXMODE); 58 59 return mode; 60 } 61 62 #define map_flag(_generic, _o2dlm) \ 63 if (flags & (_generic)) { \ 64 flags &= ~(_generic); \ 65 o2dlm_flags |= (_o2dlm); \ 66 } 67 static int flags_to_o2dlm(u32 flags) 68 { 69 int o2dlm_flags = 0; 70 71 map_flag(DLM_LKF_NOQUEUE, LKM_NOQUEUE); 72 map_flag(DLM_LKF_CANCEL, LKM_CANCEL); 73 map_flag(DLM_LKF_CONVERT, LKM_CONVERT); 74 map_flag(DLM_LKF_VALBLK, LKM_VALBLK); 75 map_flag(DLM_LKF_IVVALBLK, LKM_INVVALBLK); 76 map_flag(DLM_LKF_ORPHAN, LKM_ORPHAN); 77 map_flag(DLM_LKF_FORCEUNLOCK, LKM_FORCE); 78 map_flag(DLM_LKF_TIMEOUT, LKM_TIMEOUT); 79 map_flag(DLM_LKF_LOCAL, LKM_LOCAL); 80 81 /* map_flag() should have cleared every flag passed in */ 82 BUG_ON(flags != 0); 83 84 return o2dlm_flags; 85 } 86 #undef map_flag 87 88 /* 89 * Map an o2dlm status to standard errno values. 90 * 91 * o2dlm only uses a handful of these, and returns even fewer to the 92 * caller. Still, we try to assign sane values to each error. 93 * 94 * The following value pairs have special meanings to dlmglue, thus 95 * the right hand side needs to stay unique - never duplicate the 96 * mapping elsewhere in the table! 97 * 98 * DLM_NORMAL: 0 99 * DLM_NOTQUEUED: -EAGAIN 100 * DLM_CANCELGRANT: -EBUSY 101 * DLM_CANCEL: -DLM_ECANCEL 102 */ 103 /* Keep in sync with dlmapi.h */ 104 static int status_map[] = { 105 [DLM_NORMAL] = 0, /* Success */ 106 [DLM_GRANTED] = -EINVAL, 107 [DLM_DENIED] = -EACCES, 108 [DLM_DENIED_NOLOCKS] = -EACCES, 109 [DLM_WORKING] = -EACCES, 110 [DLM_BLOCKED] = -EINVAL, 111 [DLM_BLOCKED_ORPHAN] = -EINVAL, 112 [DLM_DENIED_GRACE_PERIOD] = -EACCES, 113 [DLM_SYSERR] = -ENOMEM, /* It is what it is */ 114 [DLM_NOSUPPORT] = -EPROTO, 115 [DLM_CANCELGRANT] = -EBUSY, /* Cancel after grant */ 116 [DLM_IVLOCKID] = -EINVAL, 117 [DLM_SYNC] = -EINVAL, 118 [DLM_BADTYPE] = -EINVAL, 119 [DLM_BADRESOURCE] = -EINVAL, 120 [DLM_MAXHANDLES] = -ENOMEM, 121 [DLM_NOCLINFO] = -EINVAL, 122 [DLM_NOLOCKMGR] = -EINVAL, 123 [DLM_NOPURGED] = -EINVAL, 124 [DLM_BADARGS] = -EINVAL, 125 [DLM_VOID] = -EINVAL, 126 [DLM_NOTQUEUED] = -EAGAIN, /* Trylock failed */ 127 [DLM_IVBUFLEN] = -EINVAL, 128 [DLM_CVTUNGRANT] = -EPERM, 129 [DLM_BADPARAM] = -EINVAL, 130 [DLM_VALNOTVALID] = -EINVAL, 131 [DLM_REJECTED] = -EPERM, 132 [DLM_ABORT] = -EINVAL, 133 [DLM_CANCEL] = -DLM_ECANCEL, /* Successful cancel */ 134 [DLM_IVRESHANDLE] = -EINVAL, 135 [DLM_DEADLOCK] = -EDEADLK, 136 [DLM_DENIED_NOASTS] = -EINVAL, 137 [DLM_FORWARD] = -EINVAL, 138 [DLM_TIMEOUT] = -ETIMEDOUT, 139 [DLM_IVGROUPID] = -EINVAL, 140 [DLM_VERS_CONFLICT] = -EOPNOTSUPP, 141 [DLM_BAD_DEVICE_PATH] = -ENOENT, 142 [DLM_NO_DEVICE_PERMISSION] = -EPERM, 143 [DLM_NO_CONTROL_DEVICE] = -ENOENT, 144 [DLM_RECOVERING] = -ENOTCONN, 145 [DLM_MIGRATING] = -ERESTART, 146 [DLM_MAXSTATS] = -EINVAL, 147 }; 148 149 static int dlm_status_to_errno(enum dlm_status status) 150 { 151 BUG_ON(status < 0 || status >= ARRAY_SIZE(status_map)); 152 153 return status_map[status]; 154 } 155 156 static void o2dlm_lock_ast_wrapper(void *astarg) 157 { 158 struct ocfs2_dlm_lksb *lksb = astarg; 159 160 lksb->lksb_conn->cc_proto->lp_lock_ast(lksb); 161 } 162 163 static void o2dlm_blocking_ast_wrapper(void *astarg, int level) 164 { 165 struct ocfs2_dlm_lksb *lksb = astarg; 166 167 lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level); 168 } 169 170 static void o2dlm_unlock_ast_wrapper(void *astarg, enum dlm_status status) 171 { 172 struct ocfs2_dlm_lksb *lksb = astarg; 173 int error = dlm_status_to_errno(status); 174 175 /* 176 * In o2dlm, you can get both the lock_ast() for the lock being 177 * granted and the unlock_ast() for the CANCEL failing. A 178 * successful cancel sends DLM_NORMAL here. If the 179 * lock grant happened before the cancel arrived, you get 180 * DLM_CANCELGRANT. 181 * 182 * There's no need for the double-ast. If we see DLM_CANCELGRANT, 183 * we just ignore it. We expect the lock_ast() to handle the 184 * granted lock. 185 */ 186 if (status == DLM_CANCELGRANT) 187 return; 188 189 lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, error); 190 } 191 192 static int o2cb_dlm_lock(struct ocfs2_cluster_connection *conn, 193 int mode, 194 struct ocfs2_dlm_lksb *lksb, 195 u32 flags, 196 void *name, 197 unsigned int namelen) 198 { 199 enum dlm_status status; 200 int o2dlm_mode = mode_to_o2dlm(mode); 201 int o2dlm_flags = flags_to_o2dlm(flags); 202 int ret; 203 204 status = dlmlock(conn->cc_lockspace, o2dlm_mode, &lksb->lksb_o2dlm, 205 o2dlm_flags, name, namelen, 206 o2dlm_lock_ast_wrapper, lksb, 207 o2dlm_blocking_ast_wrapper); 208 ret = dlm_status_to_errno(status); 209 return ret; 210 } 211 212 static int o2cb_dlm_unlock(struct ocfs2_cluster_connection *conn, 213 struct ocfs2_dlm_lksb *lksb, 214 u32 flags) 215 { 216 enum dlm_status status; 217 int o2dlm_flags = flags_to_o2dlm(flags); 218 int ret; 219 220 status = dlmunlock(conn->cc_lockspace, &lksb->lksb_o2dlm, 221 o2dlm_flags, o2dlm_unlock_ast_wrapper, lksb); 222 ret = dlm_status_to_errno(status); 223 return ret; 224 } 225 226 static int o2cb_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) 227 { 228 return dlm_status_to_errno(lksb->lksb_o2dlm.status); 229 } 230 231 /* 232 * o2dlm aways has a "valid" LVB. If the dlm loses track of the LVB 233 * contents, it will zero out the LVB. Thus the caller can always trust 234 * the contents. 235 */ 236 static int o2cb_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) 237 { 238 return 1; 239 } 240 241 static void *o2cb_dlm_lvb(struct ocfs2_dlm_lksb *lksb) 242 { 243 return (void *)(lksb->lksb_o2dlm.lvb); 244 } 245 246 static void o2cb_dump_lksb(struct ocfs2_dlm_lksb *lksb) 247 { 248 dlm_print_one_lock(lksb->lksb_o2dlm.lockid); 249 } 250 251 /* 252 * Check if this node is heartbeating and is connected to all other 253 * heartbeating nodes. 254 */ 255 static int o2cb_cluster_check(void) 256 { 257 u8 node_num; 258 int i; 259 unsigned long hbmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; 260 unsigned long netmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; 261 262 node_num = o2nm_this_node(); 263 if (node_num == O2NM_MAX_NODES) { 264 printk(KERN_ERR "o2cb: This node has not been configured.\n"); 265 return -EINVAL; 266 } 267 268 /* 269 * o2dlm expects o2net sockets to be created. If not, then 270 * dlm_join_domain() fails with a stack of errors which are both cryptic 271 * and incomplete. The idea here is to detect upfront whether we have 272 * managed to connect to all nodes or not. If not, then list the nodes 273 * to allow the user to check the configuration (incorrect IP, firewall, 274 * etc.) Yes, this is racy. But its not the end of the world. 275 */ 276 #define O2CB_MAP_STABILIZE_COUNT 60 277 for (i = 0; i < O2CB_MAP_STABILIZE_COUNT; ++i) { 278 o2hb_fill_node_map(hbmap, sizeof(hbmap)); 279 if (!test_bit(node_num, hbmap)) { 280 printk(KERN_ERR "o2cb: %s heartbeat has not been " 281 "started.\n", (o2hb_global_heartbeat_active() ? 282 "Global" : "Local")); 283 return -EINVAL; 284 } 285 o2net_fill_node_map(netmap, sizeof(netmap)); 286 /* Force set the current node to allow easy compare */ 287 set_bit(node_num, netmap); 288 if (!memcmp(hbmap, netmap, sizeof(hbmap))) 289 return 0; 290 if (i < O2CB_MAP_STABILIZE_COUNT - 1) 291 msleep(1000); 292 } 293 294 printk(KERN_ERR "o2cb: This node could not connect to nodes:"); 295 i = -1; 296 while ((i = find_next_bit(hbmap, O2NM_MAX_NODES, 297 i + 1)) < O2NM_MAX_NODES) { 298 if (!test_bit(i, netmap)) 299 printk(" %u", i); 300 } 301 printk(".\n"); 302 303 return -ENOTCONN; 304 } 305 306 /* 307 * Called from the dlm when it's about to evict a node. This is how the 308 * classic stack signals node death. 309 */ 310 static void o2dlm_eviction_cb(int node_num, void *data) 311 { 312 struct ocfs2_cluster_connection *conn = data; 313 314 printk(KERN_NOTICE "o2cb: o2dlm has evicted node %d from domain %.*s\n", 315 node_num, conn->cc_namelen, conn->cc_name); 316 317 conn->cc_recovery_handler(node_num, conn->cc_recovery_data); 318 } 319 320 static int o2cb_cluster_connect(struct ocfs2_cluster_connection *conn) 321 { 322 int rc = 0; 323 u32 dlm_key; 324 struct dlm_ctxt *dlm; 325 struct o2dlm_private *priv; 326 struct dlm_protocol_version fs_version; 327 328 BUG_ON(conn == NULL); 329 BUG_ON(conn->cc_proto == NULL); 330 331 /* Ensure cluster stack is up and all nodes are connected */ 332 rc = o2cb_cluster_check(); 333 if (rc) { 334 printk(KERN_ERR "o2cb: Cluster check failed. Fix errors " 335 "before retrying.\n"); 336 goto out; 337 } 338 339 priv = kzalloc(sizeof(struct o2dlm_private), GFP_KERNEL); 340 if (!priv) { 341 rc = -ENOMEM; 342 goto out_free; 343 } 344 345 /* This just fills the structure in. It is safe to pass conn. */ 346 dlm_setup_eviction_cb(&priv->op_eviction_cb, o2dlm_eviction_cb, 347 conn); 348 349 conn->cc_private = priv; 350 351 /* used by the dlm code to make message headers unique, each 352 * node in this domain must agree on this. */ 353 dlm_key = crc32_le(0, conn->cc_name, conn->cc_namelen); 354 fs_version.pv_major = conn->cc_version.pv_major; 355 fs_version.pv_minor = conn->cc_version.pv_minor; 356 357 dlm = dlm_register_domain(conn->cc_name, dlm_key, &fs_version); 358 if (IS_ERR(dlm)) { 359 rc = PTR_ERR(dlm); 360 mlog_errno(rc); 361 goto out_free; 362 } 363 364 conn->cc_version.pv_major = fs_version.pv_major; 365 conn->cc_version.pv_minor = fs_version.pv_minor; 366 conn->cc_lockspace = dlm; 367 368 dlm_register_eviction_cb(dlm, &priv->op_eviction_cb); 369 370 out_free: 371 if (rc) 372 kfree(conn->cc_private); 373 374 out: 375 return rc; 376 } 377 378 static int o2cb_cluster_disconnect(struct ocfs2_cluster_connection *conn) 379 { 380 struct dlm_ctxt *dlm = conn->cc_lockspace; 381 struct o2dlm_private *priv = conn->cc_private; 382 383 dlm_unregister_eviction_cb(&priv->op_eviction_cb); 384 conn->cc_private = NULL; 385 kfree(priv); 386 387 dlm_unregister_domain(dlm); 388 conn->cc_lockspace = NULL; 389 390 return 0; 391 } 392 393 static int o2cb_cluster_this_node(struct ocfs2_cluster_connection *conn, 394 unsigned int *node) 395 { 396 int node_num; 397 398 node_num = o2nm_this_node(); 399 if (node_num == O2NM_INVALID_NODE_NUM) 400 return -ENOENT; 401 402 if (node_num >= O2NM_MAX_NODES) 403 return -EOVERFLOW; 404 405 *node = node_num; 406 return 0; 407 } 408 409 static struct ocfs2_stack_operations o2cb_stack_ops = { 410 .connect = o2cb_cluster_connect, 411 .disconnect = o2cb_cluster_disconnect, 412 .this_node = o2cb_cluster_this_node, 413 .dlm_lock = o2cb_dlm_lock, 414 .dlm_unlock = o2cb_dlm_unlock, 415 .lock_status = o2cb_dlm_lock_status, 416 .lvb_valid = o2cb_dlm_lvb_valid, 417 .lock_lvb = o2cb_dlm_lvb, 418 .dump_lksb = o2cb_dump_lksb, 419 }; 420 421 static struct ocfs2_stack_plugin o2cb_stack = { 422 .sp_name = "o2cb", 423 .sp_ops = &o2cb_stack_ops, 424 .sp_owner = THIS_MODULE, 425 }; 426 427 static int __init o2cb_stack_init(void) 428 { 429 return ocfs2_stack_glue_register(&o2cb_stack); 430 } 431 432 static void __exit o2cb_stack_exit(void) 433 { 434 ocfs2_stack_glue_unregister(&o2cb_stack); 435 } 436 437 MODULE_AUTHOR("Oracle"); 438 MODULE_DESCRIPTION("ocfs2 driver for the classic o2cb stack"); 439 MODULE_LICENSE("GPL"); 440 module_init(o2cb_stack_init); 441 module_exit(o2cb_stack_exit); 442