1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #include "rxe.h" 8 9 #define RXE_POOL_ALIGN (16) 10 11 static const struct rxe_type_info { 12 const char *name; 13 size_t size; 14 size_t elem_offset; 15 void (*cleanup)(struct rxe_pool_elem *obj); 16 enum rxe_pool_flags flags; 17 u32 min_index; 18 u32 max_index; 19 } rxe_type_info[RXE_NUM_TYPES] = { 20 [RXE_TYPE_UC] = { 21 .name = "rxe-uc", 22 .size = sizeof(struct rxe_ucontext), 23 .elem_offset = offsetof(struct rxe_ucontext, elem), 24 .flags = RXE_POOL_NO_ALLOC, 25 }, 26 [RXE_TYPE_PD] = { 27 .name = "rxe-pd", 28 .size = sizeof(struct rxe_pd), 29 .elem_offset = offsetof(struct rxe_pd, elem), 30 .flags = RXE_POOL_NO_ALLOC, 31 }, 32 [RXE_TYPE_AH] = { 33 .name = "rxe-ah", 34 .size = sizeof(struct rxe_ah), 35 .elem_offset = offsetof(struct rxe_ah, elem), 36 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 37 .min_index = RXE_MIN_AH_INDEX, 38 .max_index = RXE_MAX_AH_INDEX, 39 }, 40 [RXE_TYPE_SRQ] = { 41 .name = "rxe-srq", 42 .size = sizeof(struct rxe_srq), 43 .elem_offset = offsetof(struct rxe_srq, elem), 44 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 45 .min_index = RXE_MIN_SRQ_INDEX, 46 .max_index = RXE_MAX_SRQ_INDEX, 47 }, 48 [RXE_TYPE_QP] = { 49 .name = "rxe-qp", 50 .size = sizeof(struct rxe_qp), 51 .elem_offset = offsetof(struct rxe_qp, elem), 52 .cleanup = rxe_qp_cleanup, 53 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 54 .min_index = RXE_MIN_QP_INDEX, 55 .max_index = RXE_MAX_QP_INDEX, 56 }, 57 [RXE_TYPE_CQ] = { 58 .name = "rxe-cq", 59 .size = sizeof(struct rxe_cq), 60 .elem_offset = offsetof(struct rxe_cq, elem), 61 .flags = RXE_POOL_NO_ALLOC, 62 .cleanup = rxe_cq_cleanup, 63 }, 64 [RXE_TYPE_MR] = { 65 .name = "rxe-mr", 66 .size = sizeof(struct rxe_mr), 67 .elem_offset = offsetof(struct rxe_mr, elem), 68 .cleanup = rxe_mr_cleanup, 69 .flags = RXE_POOL_INDEX, 70 .min_index = RXE_MIN_MR_INDEX, 71 .max_index = RXE_MAX_MR_INDEX, 72 }, 73 [RXE_TYPE_MW] = { 74 .name = "rxe-mw", 75 .size = sizeof(struct rxe_mw), 76 .elem_offset = offsetof(struct rxe_mw, elem), 77 .cleanup = rxe_mw_cleanup, 78 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 79 .min_index = RXE_MIN_MW_INDEX, 80 .max_index = RXE_MAX_MW_INDEX, 81 }, 82 [RXE_TYPE_MC_GRP] = { 83 .name = "rxe-mc_grp", 84 .size = sizeof(struct rxe_mcg), 85 .elem_offset = offsetof(struct rxe_mcg, elem), 86 .cleanup = rxe_mc_cleanup, 87 .flags = RXE_POOL_KEY, 88 .key_offset = offsetof(struct rxe_mcg, mgid), 89 .key_size = sizeof(union ib_gid), 90 }, 91 }; 92 93 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min) 94 { 95 int err = 0; 96 97 if ((max - min + 1) < pool->max_elem) { 98 pr_warn("not enough indices for max_elem\n"); 99 err = -EINVAL; 100 goto out; 101 } 102 103 pool->index.max_index = max; 104 pool->index.min_index = min; 105 106 pool->index.table = bitmap_zalloc(max - min + 1, GFP_KERNEL); 107 if (!pool->index.table) { 108 err = -ENOMEM; 109 goto out; 110 } 111 112 out: 113 return err; 114 } 115 116 int rxe_pool_init( 117 struct rxe_dev *rxe, 118 struct rxe_pool *pool, 119 enum rxe_elem_type type, 120 unsigned int max_elem) 121 { 122 const struct rxe_type_info *info = &rxe_type_info[type]; 123 int err = 0; 124 125 memset(pool, 0, sizeof(*pool)); 126 127 pool->rxe = rxe; 128 pool->name = info->name; 129 pool->type = type; 130 pool->max_elem = max_elem; 131 pool->elem_size = ALIGN(info->size, RXE_POOL_ALIGN); 132 pool->elem_offset = info->elem_offset; 133 pool->flags = info->flags; 134 pool->cleanup = info->cleanup; 135 136 atomic_set(&pool->num_elem, 0); 137 138 rwlock_init(&pool->pool_lock); 139 140 if (pool->flags & RXE_POOL_INDEX) { 141 pool->index.tree = RB_ROOT; 142 err = rxe_pool_init_index(pool, info->max_index, 143 info->min_index); 144 if (err) 145 goto out; 146 } 147 148 out: 149 return err; 150 } 151 152 void rxe_pool_cleanup(struct rxe_pool *pool) 153 { 154 if (atomic_read(&pool->num_elem) > 0) 155 pr_warn("%s pool destroyed with unfree'd elem\n", 156 pool->name); 157 158 if (pool->flags & RXE_POOL_INDEX) 159 bitmap_free(pool->index.table); 160 } 161 162 static u32 alloc_index(struct rxe_pool *pool) 163 { 164 u32 index; 165 u32 range = pool->index.max_index - pool->index.min_index + 1; 166 167 index = find_next_zero_bit(pool->index.table, range, pool->index.last); 168 if (index >= range) 169 index = find_first_zero_bit(pool->index.table, range); 170 171 WARN_ON_ONCE(index >= range); 172 set_bit(index, pool->index.table); 173 pool->index.last = index; 174 return index + pool->index.min_index; 175 } 176 177 static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_elem *new) 178 { 179 struct rb_node **link = &pool->index.tree.rb_node; 180 struct rb_node *parent = NULL; 181 struct rxe_pool_elem *elem; 182 183 while (*link) { 184 parent = *link; 185 elem = rb_entry(parent, struct rxe_pool_elem, index_node); 186 187 if (elem->index == new->index) { 188 pr_warn("element already exists!\n"); 189 return -EINVAL; 190 } 191 192 if (elem->index > new->index) 193 link = &(*link)->rb_left; 194 else 195 link = &(*link)->rb_right; 196 } 197 198 rb_link_node(&new->index_node, parent, link); 199 rb_insert_color(&new->index_node, &pool->index.tree); 200 201 return 0; 202 } 203 204 int __rxe_add_index_locked(struct rxe_pool_elem *elem) 205 { 206 struct rxe_pool *pool = elem->pool; 207 int err; 208 209 elem->index = alloc_index(pool); 210 err = rxe_insert_index(pool, elem); 211 212 return err; 213 } 214 215 int __rxe_add_index(struct rxe_pool_elem *elem) 216 { 217 struct rxe_pool *pool = elem->pool; 218 unsigned long flags; 219 int err; 220 221 write_lock_irqsave(&pool->pool_lock, flags); 222 err = __rxe_add_index_locked(elem); 223 write_unlock_irqrestore(&pool->pool_lock, flags); 224 225 return err; 226 } 227 228 void __rxe_drop_index_locked(struct rxe_pool_elem *elem) 229 { 230 struct rxe_pool *pool = elem->pool; 231 232 clear_bit(elem->index - pool->index.min_index, pool->index.table); 233 rb_erase(&elem->index_node, &pool->index.tree); 234 } 235 236 void __rxe_drop_index(struct rxe_pool_elem *elem) 237 { 238 struct rxe_pool *pool = elem->pool; 239 unsigned long flags; 240 241 write_lock_irqsave(&pool->pool_lock, flags); 242 __rxe_drop_index_locked(elem); 243 write_unlock_irqrestore(&pool->pool_lock, flags); 244 } 245 246 void *rxe_alloc_locked(struct rxe_pool *pool) 247 { 248 struct rxe_pool_elem *elem; 249 void *obj; 250 251 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 252 goto out_cnt; 253 254 obj = kzalloc(pool->elem_size, GFP_ATOMIC); 255 if (!obj) 256 goto out_cnt; 257 258 elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); 259 260 elem->pool = pool; 261 elem->obj = obj; 262 kref_init(&elem->ref_cnt); 263 264 return obj; 265 266 out_cnt: 267 atomic_dec(&pool->num_elem); 268 return NULL; 269 } 270 271 void *rxe_alloc(struct rxe_pool *pool) 272 { 273 struct rxe_pool_elem *elem; 274 void *obj; 275 276 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 277 goto out_cnt; 278 279 obj = kzalloc(pool->elem_size, GFP_KERNEL); 280 if (!obj) 281 goto out_cnt; 282 283 elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); 284 285 elem->pool = pool; 286 elem->obj = obj; 287 kref_init(&elem->ref_cnt); 288 289 return obj; 290 291 out_cnt: 292 atomic_dec(&pool->num_elem); 293 return NULL; 294 } 295 296 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem) 297 { 298 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 299 goto out_cnt; 300 301 elem->pool = pool; 302 elem->obj = (u8 *)elem - pool->elem_offset; 303 kref_init(&elem->ref_cnt); 304 305 return 0; 306 307 out_cnt: 308 atomic_dec(&pool->num_elem); 309 return -EINVAL; 310 } 311 312 void rxe_elem_release(struct kref *kref) 313 { 314 struct rxe_pool_elem *elem = 315 container_of(kref, struct rxe_pool_elem, ref_cnt); 316 struct rxe_pool *pool = elem->pool; 317 void *obj; 318 319 if (pool->cleanup) 320 pool->cleanup(elem); 321 322 if (!(pool->flags & RXE_POOL_NO_ALLOC)) { 323 obj = elem->obj; 324 kfree(obj); 325 } 326 327 atomic_dec(&pool->num_elem); 328 } 329 330 void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) 331 { 332 struct rb_node *node; 333 struct rxe_pool_elem *elem; 334 void *obj; 335 336 node = pool->index.tree.rb_node; 337 338 while (node) { 339 elem = rb_entry(node, struct rxe_pool_elem, index_node); 340 341 if (elem->index > index) 342 node = node->rb_left; 343 else if (elem->index < index) 344 node = node->rb_right; 345 else 346 break; 347 } 348 349 if (node) { 350 kref_get(&elem->ref_cnt); 351 obj = elem->obj; 352 } else { 353 obj = NULL; 354 } 355 356 return obj; 357 } 358 359 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) 360 { 361 unsigned long flags; 362 void *obj; 363 364 read_lock_irqsave(&pool->pool_lock, flags); 365 obj = rxe_pool_get_index_locked(pool, index); 366 read_unlock_irqrestore(&pool->pool_lock, flags); 367 368 return obj; 369 } 370