1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #include "rxe.h" 8 9 #define RXE_POOL_ALIGN (16) 10 11 static const struct rxe_type_info { 12 const char *name; 13 size_t size; 14 size_t elem_offset; 15 void (*cleanup)(struct rxe_pool_elem *obj); 16 enum rxe_pool_flags flags; 17 u32 min_index; 18 u32 max_index; 19 } rxe_type_info[RXE_NUM_TYPES] = { 20 [RXE_TYPE_UC] = { 21 .name = "rxe-uc", 22 .size = sizeof(struct rxe_ucontext), 23 .elem_offset = offsetof(struct rxe_ucontext, elem), 24 .flags = RXE_POOL_NO_ALLOC, 25 }, 26 [RXE_TYPE_PD] = { 27 .name = "rxe-pd", 28 .size = sizeof(struct rxe_pd), 29 .elem_offset = offsetof(struct rxe_pd, elem), 30 .flags = RXE_POOL_NO_ALLOC, 31 }, 32 [RXE_TYPE_AH] = { 33 .name = "rxe-ah", 34 .size = sizeof(struct rxe_ah), 35 .elem_offset = offsetof(struct rxe_ah, elem), 36 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 37 .min_index = RXE_MIN_AH_INDEX, 38 .max_index = RXE_MAX_AH_INDEX, 39 }, 40 [RXE_TYPE_SRQ] = { 41 .name = "rxe-srq", 42 .size = sizeof(struct rxe_srq), 43 .elem_offset = offsetof(struct rxe_srq, elem), 44 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 45 .min_index = RXE_MIN_SRQ_INDEX, 46 .max_index = RXE_MAX_SRQ_INDEX, 47 }, 48 [RXE_TYPE_QP] = { 49 .name = "rxe-qp", 50 .size = sizeof(struct rxe_qp), 51 .elem_offset = offsetof(struct rxe_qp, elem), 52 .cleanup = rxe_qp_cleanup, 53 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 54 .min_index = RXE_MIN_QP_INDEX, 55 .max_index = RXE_MAX_QP_INDEX, 56 }, 57 [RXE_TYPE_CQ] = { 58 .name = "rxe-cq", 59 .size = sizeof(struct rxe_cq), 60 .elem_offset = offsetof(struct rxe_cq, elem), 61 .flags = RXE_POOL_NO_ALLOC, 62 .cleanup = rxe_cq_cleanup, 63 }, 64 [RXE_TYPE_MR] = { 65 .name = "rxe-mr", 66 .size = sizeof(struct rxe_mr), 67 .elem_offset = offsetof(struct rxe_mr, elem), 68 .cleanup = rxe_mr_cleanup, 69 .flags = RXE_POOL_INDEX, 70 .min_index = RXE_MIN_MR_INDEX, 71 .max_index = RXE_MAX_MR_INDEX, 72 }, 73 [RXE_TYPE_MW] = { 74 .name = "rxe-mw", 75 .size = sizeof(struct rxe_mw), 76 .elem_offset = offsetof(struct rxe_mw, elem), 77 .cleanup = rxe_mw_cleanup, 78 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, 79 .min_index = RXE_MIN_MW_INDEX, 80 .max_index = RXE_MAX_MW_INDEX, 81 }, 82 }; 83 84 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min) 85 { 86 int err = 0; 87 88 if ((max - min + 1) < pool->max_elem) { 89 pr_warn("not enough indices for max_elem\n"); 90 err = -EINVAL; 91 goto out; 92 } 93 94 pool->index.max_index = max; 95 pool->index.min_index = min; 96 97 pool->index.table = bitmap_zalloc(max - min + 1, GFP_KERNEL); 98 if (!pool->index.table) { 99 err = -ENOMEM; 100 goto out; 101 } 102 103 out: 104 return err; 105 } 106 107 int rxe_pool_init( 108 struct rxe_dev *rxe, 109 struct rxe_pool *pool, 110 enum rxe_elem_type type, 111 unsigned int max_elem) 112 { 113 const struct rxe_type_info *info = &rxe_type_info[type]; 114 int err = 0; 115 116 memset(pool, 0, sizeof(*pool)); 117 118 pool->rxe = rxe; 119 pool->name = info->name; 120 pool->type = type; 121 pool->max_elem = max_elem; 122 pool->elem_size = ALIGN(info->size, RXE_POOL_ALIGN); 123 pool->elem_offset = info->elem_offset; 124 pool->flags = info->flags; 125 pool->cleanup = info->cleanup; 126 127 atomic_set(&pool->num_elem, 0); 128 129 rwlock_init(&pool->pool_lock); 130 131 if (pool->flags & RXE_POOL_INDEX) { 132 pool->index.tree = RB_ROOT; 133 err = rxe_pool_init_index(pool, info->max_index, 134 info->min_index); 135 if (err) 136 goto out; 137 } 138 139 out: 140 return err; 141 } 142 143 void rxe_pool_cleanup(struct rxe_pool *pool) 144 { 145 if (atomic_read(&pool->num_elem) > 0) 146 pr_warn("%s pool destroyed with unfree'd elem\n", 147 pool->name); 148 149 if (pool->flags & RXE_POOL_INDEX) 150 bitmap_free(pool->index.table); 151 } 152 153 static u32 alloc_index(struct rxe_pool *pool) 154 { 155 u32 index; 156 u32 range = pool->index.max_index - pool->index.min_index + 1; 157 158 index = find_next_zero_bit(pool->index.table, range, pool->index.last); 159 if (index >= range) 160 index = find_first_zero_bit(pool->index.table, range); 161 162 WARN_ON_ONCE(index >= range); 163 set_bit(index, pool->index.table); 164 pool->index.last = index; 165 return index + pool->index.min_index; 166 } 167 168 static int rxe_insert_index(struct rxe_pool *pool, struct rxe_pool_elem *new) 169 { 170 struct rb_node **link = &pool->index.tree.rb_node; 171 struct rb_node *parent = NULL; 172 struct rxe_pool_elem *elem; 173 174 while (*link) { 175 parent = *link; 176 elem = rb_entry(parent, struct rxe_pool_elem, index_node); 177 178 if (elem->index == new->index) { 179 pr_warn("element already exists!\n"); 180 return -EINVAL; 181 } 182 183 if (elem->index > new->index) 184 link = &(*link)->rb_left; 185 else 186 link = &(*link)->rb_right; 187 } 188 189 rb_link_node(&new->index_node, parent, link); 190 rb_insert_color(&new->index_node, &pool->index.tree); 191 192 return 0; 193 } 194 195 int __rxe_add_index_locked(struct rxe_pool_elem *elem) 196 { 197 struct rxe_pool *pool = elem->pool; 198 int err; 199 200 elem->index = alloc_index(pool); 201 err = rxe_insert_index(pool, elem); 202 203 return err; 204 } 205 206 int __rxe_add_index(struct rxe_pool_elem *elem) 207 { 208 struct rxe_pool *pool = elem->pool; 209 unsigned long flags; 210 int err; 211 212 write_lock_irqsave(&pool->pool_lock, flags); 213 err = __rxe_add_index_locked(elem); 214 write_unlock_irqrestore(&pool->pool_lock, flags); 215 216 return err; 217 } 218 219 void __rxe_drop_index_locked(struct rxe_pool_elem *elem) 220 { 221 struct rxe_pool *pool = elem->pool; 222 223 clear_bit(elem->index - pool->index.min_index, pool->index.table); 224 rb_erase(&elem->index_node, &pool->index.tree); 225 } 226 227 void __rxe_drop_index(struct rxe_pool_elem *elem) 228 { 229 struct rxe_pool *pool = elem->pool; 230 unsigned long flags; 231 232 write_lock_irqsave(&pool->pool_lock, flags); 233 __rxe_drop_index_locked(elem); 234 write_unlock_irqrestore(&pool->pool_lock, flags); 235 } 236 237 void *rxe_alloc_locked(struct rxe_pool *pool) 238 { 239 struct rxe_pool_elem *elem; 240 void *obj; 241 242 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 243 goto out_cnt; 244 245 obj = kzalloc(pool->elem_size, GFP_ATOMIC); 246 if (!obj) 247 goto out_cnt; 248 249 elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); 250 251 elem->pool = pool; 252 elem->obj = obj; 253 kref_init(&elem->ref_cnt); 254 255 return obj; 256 257 out_cnt: 258 atomic_dec(&pool->num_elem); 259 return NULL; 260 } 261 262 void *rxe_alloc(struct rxe_pool *pool) 263 { 264 struct rxe_pool_elem *elem; 265 void *obj; 266 267 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 268 goto out_cnt; 269 270 obj = kzalloc(pool->elem_size, GFP_KERNEL); 271 if (!obj) 272 goto out_cnt; 273 274 elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); 275 276 elem->pool = pool; 277 elem->obj = obj; 278 kref_init(&elem->ref_cnt); 279 280 return obj; 281 282 out_cnt: 283 atomic_dec(&pool->num_elem); 284 return NULL; 285 } 286 287 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem) 288 { 289 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 290 goto out_cnt; 291 292 elem->pool = pool; 293 elem->obj = (u8 *)elem - pool->elem_offset; 294 kref_init(&elem->ref_cnt); 295 296 return 0; 297 298 out_cnt: 299 atomic_dec(&pool->num_elem); 300 return -EINVAL; 301 } 302 303 void rxe_elem_release(struct kref *kref) 304 { 305 struct rxe_pool_elem *elem = 306 container_of(kref, struct rxe_pool_elem, ref_cnt); 307 struct rxe_pool *pool = elem->pool; 308 void *obj; 309 310 if (pool->cleanup) 311 pool->cleanup(elem); 312 313 if (!(pool->flags & RXE_POOL_NO_ALLOC)) { 314 obj = elem->obj; 315 kfree(obj); 316 } 317 318 atomic_dec(&pool->num_elem); 319 } 320 321 void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) 322 { 323 struct rb_node *node; 324 struct rxe_pool_elem *elem; 325 void *obj; 326 327 node = pool->index.tree.rb_node; 328 329 while (node) { 330 elem = rb_entry(node, struct rxe_pool_elem, index_node); 331 332 if (elem->index > index) 333 node = node->rb_left; 334 else if (elem->index < index) 335 node = node->rb_right; 336 else 337 break; 338 } 339 340 if (node) { 341 kref_get(&elem->ref_cnt); 342 obj = elem->obj; 343 } else { 344 obj = NULL; 345 } 346 347 return obj; 348 } 349 350 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) 351 { 352 unsigned long flags; 353 void *obj; 354 355 read_lock_irqsave(&pool->pool_lock, flags); 356 obj = rxe_pool_get_index_locked(pool, index); 357 read_unlock_irqrestore(&pool->pool_lock, flags); 358 359 return obj; 360 } 361