1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #include "rxe.h" 8 9 #define RXE_POOL_ALIGN (16) 10 11 static const struct rxe_type_info { 12 const char *name; 13 size_t size; 14 size_t elem_offset; 15 void (*cleanup)(struct rxe_pool_elem *elem); 16 u32 min_index; 17 u32 max_index; 18 u32 max_elem; 19 } rxe_type_info[RXE_NUM_TYPES] = { 20 [RXE_TYPE_UC] = { 21 .name = "uc", 22 .size = sizeof(struct rxe_ucontext), 23 .elem_offset = offsetof(struct rxe_ucontext, elem), 24 .min_index = 1, 25 .max_index = UINT_MAX, 26 .max_elem = UINT_MAX, 27 }, 28 [RXE_TYPE_PD] = { 29 .name = "pd", 30 .size = sizeof(struct rxe_pd), 31 .elem_offset = offsetof(struct rxe_pd, elem), 32 .min_index = 1, 33 .max_index = UINT_MAX, 34 .max_elem = UINT_MAX, 35 }, 36 [RXE_TYPE_AH] = { 37 .name = "ah", 38 .size = sizeof(struct rxe_ah), 39 .elem_offset = offsetof(struct rxe_ah, elem), 40 .min_index = RXE_MIN_AH_INDEX, 41 .max_index = RXE_MAX_AH_INDEX, 42 .max_elem = RXE_MAX_AH_INDEX - RXE_MIN_AH_INDEX + 1, 43 }, 44 [RXE_TYPE_SRQ] = { 45 .name = "srq", 46 .size = sizeof(struct rxe_srq), 47 .elem_offset = offsetof(struct rxe_srq, elem), 48 .min_index = RXE_MIN_SRQ_INDEX, 49 .max_index = RXE_MAX_SRQ_INDEX, 50 .max_elem = RXE_MAX_SRQ_INDEX - RXE_MIN_SRQ_INDEX + 1, 51 }, 52 [RXE_TYPE_QP] = { 53 .name = "qp", 54 .size = sizeof(struct rxe_qp), 55 .elem_offset = offsetof(struct rxe_qp, elem), 56 .cleanup = rxe_qp_cleanup, 57 .min_index = RXE_MIN_QP_INDEX, 58 .max_index = RXE_MAX_QP_INDEX, 59 .max_elem = RXE_MAX_QP_INDEX - RXE_MIN_QP_INDEX + 1, 60 }, 61 [RXE_TYPE_CQ] = { 62 .name = "cq", 63 .size = sizeof(struct rxe_cq), 64 .elem_offset = offsetof(struct rxe_cq, elem), 65 .cleanup = rxe_cq_cleanup, 66 .min_index = 1, 67 .max_index = UINT_MAX, 68 .max_elem = UINT_MAX, 69 }, 70 [RXE_TYPE_MR] = { 71 .name = "mr", 72 .size = sizeof(struct rxe_mr), 73 .elem_offset = offsetof(struct rxe_mr, elem), 74 .cleanup = rxe_mr_cleanup, 75 .min_index = RXE_MIN_MR_INDEX, 76 .max_index = RXE_MAX_MR_INDEX, 77 .max_elem = RXE_MAX_MR_INDEX - RXE_MIN_MR_INDEX + 1, 78 }, 79 [RXE_TYPE_MW] = { 80 .name = "mw", 81 .size = sizeof(struct rxe_mw), 82 .elem_offset = offsetof(struct rxe_mw, elem), 83 .min_index = RXE_MIN_MW_INDEX, 84 .max_index = RXE_MAX_MW_INDEX, 85 .max_elem = RXE_MAX_MW_INDEX - RXE_MIN_MW_INDEX + 1, 86 }, 87 }; 88 89 void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool, 90 enum rxe_elem_type type) 91 { 92 const struct rxe_type_info *info = &rxe_type_info[type]; 93 94 memset(pool, 0, sizeof(*pool)); 95 96 pool->rxe = rxe; 97 pool->name = info->name; 98 pool->type = type; 99 pool->max_elem = info->max_elem; 100 pool->elem_size = ALIGN(info->size, RXE_POOL_ALIGN); 101 pool->elem_offset = info->elem_offset; 102 pool->cleanup = info->cleanup; 103 104 atomic_set(&pool->num_elem, 0); 105 106 xa_init_flags(&pool->xa, XA_FLAGS_ALLOC); 107 pool->limit.min = info->min_index; 108 pool->limit.max = info->max_index; 109 } 110 111 void rxe_pool_cleanup(struct rxe_pool *pool) 112 { 113 WARN_ON(!xa_empty(&pool->xa)); 114 } 115 116 void *rxe_alloc(struct rxe_pool *pool) 117 { 118 struct rxe_pool_elem *elem; 119 void *obj; 120 int err; 121 122 if (WARN_ON(!(pool->type == RXE_TYPE_MR))) 123 return NULL; 124 125 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 126 goto err_cnt; 127 128 obj = kzalloc(pool->elem_size, GFP_KERNEL); 129 if (!obj) 130 goto err_cnt; 131 132 elem = (struct rxe_pool_elem *)((u8 *)obj + pool->elem_offset); 133 134 elem->pool = pool; 135 elem->obj = obj; 136 kref_init(&elem->ref_cnt); 137 138 err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit, 139 &pool->next, GFP_KERNEL); 140 if (err) 141 goto err_free; 142 143 return obj; 144 145 err_free: 146 kfree(obj); 147 err_cnt: 148 atomic_dec(&pool->num_elem); 149 return NULL; 150 } 151 152 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem) 153 { 154 int err; 155 156 if (WARN_ON(pool->type == RXE_TYPE_MR)) 157 return -EINVAL; 158 159 if (atomic_inc_return(&pool->num_elem) > pool->max_elem) 160 goto err_cnt; 161 162 elem->pool = pool; 163 elem->obj = (u8 *)elem - pool->elem_offset; 164 kref_init(&elem->ref_cnt); 165 166 err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit, 167 &pool->next, GFP_KERNEL); 168 if (err) 169 goto err_cnt; 170 171 return 0; 172 173 err_cnt: 174 atomic_dec(&pool->num_elem); 175 return -EINVAL; 176 } 177 178 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) 179 { 180 struct rxe_pool_elem *elem; 181 struct xarray *xa = &pool->xa; 182 unsigned long flags; 183 void *obj; 184 185 xa_lock_irqsave(xa, flags); 186 elem = xa_load(xa, index); 187 if (elem && kref_get_unless_zero(&elem->ref_cnt)) 188 obj = elem->obj; 189 else 190 obj = NULL; 191 xa_unlock_irqrestore(xa, flags); 192 193 return obj; 194 } 195 196 static void rxe_elem_release(struct kref *kref) 197 { 198 struct rxe_pool_elem *elem = container_of(kref, typeof(*elem), ref_cnt); 199 struct rxe_pool *pool = elem->pool; 200 201 xa_erase(&pool->xa, elem->index); 202 203 if (pool->cleanup) 204 pool->cleanup(elem); 205 206 if (pool->type == RXE_TYPE_MR) 207 kfree(elem->obj); 208 209 atomic_dec(&pool->num_elem); 210 } 211 212 int __rxe_get(struct rxe_pool_elem *elem) 213 { 214 return kref_get_unless_zero(&elem->ref_cnt); 215 } 216 217 int __rxe_put(struct rxe_pool_elem *elem) 218 { 219 return kref_put(&elem->ref_cnt, rxe_elem_release); 220 } 221