1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 3 4 #include <linux/mlx5/driver.h> 5 #include "mlx5_vdpa.h" 6 7 static int alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid) 8 { 9 struct mlx5_core_dev *mdev = dev->mdev; 10 11 u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {}; 12 u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {}; 13 int err; 14 15 MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD); 16 MLX5_SET(alloc_pd_in, in, uid, uid); 17 18 err = mlx5_cmd_exec_inout(mdev, alloc_pd, in, out); 19 if (!err) 20 *pdn = MLX5_GET(alloc_pd_out, out, pd); 21 22 return err; 23 } 24 25 static int dealloc_pd(struct mlx5_vdpa_dev *dev, u32 pdn, u16 uid) 26 { 27 u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {}; 28 struct mlx5_core_dev *mdev = dev->mdev; 29 30 MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD); 31 MLX5_SET(dealloc_pd_in, in, pd, pdn); 32 MLX5_SET(dealloc_pd_in, in, uid, uid); 33 return mlx5_cmd_exec_in(mdev, dealloc_pd, in); 34 } 35 36 static int get_null_mkey(struct mlx5_vdpa_dev *dev, u32 *null_mkey) 37 { 38 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {}; 39 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {}; 40 struct mlx5_core_dev *mdev = dev->mdev; 41 int err; 42 43 MLX5_SET(query_special_contexts_in, in, opcode, MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); 44 err = mlx5_cmd_exec_inout(mdev, query_special_contexts, in, out); 45 if (!err) 46 *null_mkey = MLX5_GET(query_special_contexts_out, out, null_mkey); 47 return err; 48 } 49 50 static int create_uctx(struct mlx5_vdpa_dev *mvdev, u16 *uid) 51 { 52 u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {}; 53 int inlen; 54 void *in; 55 int err; 56 57 /* 0 means not supported */ 58 if (!MLX5_CAP_GEN(mvdev->mdev, log_max_uctx)) 59 return -EOPNOTSUPP; 60 61 inlen = MLX5_ST_SZ_BYTES(create_uctx_in); 62 in = kzalloc(inlen, GFP_KERNEL); 63 if (!in) 64 return -ENOMEM; 65 66 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX); 67 MLX5_SET(create_uctx_in, in, uctx.cap, MLX5_UCTX_CAP_RAW_TX); 68 69 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out)); 70 kfree(in); 71 if (!err) 72 *uid = MLX5_GET(create_uctx_out, out, uid); 73 74 return err; 75 } 76 77 static void destroy_uctx(struct mlx5_vdpa_dev *mvdev, u32 uid) 78 { 79 u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {}; 80 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {}; 81 82 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX); 83 MLX5_SET(destroy_uctx_in, in, uid, uid); 84 85 mlx5_cmd_exec(mvdev->mdev, in, sizeof(in), out, sizeof(out)); 86 } 87 88 int mlx5_vdpa_create_tis(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tisn) 89 { 90 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {}; 91 int err; 92 93 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); 94 MLX5_SET(create_tis_in, in, uid, mvdev->res.uid); 95 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tis, in, out); 96 if (!err) 97 *tisn = MLX5_GET(create_tis_out, out, tisn); 98 99 return err; 100 } 101 102 void mlx5_vdpa_destroy_tis(struct mlx5_vdpa_dev *mvdev, u32 tisn) 103 { 104 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {}; 105 106 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); 107 MLX5_SET(destroy_tis_in, in, uid, mvdev->res.uid); 108 MLX5_SET(destroy_tis_in, in, tisn, tisn); 109 mlx5_cmd_exec_in(mvdev->mdev, destroy_tis, in); 110 } 111 112 int mlx5_vdpa_create_rqt(struct mlx5_vdpa_dev *mvdev, void *in, int inlen, u32 *rqtn) 113 { 114 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {}; 115 int err; 116 117 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT); 118 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, out, sizeof(out)); 119 if (!err) 120 *rqtn = MLX5_GET(create_rqt_out, out, rqtn); 121 122 return err; 123 } 124 125 void mlx5_vdpa_destroy_rqt(struct mlx5_vdpa_dev *mvdev, u32 rqtn) 126 { 127 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {}; 128 129 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); 130 MLX5_SET(destroy_rqt_in, in, uid, mvdev->res.uid); 131 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn); 132 mlx5_cmd_exec_in(mvdev->mdev, destroy_rqt, in); 133 } 134 135 int mlx5_vdpa_create_tir(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tirn) 136 { 137 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {}; 138 int err; 139 140 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR); 141 err = mlx5_cmd_exec_inout(mvdev->mdev, create_tir, in, out); 142 if (!err) 143 *tirn = MLX5_GET(create_tir_out, out, tirn); 144 145 return err; 146 } 147 148 void mlx5_vdpa_destroy_tir(struct mlx5_vdpa_dev *mvdev, u32 tirn) 149 { 150 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {}; 151 152 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR); 153 MLX5_SET(destroy_tir_in, in, uid, mvdev->res.uid); 154 MLX5_SET(destroy_tir_in, in, tirn, tirn); 155 mlx5_cmd_exec_in(mvdev->mdev, destroy_tir, in); 156 } 157 158 int mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 *tdn) 159 { 160 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {}; 161 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {}; 162 int err; 163 164 MLX5_SET(alloc_transport_domain_in, in, opcode, MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN); 165 MLX5_SET(alloc_transport_domain_in, in, uid, mvdev->res.uid); 166 167 err = mlx5_cmd_exec_inout(mvdev->mdev, alloc_transport_domain, in, out); 168 if (!err) 169 *tdn = MLX5_GET(alloc_transport_domain_out, out, transport_domain); 170 171 return err; 172 } 173 174 void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn) 175 { 176 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {}; 177 178 MLX5_SET(dealloc_transport_domain_in, in, opcode, MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 179 MLX5_SET(dealloc_transport_domain_in, in, uid, mvdev->res.uid); 180 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn); 181 mlx5_cmd_exec_in(mvdev->mdev, dealloc_transport_domain, in); 182 } 183 184 int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey, u32 *in, 185 int inlen) 186 { 187 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {}; 188 u32 mkey_index; 189 void *mkc; 190 int err; 191 192 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); 193 MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid); 194 195 err = mlx5_cmd_exec(mvdev->mdev, in, inlen, lout, sizeof(lout)); 196 if (err) 197 return err; 198 199 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 200 mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index); 201 mkey->iova = MLX5_GET64(mkc, mkc, start_addr); 202 mkey->size = MLX5_GET64(mkc, mkc, len); 203 mkey->key |= mlx5_idx_to_mkey(mkey_index); 204 mkey->pd = MLX5_GET(mkc, mkc, pd); 205 return 0; 206 } 207 208 int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey) 209 { 210 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {}; 211 212 MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid); 213 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); 214 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key)); 215 return mlx5_cmd_exec_in(mvdev->mdev, destroy_mkey, in); 216 } 217 218 int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev) 219 { 220 u64 offset = MLX5_CAP64_DEV_VDPA_EMULATION(mvdev->mdev, doorbell_bar_offset); 221 struct mlx5_vdpa_resources *res = &mvdev->res; 222 struct mlx5_core_dev *mdev = mvdev->mdev; 223 u64 kick_addr; 224 int err; 225 226 if (res->valid) { 227 mlx5_vdpa_warn(mvdev, "resources already allocated\n"); 228 return -EINVAL; 229 } 230 mutex_init(&mvdev->mr.mkey_mtx); 231 res->uar = mlx5_get_uars_page(mdev); 232 if (IS_ERR(res->uar)) { 233 err = PTR_ERR(res->uar); 234 goto err_uars; 235 } 236 237 err = create_uctx(mvdev, &res->uid); 238 if (err) 239 goto err_uctx; 240 241 err = alloc_pd(mvdev, &res->pdn, res->uid); 242 if (err) 243 goto err_pd; 244 245 err = get_null_mkey(mvdev, &res->null_mkey); 246 if (err) 247 goto err_key; 248 249 kick_addr = mdev->bar_addr + offset; 250 251 res->kick_addr = ioremap(kick_addr, PAGE_SIZE); 252 if (!res->kick_addr) { 253 err = -ENOMEM; 254 goto err_key; 255 } 256 res->valid = true; 257 258 return 0; 259 260 err_key: 261 dealloc_pd(mvdev, res->pdn, res->uid); 262 err_pd: 263 destroy_uctx(mvdev, res->uid); 264 err_uctx: 265 mlx5_put_uars_page(mdev, res->uar); 266 err_uars: 267 mutex_destroy(&mvdev->mr.mkey_mtx); 268 return err; 269 } 270 271 void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev) 272 { 273 struct mlx5_vdpa_resources *res = &mvdev->res; 274 275 if (!res->valid) 276 return; 277 278 iounmap(res->kick_addr); 279 res->kick_addr = NULL; 280 dealloc_pd(mvdev, res->pdn, res->uid); 281 destroy_uctx(mvdev, res->uid); 282 mlx5_put_uars_page(mvdev->mdev, res->uar); 283 mutex_destroy(&mvdev->mr.mkey_mtx); 284 res->valid = false; 285 } 286