main.c (5fe9dec0d045437e48f112b8fa705197bd7bc3c0) main.c (b037c29a8056b8e896c4e084ba7cc30d6a1f165f)
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 978 unchanged lines hidden (view full) ---

987
988 err = mlx5_set_port_caps(dev->mdev, port, tmp);
989
990out:
991 mutex_unlock(&dev->cap_mask_mutex);
992 return err;
993}
994
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 978 unchanged lines hidden (view full) ---

987
988 err = mlx5_set_port_caps(dev->mdev, port, tmp);
989
990out:
991 mutex_unlock(&dev->cap_mask_mutex);
992 return err;
993}
994
995static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
996 struct mlx5_ib_alloc_ucontext_req_v2 *req,
997 u32 *num_sys_pages)
998{
999 int uars_per_sys_page;
1000 int bfregs_per_sys_page;
1001 int ref_bfregs = req->total_num_bfregs;
1002
1003 if (req->total_num_bfregs == 0)
1004 return -EINVAL;
1005
1006 BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1007 BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1008
1009 if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1010 return -ENOMEM;
1011
1012 uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1013 bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1014 req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1015 *num_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1016
1017 if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1018 return -EINVAL;
1019
1020 mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, alloated %d, using %d sys pages\n",
1021 MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1022 lib_uar_4k ? "yes" : "no", ref_bfregs,
1023 req->total_num_bfregs, *num_sys_pages);
1024
1025 return 0;
1026}
1027
1028static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1029{
1030 struct mlx5_bfreg_info *bfregi;
1031 int err;
1032 int i;
1033
1034 bfregi = &context->bfregi;
1035 for (i = 0; i < bfregi->num_sys_pages; i++) {
1036 err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1037 if (err)
1038 goto error;
1039
1040 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1041 }
1042 return 0;
1043
1044error:
1045 for (--i; i >= 0; i--)
1046 if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1047 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1048
1049 return err;
1050}
1051
1052static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1053{
1054 struct mlx5_bfreg_info *bfregi;
1055 int err;
1056 int i;
1057
1058 bfregi = &context->bfregi;
1059 for (i = 0; i < bfregi->num_sys_pages; i++) {
1060 err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1061 if (err) {
1062 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1063 return err;
1064 }
1065 }
1066 return 0;
1067}
1068
995static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
996 struct ib_udata *udata)
997{
998 struct mlx5_ib_dev *dev = to_mdev(ibdev);
999 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1000 struct mlx5_ib_alloc_ucontext_resp resp = {};
1001 struct mlx5_ib_ucontext *context;
1002 struct mlx5_bfreg_info *bfregi;
1069static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1070 struct ib_udata *udata)
1071{
1072 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1073 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1074 struct mlx5_ib_alloc_ucontext_resp resp = {};
1075 struct mlx5_ib_ucontext *context;
1076 struct mlx5_bfreg_info *bfregi;
1003 struct mlx5_uar *uars;
1004 int gross_bfregs;
1005 int num_uars;
1006 int ver;
1077 int ver;
1007 int bfregn;
1008 int err;
1078 int err;
1009 int i;
1010 size_t reqlen;
1011 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1012 max_cqe_version);
1079 size_t reqlen;
1080 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1081 max_cqe_version);
1082 bool lib_uar_4k;
1013
1014 if (!dev->ib_active)
1015 return ERR_PTR(-EAGAIN);
1016
1017 if (udata->inlen < sizeof(struct ib_uverbs_cmd_hdr))
1018 return ERR_PTR(-EINVAL);
1019
1020 reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);

--- 6 unchanged lines hidden (view full) ---

1027
1028 err = ib_copy_from_udata(&req, udata, min(reqlen, sizeof(req)));
1029 if (err)
1030 return ERR_PTR(err);
1031
1032 if (req.flags)
1033 return ERR_PTR(-EINVAL);
1034
1083
1084 if (!dev->ib_active)
1085 return ERR_PTR(-EAGAIN);
1086
1087 if (udata->inlen < sizeof(struct ib_uverbs_cmd_hdr))
1088 return ERR_PTR(-EINVAL);
1089
1090 reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);

--- 6 unchanged lines hidden (view full) ---

1097
1098 err = ib_copy_from_udata(&req, udata, min(reqlen, sizeof(req)));
1099 if (err)
1100 return ERR_PTR(err);
1101
1102 if (req.flags)
1103 return ERR_PTR(-EINVAL);
1104
1035 if (req.total_num_bfregs > MLX5_MAX_BFREGS)
1036 return ERR_PTR(-ENOMEM);
1037
1038 if (req.total_num_bfregs == 0)
1039 return ERR_PTR(-EINVAL);
1040
1041 if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1042 return ERR_PTR(-EOPNOTSUPP);
1043
1105 if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1106 return ERR_PTR(-EOPNOTSUPP);
1107
1044 if (reqlen > sizeof(req) &&
1045 !ib_is_udata_cleared(udata, sizeof(req),
1046 reqlen - sizeof(req)))
1047 return ERR_PTR(-EOPNOTSUPP);
1048
1049 req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1050 MLX5_NON_FP_BFREGS_PER_UAR);
1051 if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1052 return ERR_PTR(-EINVAL);
1053
1108 req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1109 MLX5_NON_FP_BFREGS_PER_UAR);
1110 if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1111 return ERR_PTR(-EINVAL);
1112
1054 num_uars = req.total_num_bfregs / MLX5_NON_FP_BFREGS_PER_UAR;
1055 gross_bfregs = num_uars * MLX5_BFREGS_PER_UAR;
1056 resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1057 if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
1058 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
1059 resp.cache_line_size = cache_line_size();
1060 resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1061 resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1062 resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1063 resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1064 resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1065 resp.cqe_version = min_t(__u8,
1066 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1067 req.max_cqe_version);
1068 resp.response_length = min(offsetof(typeof(resp), response_length) +
1069 sizeof(resp.response_length), udata->outlen);
1070
1071 context = kzalloc(sizeof(*context), GFP_KERNEL);
1072 if (!context)
1073 return ERR_PTR(-ENOMEM);
1074
1113 resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1114 if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
1115 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
1116 resp.cache_line_size = cache_line_size();
1117 resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1118 resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1119 resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1120 resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1121 resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1122 resp.cqe_version = min_t(__u8,
1123 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1124 req.max_cqe_version);
1125 resp.response_length = min(offsetof(typeof(resp), response_length) +
1126 sizeof(resp.response_length), udata->outlen);
1127
1128 context = kzalloc(sizeof(*context), GFP_KERNEL);
1129 if (!context)
1130 return ERR_PTR(-ENOMEM);
1131
1132 lib_uar_4k = false;
1075 bfregi = &context->bfregi;
1133 bfregi = &context->bfregi;
1076 mutex_init(&bfregi->lock);
1077 uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL);
1078 if (!uars) {
1079 err = -ENOMEM;
1134
1135 /* updates req->total_num_bfregs */
1136 err = calc_total_bfregs(dev, lib_uar_4k, &req, &bfregi->num_sys_pages);
1137 if (err)
1080 goto out_ctx;
1138 goto out_ctx;
1081 }
1082
1139
1083 bfregi->bitmap = kcalloc(BITS_TO_LONGS(gross_bfregs),
1084 sizeof(*bfregi->bitmap),
1140 mutex_init(&bfregi->lock);
1141 bfregi->lib_uar_4k = lib_uar_4k;
1142 bfregi->count = kcalloc(req.total_num_bfregs, sizeof(*bfregi->count),
1085 GFP_KERNEL);
1143 GFP_KERNEL);
1086 if (!bfregi->bitmap) {
1144 if (!bfregi->count) {
1087 err = -ENOMEM;
1145 err = -ENOMEM;
1088 goto out_uar_ctx;
1146 goto out_ctx;
1089 }
1147 }
1090 /*
1091 * clear all fast path bfregs
1092 */
1093 for (i = 0; i < gross_bfregs; i++) {
1094 bfregn = i & 3;
1095 if (bfregn == 2 || bfregn == 3)
1096 set_bit(i, bfregi->bitmap);
1097 }
1098
1148
1099 bfregi->count = kcalloc(gross_bfregs,
1100 sizeof(*bfregi->count), GFP_KERNEL);
1101 if (!bfregi->count) {
1149 bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1150 sizeof(*bfregi->sys_pages),
1151 GFP_KERNEL);
1152 if (!bfregi->sys_pages) {
1102 err = -ENOMEM;
1153 err = -ENOMEM;
1103 goto out_bitmap;
1154 goto out_count;
1104 }
1105
1155 }
1156
1106 for (i = 0; i < num_uars; i++) {
1107 err = mlx5_cmd_alloc_uar(dev->mdev, &uars[i].index);
1108 if (err)
1109 goto out_count;
1110 }
1157 err = allocate_uars(dev, context);
1158 if (err)
1159 goto out_sys_pages;
1111
1112#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1113 context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
1114#endif
1115
1116 context->upd_xlt_page = __get_free_page(GFP_KERNEL);
1117 if (!context->upd_xlt_page) {
1118 err = -ENOMEM;

--- 42 unchanged lines hidden (view full) ---

1161 }
1162
1163 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1164 if (err)
1165 goto out_td;
1166
1167 bfregi->ver = ver;
1168 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1160
1161#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1162 context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
1163#endif
1164
1165 context->upd_xlt_page = __get_free_page(GFP_KERNEL);
1166 if (!context->upd_xlt_page) {
1167 err = -ENOMEM;

--- 42 unchanged lines hidden (view full) ---

1210 }
1211
1212 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1213 if (err)
1214 goto out_td;
1215
1216 bfregi->ver = ver;
1217 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1169 bfregi->uars = uars;
1170 bfregi->num_uars = num_uars;
1171 context->cqe_version = resp.cqe_version;
1218 context->cqe_version = resp.cqe_version;
1219 context->lib_caps = false;
1172
1173 return &context->ibucontext;
1174
1175out_td:
1176 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1177 mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
1178
1179out_page:
1180 free_page(context->upd_xlt_page);
1181
1182out_uars:
1220
1221 return &context->ibucontext;
1222
1223out_td:
1224 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1225 mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
1226
1227out_page:
1228 free_page(context->upd_xlt_page);
1229
1230out_uars:
1183 for (i--; i >= 0; i--)
1184 mlx5_cmd_free_uar(dev->mdev, uars[i].index);
1231 deallocate_uars(dev, context);
1232
1233out_sys_pages:
1234 kfree(bfregi->sys_pages);
1235
1185out_count:
1186 kfree(bfregi->count);
1187
1236out_count:
1237 kfree(bfregi->count);
1238
1188out_bitmap:
1189 kfree(bfregi->bitmap);
1190
1191out_uar_ctx:
1192 kfree(uars);
1193
1194out_ctx:
1195 kfree(context);
1239out_ctx:
1240 kfree(context);
1241
1196 return ERR_PTR(err);
1197}
1198
1199static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1200{
1201 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1202 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1242 return ERR_PTR(err);
1243}
1244
1245static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1246{
1247 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1248 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1203 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1204 int i;
1249 struct mlx5_bfreg_info *bfregi;
1205
1250
1251 bfregi = &context->bfregi;
1206 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1207 mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
1208
1209 free_page(context->upd_xlt_page);
1252 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1253 mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
1254
1255 free_page(context->upd_xlt_page);
1210
1211 for (i = 0; i < bfregi->num_uars; i++) {
1212 if (mlx5_cmd_free_uar(dev->mdev, bfregi->uars[i].index))
1213 mlx5_ib_warn(dev, "Failed to free UAR 0x%x\n",
1214 bfregi->uars[i].index);
1215 }
1216
1256 deallocate_uars(dev, context);
1257 kfree(bfregi->sys_pages);
1217 kfree(bfregi->count);
1258 kfree(bfregi->count);
1218 kfree(bfregi->bitmap);
1219 kfree(bfregi->uars);
1220 kfree(context);
1221
1222 return 0;
1223}
1224
1259 kfree(context);
1260
1261 return 0;
1262}
1263
1225static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index)
1264static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
1265 struct mlx5_bfreg_info *bfregi,
1266 int idx)
1226{
1267{
1227 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + index;
1268 int fw_uars_per_page;
1269
1270 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
1271
1272 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) +
1273 bfregi->sys_pages[idx] / fw_uars_per_page;
1228}
1229
1230static int get_command(unsigned long offset)
1231{
1232 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
1233}
1234
1235static int get_arg(unsigned long offset)

--- 143 unchanged lines hidden (view full) ---

1379 struct vm_area_struct *vma,
1380 struct mlx5_ib_ucontext *context)
1381{
1382 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1383 int err;
1384 unsigned long idx;
1385 phys_addr_t pfn, pa;
1386 pgprot_t prot;
1274}
1275
1276static int get_command(unsigned long offset)
1277{
1278 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
1279}
1280
1281static int get_arg(unsigned long offset)

--- 143 unchanged lines hidden (view full) ---

1425 struct vm_area_struct *vma,
1426 struct mlx5_ib_ucontext *context)
1427{
1428 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1429 int err;
1430 unsigned long idx;
1431 phys_addr_t pfn, pa;
1432 pgprot_t prot;
1433 int uars_per_page;
1387
1434
1435 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1436 return -EINVAL;
1437
1438 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
1439 idx = get_index(vma->vm_pgoff);
1440 if (idx % uars_per_page ||
1441 idx * uars_per_page >= bfregi->num_sys_pages) {
1442 mlx5_ib_warn(dev, "invalid uar index %lu\n", idx);
1443 return -EINVAL;
1444 }
1445
1388 switch (cmd) {
1389 case MLX5_IB_MMAP_WC_PAGE:
1390/* Some architectures don't support WC memory */
1391#if defined(CONFIG_X86)
1392 if (!pat_enabled())
1393 return -EPERM;
1394#elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
1395 return -EPERM;

--- 5 unchanged lines hidden (view full) ---

1401 break;
1402 case MLX5_IB_MMAP_NC_PAGE:
1403 prot = pgprot_noncached(vma->vm_page_prot);
1404 break;
1405 default:
1406 return -EINVAL;
1407 }
1408
1446 switch (cmd) {
1447 case MLX5_IB_MMAP_WC_PAGE:
1448/* Some architectures don't support WC memory */
1449#if defined(CONFIG_X86)
1450 if (!pat_enabled())
1451 return -EPERM;
1452#elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
1453 return -EPERM;

--- 5 unchanged lines hidden (view full) ---

1459 break;
1460 case MLX5_IB_MMAP_NC_PAGE:
1461 prot = pgprot_noncached(vma->vm_page_prot);
1462 break;
1463 default:
1464 return -EINVAL;
1465 }
1466
1409 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1410 return -EINVAL;
1411
1412 idx = get_index(vma->vm_pgoff);
1413 if (idx >= bfregi->num_uars)
1414 return -EINVAL;
1415
1416 pfn = uar_index2pfn(dev, bfregi->uars[idx].index);
1467 pfn = uar_index2pfn(dev, bfregi, idx);
1417 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
1418
1419 vma->vm_page_prot = prot;
1420 err = io_remap_pfn_range(vma, vma->vm_start, pfn,
1421 PAGE_SIZE, vma->vm_page_prot);
1422 if (err) {
1423 mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%lx, pfn=%pa, mmap_cmd=%s\n",
1424 err, vma->vm_start, &pfn, mmap_cmd2str(cmd));

--- 1946 unchanged lines hidden --- 		1468 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
1469
1470 vma->vm_page_prot = prot;
1471 err = io_remap_pfn_range(vma, vma->vm_start, pfn,
1472 PAGE_SIZE, vma->vm_page_prot);
1473 if (err) {
1474 mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%lx, pfn=%pa, mmap_cmd=%s\n",
1475 err, vma->vm_start, &pfn, mmap_cmd2str(cmd));

--- 1946 unchanged lines hidden ---