11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. 31da177e4SLinus Torvalds * Copyright (c) 2004 Infinicon Corporation. All rights reserved. 41da177e4SLinus Torvalds * Copyright (c) 2004 Intel Corporation. All rights reserved. 51da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Corporation. All rights reserved. 61da177e4SLinus Torvalds * Copyright (c) 2004 Voltaire Corporation. All rights reserved. 72a1d9b7fSRoland Dreier * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 833b9b3eeSRoland Dreier * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * This software is available to you under a choice of one of two 111da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 121da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 131da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 141da177e4SLinus Torvalds * OpenIB.org BSD license below: 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 171da177e4SLinus Torvalds * without modification, are permitted provided that the following 181da177e4SLinus Torvalds * conditions are met: 191da177e4SLinus Torvalds * 201da177e4SLinus Torvalds * - Redistributions of source code must retain the above 211da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 221da177e4SLinus Torvalds * disclaimer. 231da177e4SLinus Torvalds * 241da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 251da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 261da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 271da177e4SLinus Torvalds * provided with the distribution. 281da177e4SLinus Torvalds * 291da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 301da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 311da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 321da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 331da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 341da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 351da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 361da177e4SLinus Torvalds * SOFTWARE. 371da177e4SLinus Torvalds */ 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds #include <linux/errno.h> 401da177e4SLinus Torvalds #include <linux/err.h> 41b108d976SPaul Gortmaker #include <linux/export.h> 428c65b4a6STim Schmielau #include <linux/string.h> 430e0ec7e0SSean Hefty #include <linux/slab.h> 44dbf727deSMatan Barak #include <linux/in.h> 45dbf727deSMatan Barak #include <linux/in6.h> 46dbf727deSMatan Barak #include <net/addrconf.h> 471da177e4SLinus Torvalds 48a4d61e84SRoland Dreier #include <rdma/ib_verbs.h> 49a4d61e84SRoland Dreier #include <rdma/ib_cache.h> 50dd5f03beSMatan Barak #include <rdma/ib_addr.h> 511da177e4SLinus Torvalds 52ed4c54e5SOr Gerlitz #include "core_priv.h" 531da177e4SLinus Torvalds 542b1b5b60SSagi Grimberg static const char * const ib_events[] = { 552b1b5b60SSagi Grimberg [IB_EVENT_CQ_ERR] = "CQ error", 562b1b5b60SSagi Grimberg [IB_EVENT_QP_FATAL] = "QP fatal error", 572b1b5b60SSagi Grimberg [IB_EVENT_QP_REQ_ERR] = "QP request error", 582b1b5b60SSagi Grimberg [IB_EVENT_QP_ACCESS_ERR] = "QP access error", 592b1b5b60SSagi Grimberg [IB_EVENT_COMM_EST] = "communication established", 602b1b5b60SSagi Grimberg [IB_EVENT_SQ_DRAINED] = "send queue drained", 612b1b5b60SSagi Grimberg [IB_EVENT_PATH_MIG] = "path migration successful", 622b1b5b60SSagi Grimberg [IB_EVENT_PATH_MIG_ERR] = "path migration error", 632b1b5b60SSagi Grimberg [IB_EVENT_DEVICE_FATAL] = "device fatal error", 642b1b5b60SSagi Grimberg [IB_EVENT_PORT_ACTIVE] = "port active", 652b1b5b60SSagi Grimberg [IB_EVENT_PORT_ERR] = "port error", 662b1b5b60SSagi Grimberg [IB_EVENT_LID_CHANGE] = "LID change", 672b1b5b60SSagi Grimberg [IB_EVENT_PKEY_CHANGE] = "P_key change", 682b1b5b60SSagi Grimberg [IB_EVENT_SM_CHANGE] = "SM change", 692b1b5b60SSagi Grimberg [IB_EVENT_SRQ_ERR] = "SRQ error", 702b1b5b60SSagi Grimberg [IB_EVENT_SRQ_LIMIT_REACHED] = "SRQ limit reached", 712b1b5b60SSagi Grimberg [IB_EVENT_QP_LAST_WQE_REACHED] = "last WQE reached", 722b1b5b60SSagi Grimberg [IB_EVENT_CLIENT_REREGISTER] = "client reregister", 732b1b5b60SSagi Grimberg [IB_EVENT_GID_CHANGE] = "GID changed", 742b1b5b60SSagi Grimberg }; 752b1b5b60SSagi Grimberg 76db7489e0SBart Van Assche const char *__attribute_const__ ib_event_msg(enum ib_event_type event) 772b1b5b60SSagi Grimberg { 782b1b5b60SSagi Grimberg size_t index = event; 792b1b5b60SSagi Grimberg 802b1b5b60SSagi Grimberg return (index < ARRAY_SIZE(ib_events) && ib_events[index]) ? 812b1b5b60SSagi Grimberg ib_events[index] : "unrecognized event"; 822b1b5b60SSagi Grimberg } 832b1b5b60SSagi Grimberg EXPORT_SYMBOL(ib_event_msg); 842b1b5b60SSagi Grimberg 852b1b5b60SSagi Grimberg static const char * const wc_statuses[] = { 862b1b5b60SSagi Grimberg [IB_WC_SUCCESS] = "success", 872b1b5b60SSagi Grimberg [IB_WC_LOC_LEN_ERR] = "local length error", 882b1b5b60SSagi Grimberg [IB_WC_LOC_QP_OP_ERR] = "local QP operation error", 892b1b5b60SSagi Grimberg [IB_WC_LOC_EEC_OP_ERR] = "local EE context operation error", 902b1b5b60SSagi Grimberg [IB_WC_LOC_PROT_ERR] = "local protection error", 912b1b5b60SSagi Grimberg [IB_WC_WR_FLUSH_ERR] = "WR flushed", 922b1b5b60SSagi Grimberg [IB_WC_MW_BIND_ERR] = "memory management operation error", 932b1b5b60SSagi Grimberg [IB_WC_BAD_RESP_ERR] = "bad response error", 942b1b5b60SSagi Grimberg [IB_WC_LOC_ACCESS_ERR] = "local access error", 952b1b5b60SSagi Grimberg [IB_WC_REM_INV_REQ_ERR] = "invalid request error", 962b1b5b60SSagi Grimberg [IB_WC_REM_ACCESS_ERR] = "remote access error", 972b1b5b60SSagi Grimberg [IB_WC_REM_OP_ERR] = "remote operation error", 982b1b5b60SSagi Grimberg [IB_WC_RETRY_EXC_ERR] = "transport retry counter exceeded", 992b1b5b60SSagi Grimberg [IB_WC_RNR_RETRY_EXC_ERR] = "RNR retry counter exceeded", 1002b1b5b60SSagi Grimberg [IB_WC_LOC_RDD_VIOL_ERR] = "local RDD violation error", 1012b1b5b60SSagi Grimberg [IB_WC_REM_INV_RD_REQ_ERR] = "remote invalid RD request", 1022b1b5b60SSagi Grimberg [IB_WC_REM_ABORT_ERR] = "operation aborted", 1032b1b5b60SSagi Grimberg [IB_WC_INV_EECN_ERR] = "invalid EE context number", 1042b1b5b60SSagi Grimberg [IB_WC_INV_EEC_STATE_ERR] = "invalid EE context state", 1052b1b5b60SSagi Grimberg [IB_WC_FATAL_ERR] = "fatal error", 1062b1b5b60SSagi Grimberg [IB_WC_RESP_TIMEOUT_ERR] = "response timeout error", 1072b1b5b60SSagi Grimberg [IB_WC_GENERAL_ERR] = "general error", 1082b1b5b60SSagi Grimberg }; 1092b1b5b60SSagi Grimberg 110db7489e0SBart Van Assche const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status) 1112b1b5b60SSagi Grimberg { 1122b1b5b60SSagi Grimberg size_t index = status; 1132b1b5b60SSagi Grimberg 1142b1b5b60SSagi Grimberg return (index < ARRAY_SIZE(wc_statuses) && wc_statuses[index]) ? 1152b1b5b60SSagi Grimberg wc_statuses[index] : "unrecognized status"; 1162b1b5b60SSagi Grimberg } 1172b1b5b60SSagi Grimberg EXPORT_SYMBOL(ib_wc_status_msg); 1182b1b5b60SSagi Grimberg 1198385fd84SRoland Dreier __attribute_const__ int ib_rate_to_mult(enum ib_rate rate) 120bf6a9e31SJack Morgenstein { 121bf6a9e31SJack Morgenstein switch (rate) { 122bf6a9e31SJack Morgenstein case IB_RATE_2_5_GBPS: return 1; 123bf6a9e31SJack Morgenstein case IB_RATE_5_GBPS: return 2; 124bf6a9e31SJack Morgenstein case IB_RATE_10_GBPS: return 4; 125bf6a9e31SJack Morgenstein case IB_RATE_20_GBPS: return 8; 126bf6a9e31SJack Morgenstein case IB_RATE_30_GBPS: return 12; 127bf6a9e31SJack Morgenstein case IB_RATE_40_GBPS: return 16; 128bf6a9e31SJack Morgenstein case IB_RATE_60_GBPS: return 24; 129bf6a9e31SJack Morgenstein case IB_RATE_80_GBPS: return 32; 130bf6a9e31SJack Morgenstein case IB_RATE_120_GBPS: return 48; 131bf6a9e31SJack Morgenstein default: return -1; 132bf6a9e31SJack Morgenstein } 133bf6a9e31SJack Morgenstein } 134bf6a9e31SJack Morgenstein EXPORT_SYMBOL(ib_rate_to_mult); 135bf6a9e31SJack Morgenstein 1368385fd84SRoland Dreier __attribute_const__ enum ib_rate mult_to_ib_rate(int mult) 137bf6a9e31SJack Morgenstein { 138bf6a9e31SJack Morgenstein switch (mult) { 139bf6a9e31SJack Morgenstein case 1: return IB_RATE_2_5_GBPS; 140bf6a9e31SJack Morgenstein case 2: return IB_RATE_5_GBPS; 141bf6a9e31SJack Morgenstein case 4: return IB_RATE_10_GBPS; 142bf6a9e31SJack Morgenstein case 8: return IB_RATE_20_GBPS; 143bf6a9e31SJack Morgenstein case 12: return IB_RATE_30_GBPS; 144bf6a9e31SJack Morgenstein case 16: return IB_RATE_40_GBPS; 145bf6a9e31SJack Morgenstein case 24: return IB_RATE_60_GBPS; 146bf6a9e31SJack Morgenstein case 32: return IB_RATE_80_GBPS; 147bf6a9e31SJack Morgenstein case 48: return IB_RATE_120_GBPS; 148bf6a9e31SJack Morgenstein default: return IB_RATE_PORT_CURRENT; 149bf6a9e31SJack Morgenstein } 150bf6a9e31SJack Morgenstein } 151bf6a9e31SJack Morgenstein EXPORT_SYMBOL(mult_to_ib_rate); 152bf6a9e31SJack Morgenstein 1538385fd84SRoland Dreier __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate) 15471eeba16SMarcel Apfelbaum { 15571eeba16SMarcel Apfelbaum switch (rate) { 15671eeba16SMarcel Apfelbaum case IB_RATE_2_5_GBPS: return 2500; 15771eeba16SMarcel Apfelbaum case IB_RATE_5_GBPS: return 5000; 15871eeba16SMarcel Apfelbaum case IB_RATE_10_GBPS: return 10000; 15971eeba16SMarcel Apfelbaum case IB_RATE_20_GBPS: return 20000; 16071eeba16SMarcel Apfelbaum case IB_RATE_30_GBPS: return 30000; 16171eeba16SMarcel Apfelbaum case IB_RATE_40_GBPS: return 40000; 16271eeba16SMarcel Apfelbaum case IB_RATE_60_GBPS: return 60000; 16371eeba16SMarcel Apfelbaum case IB_RATE_80_GBPS: return 80000; 16471eeba16SMarcel Apfelbaum case IB_RATE_120_GBPS: return 120000; 16571eeba16SMarcel Apfelbaum case IB_RATE_14_GBPS: return 14062; 16671eeba16SMarcel Apfelbaum case IB_RATE_56_GBPS: return 56250; 16771eeba16SMarcel Apfelbaum case IB_RATE_112_GBPS: return 112500; 16871eeba16SMarcel Apfelbaum case IB_RATE_168_GBPS: return 168750; 16971eeba16SMarcel Apfelbaum case IB_RATE_25_GBPS: return 25781; 17071eeba16SMarcel Apfelbaum case IB_RATE_100_GBPS: return 103125; 17171eeba16SMarcel Apfelbaum case IB_RATE_200_GBPS: return 206250; 17271eeba16SMarcel Apfelbaum case IB_RATE_300_GBPS: return 309375; 17371eeba16SMarcel Apfelbaum default: return -1; 17471eeba16SMarcel Apfelbaum } 17571eeba16SMarcel Apfelbaum } 17671eeba16SMarcel Apfelbaum EXPORT_SYMBOL(ib_rate_to_mbps); 17771eeba16SMarcel Apfelbaum 1788385fd84SRoland Dreier __attribute_const__ enum rdma_transport_type 17907ebafbaSTom Tucker rdma_node_get_transport(enum rdma_node_type node_type) 18007ebafbaSTom Tucker { 18107ebafbaSTom Tucker switch (node_type) { 18207ebafbaSTom Tucker case RDMA_NODE_IB_CA: 18307ebafbaSTom Tucker case RDMA_NODE_IB_SWITCH: 18407ebafbaSTom Tucker case RDMA_NODE_IB_ROUTER: 18507ebafbaSTom Tucker return RDMA_TRANSPORT_IB; 18607ebafbaSTom Tucker case RDMA_NODE_RNIC: 18707ebafbaSTom Tucker return RDMA_TRANSPORT_IWARP; 188180771a3SUpinder Malhi \(umalhi\) case RDMA_NODE_USNIC: 1895db5765eSUpinder Malhi return RDMA_TRANSPORT_USNIC; 1905db5765eSUpinder Malhi case RDMA_NODE_USNIC_UDP: 191248567f7SUpinder Malhi return RDMA_TRANSPORT_USNIC_UDP; 19207ebafbaSTom Tucker default: 19307ebafbaSTom Tucker BUG(); 19407ebafbaSTom Tucker return 0; 19507ebafbaSTom Tucker } 19607ebafbaSTom Tucker } 19707ebafbaSTom Tucker EXPORT_SYMBOL(rdma_node_get_transport); 19807ebafbaSTom Tucker 199a3f5adafSEli Cohen enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num) 200a3f5adafSEli Cohen { 201a3f5adafSEli Cohen if (device->get_link_layer) 202a3f5adafSEli Cohen return device->get_link_layer(device, port_num); 203a3f5adafSEli Cohen 204a3f5adafSEli Cohen switch (rdma_node_get_transport(device->node_type)) { 205a3f5adafSEli Cohen case RDMA_TRANSPORT_IB: 206a3f5adafSEli Cohen return IB_LINK_LAYER_INFINIBAND; 207a3f5adafSEli Cohen case RDMA_TRANSPORT_IWARP: 208180771a3SUpinder Malhi \(umalhi\) case RDMA_TRANSPORT_USNIC: 209248567f7SUpinder Malhi case RDMA_TRANSPORT_USNIC_UDP: 210a3f5adafSEli Cohen return IB_LINK_LAYER_ETHERNET; 211a3f5adafSEli Cohen default: 212a3f5adafSEli Cohen return IB_LINK_LAYER_UNSPECIFIED; 213a3f5adafSEli Cohen } 214a3f5adafSEli Cohen } 215a3f5adafSEli Cohen EXPORT_SYMBOL(rdma_port_get_link_layer); 216a3f5adafSEli Cohen 2171da177e4SLinus Torvalds /* Protection domains */ 2181da177e4SLinus Torvalds 21996249d70SJason Gunthorpe /** 22096249d70SJason Gunthorpe * ib_alloc_pd - Allocates an unused protection domain. 22196249d70SJason Gunthorpe * @device: The device on which to allocate the protection domain. 22296249d70SJason Gunthorpe * 22396249d70SJason Gunthorpe * A protection domain object provides an association between QPs, shared 22496249d70SJason Gunthorpe * receive queues, address handles, memory regions, and memory windows. 22596249d70SJason Gunthorpe * 22696249d70SJason Gunthorpe * Every PD has a local_dma_lkey which can be used as the lkey value for local 22796249d70SJason Gunthorpe * memory operations. 22896249d70SJason Gunthorpe */ 2291da177e4SLinus Torvalds struct ib_pd *ib_alloc_pd(struct ib_device *device) 2301da177e4SLinus Torvalds { 2311da177e4SLinus Torvalds struct ib_pd *pd; 2321da177e4SLinus Torvalds 233b5e81bf5SRoland Dreier pd = device->alloc_pd(device, NULL, NULL); 23496249d70SJason Gunthorpe if (IS_ERR(pd)) 23596249d70SJason Gunthorpe return pd; 2361da177e4SLinus Torvalds 2371da177e4SLinus Torvalds pd->device = device; 238b5e81bf5SRoland Dreier pd->uobject = NULL; 23996249d70SJason Gunthorpe pd->local_mr = NULL; 2401da177e4SLinus Torvalds atomic_set(&pd->usecnt, 0); 24196249d70SJason Gunthorpe 24286bee4c9SOr Gerlitz if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) 24396249d70SJason Gunthorpe pd->local_dma_lkey = device->local_dma_lkey; 24496249d70SJason Gunthorpe else { 24596249d70SJason Gunthorpe struct ib_mr *mr; 24696249d70SJason Gunthorpe 24796249d70SJason Gunthorpe mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE); 24896249d70SJason Gunthorpe if (IS_ERR(mr)) { 24996249d70SJason Gunthorpe ib_dealloc_pd(pd); 25096249d70SJason Gunthorpe return (struct ib_pd *)mr; 2511da177e4SLinus Torvalds } 2521da177e4SLinus Torvalds 25396249d70SJason Gunthorpe pd->local_mr = mr; 25496249d70SJason Gunthorpe pd->local_dma_lkey = pd->local_mr->lkey; 25596249d70SJason Gunthorpe } 2561da177e4SLinus Torvalds return pd; 2571da177e4SLinus Torvalds } 2581da177e4SLinus Torvalds EXPORT_SYMBOL(ib_alloc_pd); 2591da177e4SLinus Torvalds 2607dd78647SJason Gunthorpe /** 2617dd78647SJason Gunthorpe * ib_dealloc_pd - Deallocates a protection domain. 2627dd78647SJason Gunthorpe * @pd: The protection domain to deallocate. 2637dd78647SJason Gunthorpe * 2647dd78647SJason Gunthorpe * It is an error to call this function while any resources in the pd still 2657dd78647SJason Gunthorpe * exist. The caller is responsible to synchronously destroy them and 2667dd78647SJason Gunthorpe * guarantee no new allocations will happen. 2677dd78647SJason Gunthorpe */ 2687dd78647SJason Gunthorpe void ib_dealloc_pd(struct ib_pd *pd) 2691da177e4SLinus Torvalds { 2707dd78647SJason Gunthorpe int ret; 2711da177e4SLinus Torvalds 27296249d70SJason Gunthorpe if (pd->local_mr) { 2737dd78647SJason Gunthorpe ret = ib_dereg_mr(pd->local_mr); 2747dd78647SJason Gunthorpe WARN_ON(ret); 27596249d70SJason Gunthorpe pd->local_mr = NULL; 27696249d70SJason Gunthorpe } 27796249d70SJason Gunthorpe 2787dd78647SJason Gunthorpe /* uverbs manipulates usecnt with proper locking, while the kabi 2797dd78647SJason Gunthorpe requires the caller to guarantee we can't race here. */ 2807dd78647SJason Gunthorpe WARN_ON(atomic_read(&pd->usecnt)); 2811da177e4SLinus Torvalds 2827dd78647SJason Gunthorpe /* Making delalloc_pd a void return is a WIP, no driver should return 2837dd78647SJason Gunthorpe an error here. */ 2847dd78647SJason Gunthorpe ret = pd->device->dealloc_pd(pd); 2857dd78647SJason Gunthorpe WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd"); 2861da177e4SLinus Torvalds } 2871da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_pd); 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds /* Address handles */ 2901da177e4SLinus Torvalds 2911da177e4SLinus Torvalds struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) 2921da177e4SLinus Torvalds { 2931da177e4SLinus Torvalds struct ib_ah *ah; 2941da177e4SLinus Torvalds 2951da177e4SLinus Torvalds ah = pd->device->create_ah(pd, ah_attr); 2961da177e4SLinus Torvalds 2971da177e4SLinus Torvalds if (!IS_ERR(ah)) { 2981da177e4SLinus Torvalds ah->device = pd->device; 2991da177e4SLinus Torvalds ah->pd = pd; 300b5e81bf5SRoland Dreier ah->uobject = NULL; 3011da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 3021da177e4SLinus Torvalds } 3031da177e4SLinus Torvalds 3041da177e4SLinus Torvalds return ah; 3051da177e4SLinus Torvalds } 3061da177e4SLinus Torvalds EXPORT_SYMBOL(ib_create_ah); 3071da177e4SLinus Torvalds 308c865f246SSomnath Kotur static int ib_get_header_version(const union rdma_network_hdr *hdr) 309c865f246SSomnath Kotur { 310c865f246SSomnath Kotur const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh; 311c865f246SSomnath Kotur struct iphdr ip4h_checked; 312c865f246SSomnath Kotur const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh; 313c865f246SSomnath Kotur 314c865f246SSomnath Kotur /* If it's IPv6, the version must be 6, otherwise, the first 315c865f246SSomnath Kotur * 20 bytes (before the IPv4 header) are garbled. 316c865f246SSomnath Kotur */ 317c865f246SSomnath Kotur if (ip6h->version != 6) 318c865f246SSomnath Kotur return (ip4h->version == 4) ? 4 : 0; 319c865f246SSomnath Kotur /* version may be 6 or 4 because the first 20 bytes could be garbled */ 320c865f246SSomnath Kotur 321c865f246SSomnath Kotur /* RoCE v2 requires no options, thus header length 322c865f246SSomnath Kotur * must be 5 words 323c865f246SSomnath Kotur */ 324c865f246SSomnath Kotur if (ip4h->ihl != 5) 325c865f246SSomnath Kotur return 6; 326c865f246SSomnath Kotur 327c865f246SSomnath Kotur /* Verify checksum. 328c865f246SSomnath Kotur * We can't write on scattered buffers so we need to copy to 329c865f246SSomnath Kotur * temp buffer. 330c865f246SSomnath Kotur */ 331c865f246SSomnath Kotur memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked)); 332c865f246SSomnath Kotur ip4h_checked.check = 0; 333c865f246SSomnath Kotur ip4h_checked.check = ip_fast_csum((u8 *)&ip4h_checked, 5); 334c865f246SSomnath Kotur /* if IPv4 header checksum is OK, believe it */ 335c865f246SSomnath Kotur if (ip4h->check == ip4h_checked.check) 336c865f246SSomnath Kotur return 4; 337c865f246SSomnath Kotur return 6; 338c865f246SSomnath Kotur } 339c865f246SSomnath Kotur 340c865f246SSomnath Kotur static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device, 341c865f246SSomnath Kotur u8 port_num, 342c865f246SSomnath Kotur const struct ib_grh *grh) 343c865f246SSomnath Kotur { 344c865f246SSomnath Kotur int grh_version; 345c865f246SSomnath Kotur 346c865f246SSomnath Kotur if (rdma_protocol_ib(device, port_num)) 347c865f246SSomnath Kotur return RDMA_NETWORK_IB; 348c865f246SSomnath Kotur 349c865f246SSomnath Kotur grh_version = ib_get_header_version((union rdma_network_hdr *)grh); 350c865f246SSomnath Kotur 351c865f246SSomnath Kotur if (grh_version == 4) 352c865f246SSomnath Kotur return RDMA_NETWORK_IPV4; 353c865f246SSomnath Kotur 354c865f246SSomnath Kotur if (grh->next_hdr == IPPROTO_UDP) 355c865f246SSomnath Kotur return RDMA_NETWORK_IPV6; 356c865f246SSomnath Kotur 357c865f246SSomnath Kotur return RDMA_NETWORK_ROCE_V1; 358c865f246SSomnath Kotur } 359c865f246SSomnath Kotur 360dbf727deSMatan Barak struct find_gid_index_context { 361dbf727deSMatan Barak u16 vlan_id; 362c865f246SSomnath Kotur enum ib_gid_type gid_type; 363dbf727deSMatan Barak }; 364dbf727deSMatan Barak 365dbf727deSMatan Barak static bool find_gid_index(const union ib_gid *gid, 366dbf727deSMatan Barak const struct ib_gid_attr *gid_attr, 367dbf727deSMatan Barak void *context) 368dbf727deSMatan Barak { 369dbf727deSMatan Barak struct find_gid_index_context *ctx = 370dbf727deSMatan Barak (struct find_gid_index_context *)context; 371dbf727deSMatan Barak 372c865f246SSomnath Kotur if (ctx->gid_type != gid_attr->gid_type) 373c865f246SSomnath Kotur return false; 374c865f246SSomnath Kotur 375dbf727deSMatan Barak if ((!!(ctx->vlan_id != 0xffff) == !is_vlan_dev(gid_attr->ndev)) || 376dbf727deSMatan Barak (is_vlan_dev(gid_attr->ndev) && 377dbf727deSMatan Barak vlan_dev_vlan_id(gid_attr->ndev) != ctx->vlan_id)) 378dbf727deSMatan Barak return false; 379dbf727deSMatan Barak 380dbf727deSMatan Barak return true; 381dbf727deSMatan Barak } 382dbf727deSMatan Barak 383dbf727deSMatan Barak static int get_sgid_index_from_eth(struct ib_device *device, u8 port_num, 384dbf727deSMatan Barak u16 vlan_id, const union ib_gid *sgid, 385c865f246SSomnath Kotur enum ib_gid_type gid_type, 386dbf727deSMatan Barak u16 *gid_index) 387dbf727deSMatan Barak { 388c865f246SSomnath Kotur struct find_gid_index_context context = {.vlan_id = vlan_id, 389c865f246SSomnath Kotur .gid_type = gid_type}; 390dbf727deSMatan Barak 391dbf727deSMatan Barak return ib_find_gid_by_filter(device, sgid, port_num, find_gid_index, 392dbf727deSMatan Barak &context, gid_index); 393dbf727deSMatan Barak } 394dbf727deSMatan Barak 395c865f246SSomnath Kotur static int get_gids_from_rdma_hdr(union rdma_network_hdr *hdr, 396c865f246SSomnath Kotur enum rdma_network_type net_type, 397c865f246SSomnath Kotur union ib_gid *sgid, union ib_gid *dgid) 398c865f246SSomnath Kotur { 399c865f246SSomnath Kotur struct sockaddr_in src_in; 400c865f246SSomnath Kotur struct sockaddr_in dst_in; 401c865f246SSomnath Kotur __be32 src_saddr, dst_saddr; 402c865f246SSomnath Kotur 403c865f246SSomnath Kotur if (!sgid || !dgid) 404c865f246SSomnath Kotur return -EINVAL; 405c865f246SSomnath Kotur 406c865f246SSomnath Kotur if (net_type == RDMA_NETWORK_IPV4) { 407c865f246SSomnath Kotur memcpy(&src_in.sin_addr.s_addr, 408c865f246SSomnath Kotur &hdr->roce4grh.saddr, 4); 409c865f246SSomnath Kotur memcpy(&dst_in.sin_addr.s_addr, 410c865f246SSomnath Kotur &hdr->roce4grh.daddr, 4); 411c865f246SSomnath Kotur src_saddr = src_in.sin_addr.s_addr; 412c865f246SSomnath Kotur dst_saddr = dst_in.sin_addr.s_addr; 413c865f246SSomnath Kotur ipv6_addr_set_v4mapped(src_saddr, 414c865f246SSomnath Kotur (struct in6_addr *)sgid); 415c865f246SSomnath Kotur ipv6_addr_set_v4mapped(dst_saddr, 416c865f246SSomnath Kotur (struct in6_addr *)dgid); 417c865f246SSomnath Kotur return 0; 418c865f246SSomnath Kotur } else if (net_type == RDMA_NETWORK_IPV6 || 419c865f246SSomnath Kotur net_type == RDMA_NETWORK_IB) { 420c865f246SSomnath Kotur *dgid = hdr->ibgrh.dgid; 421c865f246SSomnath Kotur *sgid = hdr->ibgrh.sgid; 422c865f246SSomnath Kotur return 0; 423c865f246SSomnath Kotur } else { 424c865f246SSomnath Kotur return -EINVAL; 425c865f246SSomnath Kotur } 426c865f246SSomnath Kotur } 427c865f246SSomnath Kotur 42873cdaaeeSIra Weiny int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, 42973cdaaeeSIra Weiny const struct ib_wc *wc, const struct ib_grh *grh, 43073cdaaeeSIra Weiny struct ib_ah_attr *ah_attr) 431513789edSHal Rosenstock { 432513789edSHal Rosenstock u32 flow_class; 433513789edSHal Rosenstock u16 gid_index; 434513789edSHal Rosenstock int ret; 435c865f246SSomnath Kotur enum rdma_network_type net_type = RDMA_NETWORK_IB; 436c865f246SSomnath Kotur enum ib_gid_type gid_type = IB_GID_TYPE_IB; 437c865f246SSomnath Kotur union ib_gid dgid; 438c865f246SSomnath Kotur union ib_gid sgid; 439513789edSHal Rosenstock 4404e00d694SSean Hefty memset(ah_attr, 0, sizeof *ah_attr); 441227128fcSMichael Wang if (rdma_cap_eth_ah(device, port_num)) { 442c865f246SSomnath Kotur if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE) 443c865f246SSomnath Kotur net_type = wc->network_hdr_type; 444c865f246SSomnath Kotur else 445c865f246SSomnath Kotur net_type = ib_get_net_type_by_grh(device, port_num, grh); 446c865f246SSomnath Kotur gid_type = ib_network_to_gid_type(net_type); 447c865f246SSomnath Kotur } 448c865f246SSomnath Kotur ret = get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type, 449c865f246SSomnath Kotur &sgid, &dgid); 450c865f246SSomnath Kotur if (ret) 451c865f246SSomnath Kotur return ret; 452c865f246SSomnath Kotur 453c865f246SSomnath Kotur if (rdma_protocol_roce(device, port_num)) { 454dbf727deSMatan Barak u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ? 455dbf727deSMatan Barak wc->vlan_id : 0xffff; 456dbf727deSMatan Barak 457dd5f03beSMatan Barak if (!(wc->wc_flags & IB_WC_GRH)) 458dd5f03beSMatan Barak return -EPROTOTYPE; 459dd5f03beSMatan Barak 460dbf727deSMatan Barak if (!(wc->wc_flags & IB_WC_WITH_SMAC) || 461dbf727deSMatan Barak !(wc->wc_flags & IB_WC_WITH_VLAN)) { 462c865f246SSomnath Kotur ret = rdma_addr_find_dmac_by_grh(&dgid, &sgid, 463dbf727deSMatan Barak ah_attr->dmac, 464dbf727deSMatan Barak wc->wc_flags & IB_WC_WITH_VLAN ? 465dbf727deSMatan Barak NULL : &vlan_id, 466dbf727deSMatan Barak 0); 467dd5f03beSMatan Barak if (ret) 468dd5f03beSMatan Barak return ret; 469dd5f03beSMatan Barak } 470dbf727deSMatan Barak 471dbf727deSMatan Barak ret = get_sgid_index_from_eth(device, port_num, vlan_id, 472c865f246SSomnath Kotur &dgid, gid_type, &gid_index); 473dbf727deSMatan Barak if (ret) 474dbf727deSMatan Barak return ret; 475dbf727deSMatan Barak 476dbf727deSMatan Barak if (wc->wc_flags & IB_WC_WITH_SMAC) 477dbf727deSMatan Barak memcpy(ah_attr->dmac, wc->smac, ETH_ALEN); 478dd5f03beSMatan Barak } 479dd5f03beSMatan Barak 4804e00d694SSean Hefty ah_attr->dlid = wc->slid; 4814e00d694SSean Hefty ah_attr->sl = wc->sl; 4824e00d694SSean Hefty ah_attr->src_path_bits = wc->dlid_path_bits; 4834e00d694SSean Hefty ah_attr->port_num = port_num; 484513789edSHal Rosenstock 485513789edSHal Rosenstock if (wc->wc_flags & IB_WC_GRH) { 4864e00d694SSean Hefty ah_attr->ah_flags = IB_AH_GRH; 487c865f246SSomnath Kotur ah_attr->grh.dgid = sgid; 488513789edSHal Rosenstock 489dbf727deSMatan Barak if (!rdma_cap_eth_ah(device, port_num)) { 490c865f246SSomnath Kotur ret = ib_find_cached_gid_by_port(device, &dgid, 491b39ffa1dSMatan Barak IB_GID_TYPE_IB, 492dbf727deSMatan Barak port_num, NULL, 493dbf727deSMatan Barak &gid_index); 494513789edSHal Rosenstock if (ret) 4954e00d694SSean Hefty return ret; 496dbf727deSMatan Barak } 497513789edSHal Rosenstock 4984e00d694SSean Hefty ah_attr->grh.sgid_index = (u8) gid_index; 499497677abSHal Rosenstock flow_class = be32_to_cpu(grh->version_tclass_flow); 5004e00d694SSean Hefty ah_attr->grh.flow_label = flow_class & 0xFFFFF; 50147645d8dSSean Hefty ah_attr->grh.hop_limit = 0xFF; 5024e00d694SSean Hefty ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF; 503513789edSHal Rosenstock } 5044e00d694SSean Hefty return 0; 5054e00d694SSean Hefty } 5064e00d694SSean Hefty EXPORT_SYMBOL(ib_init_ah_from_wc); 5074e00d694SSean Hefty 50873cdaaeeSIra Weiny struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc, 50973cdaaeeSIra Weiny const struct ib_grh *grh, u8 port_num) 5104e00d694SSean Hefty { 5114e00d694SSean Hefty struct ib_ah_attr ah_attr; 5124e00d694SSean Hefty int ret; 5134e00d694SSean Hefty 5144e00d694SSean Hefty ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr); 5154e00d694SSean Hefty if (ret) 5164e00d694SSean Hefty return ERR_PTR(ret); 517513789edSHal Rosenstock 518513789edSHal Rosenstock return ib_create_ah(pd, &ah_attr); 519513789edSHal Rosenstock } 520513789edSHal Rosenstock EXPORT_SYMBOL(ib_create_ah_from_wc); 521513789edSHal Rosenstock 5221da177e4SLinus Torvalds int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 5231da177e4SLinus Torvalds { 5241da177e4SLinus Torvalds return ah->device->modify_ah ? 5251da177e4SLinus Torvalds ah->device->modify_ah(ah, ah_attr) : 5261da177e4SLinus Torvalds -ENOSYS; 5271da177e4SLinus Torvalds } 5281da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_ah); 5291da177e4SLinus Torvalds 5301da177e4SLinus Torvalds int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 5311da177e4SLinus Torvalds { 5321da177e4SLinus Torvalds return ah->device->query_ah ? 5331da177e4SLinus Torvalds ah->device->query_ah(ah, ah_attr) : 5341da177e4SLinus Torvalds -ENOSYS; 5351da177e4SLinus Torvalds } 5361da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_ah); 5371da177e4SLinus Torvalds 5381da177e4SLinus Torvalds int ib_destroy_ah(struct ib_ah *ah) 5391da177e4SLinus Torvalds { 5401da177e4SLinus Torvalds struct ib_pd *pd; 5411da177e4SLinus Torvalds int ret; 5421da177e4SLinus Torvalds 5431da177e4SLinus Torvalds pd = ah->pd; 5441da177e4SLinus Torvalds ret = ah->device->destroy_ah(ah); 5451da177e4SLinus Torvalds if (!ret) 5461da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 5471da177e4SLinus Torvalds 5481da177e4SLinus Torvalds return ret; 5491da177e4SLinus Torvalds } 5501da177e4SLinus Torvalds EXPORT_SYMBOL(ib_destroy_ah); 5511da177e4SLinus Torvalds 552d41fcc67SRoland Dreier /* Shared receive queues */ 553d41fcc67SRoland Dreier 554d41fcc67SRoland Dreier struct ib_srq *ib_create_srq(struct ib_pd *pd, 555d41fcc67SRoland Dreier struct ib_srq_init_attr *srq_init_attr) 556d41fcc67SRoland Dreier { 557d41fcc67SRoland Dreier struct ib_srq *srq; 558d41fcc67SRoland Dreier 559d41fcc67SRoland Dreier if (!pd->device->create_srq) 560d41fcc67SRoland Dreier return ERR_PTR(-ENOSYS); 561d41fcc67SRoland Dreier 562d41fcc67SRoland Dreier srq = pd->device->create_srq(pd, srq_init_attr, NULL); 563d41fcc67SRoland Dreier 564d41fcc67SRoland Dreier if (!IS_ERR(srq)) { 565d41fcc67SRoland Dreier srq->device = pd->device; 566d41fcc67SRoland Dreier srq->pd = pd; 567d41fcc67SRoland Dreier srq->uobject = NULL; 568d41fcc67SRoland Dreier srq->event_handler = srq_init_attr->event_handler; 569d41fcc67SRoland Dreier srq->srq_context = srq_init_attr->srq_context; 57096104edaSSean Hefty srq->srq_type = srq_init_attr->srq_type; 571418d5130SSean Hefty if (srq->srq_type == IB_SRQT_XRC) { 572418d5130SSean Hefty srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd; 573418d5130SSean Hefty srq->ext.xrc.cq = srq_init_attr->ext.xrc.cq; 574418d5130SSean Hefty atomic_inc(&srq->ext.xrc.xrcd->usecnt); 575418d5130SSean Hefty atomic_inc(&srq->ext.xrc.cq->usecnt); 576418d5130SSean Hefty } 577d41fcc67SRoland Dreier atomic_inc(&pd->usecnt); 578d41fcc67SRoland Dreier atomic_set(&srq->usecnt, 0); 579d41fcc67SRoland Dreier } 580d41fcc67SRoland Dreier 581d41fcc67SRoland Dreier return srq; 582d41fcc67SRoland Dreier } 583d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_create_srq); 584d41fcc67SRoland Dreier 585d41fcc67SRoland Dreier int ib_modify_srq(struct ib_srq *srq, 586d41fcc67SRoland Dreier struct ib_srq_attr *srq_attr, 587d41fcc67SRoland Dreier enum ib_srq_attr_mask srq_attr_mask) 588d41fcc67SRoland Dreier { 5897ce5eacbSDotan Barak return srq->device->modify_srq ? 5907ce5eacbSDotan Barak srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) : 5917ce5eacbSDotan Barak -ENOSYS; 592d41fcc67SRoland Dreier } 593d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_modify_srq); 594d41fcc67SRoland Dreier 595d41fcc67SRoland Dreier int ib_query_srq(struct ib_srq *srq, 596d41fcc67SRoland Dreier struct ib_srq_attr *srq_attr) 597d41fcc67SRoland Dreier { 598d41fcc67SRoland Dreier return srq->device->query_srq ? 599d41fcc67SRoland Dreier srq->device->query_srq(srq, srq_attr) : -ENOSYS; 600d41fcc67SRoland Dreier } 601d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_query_srq); 602d41fcc67SRoland Dreier 603d41fcc67SRoland Dreier int ib_destroy_srq(struct ib_srq *srq) 604d41fcc67SRoland Dreier { 605d41fcc67SRoland Dreier struct ib_pd *pd; 606418d5130SSean Hefty enum ib_srq_type srq_type; 607418d5130SSean Hefty struct ib_xrcd *uninitialized_var(xrcd); 608418d5130SSean Hefty struct ib_cq *uninitialized_var(cq); 609d41fcc67SRoland Dreier int ret; 610d41fcc67SRoland Dreier 611d41fcc67SRoland Dreier if (atomic_read(&srq->usecnt)) 612d41fcc67SRoland Dreier return -EBUSY; 613d41fcc67SRoland Dreier 614d41fcc67SRoland Dreier pd = srq->pd; 615418d5130SSean Hefty srq_type = srq->srq_type; 616418d5130SSean Hefty if (srq_type == IB_SRQT_XRC) { 617418d5130SSean Hefty xrcd = srq->ext.xrc.xrcd; 618418d5130SSean Hefty cq = srq->ext.xrc.cq; 619418d5130SSean Hefty } 620d41fcc67SRoland Dreier 621d41fcc67SRoland Dreier ret = srq->device->destroy_srq(srq); 622418d5130SSean Hefty if (!ret) { 623d41fcc67SRoland Dreier atomic_dec(&pd->usecnt); 624418d5130SSean Hefty if (srq_type == IB_SRQT_XRC) { 625418d5130SSean Hefty atomic_dec(&xrcd->usecnt); 626418d5130SSean Hefty atomic_dec(&cq->usecnt); 627418d5130SSean Hefty } 628418d5130SSean Hefty } 629d41fcc67SRoland Dreier 630d41fcc67SRoland Dreier return ret; 631d41fcc67SRoland Dreier } 632d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_destroy_srq); 633d41fcc67SRoland Dreier 6341da177e4SLinus Torvalds /* Queue pairs */ 6351da177e4SLinus Torvalds 6360e0ec7e0SSean Hefty static void __ib_shared_qp_event_handler(struct ib_event *event, void *context) 6370e0ec7e0SSean Hefty { 6380e0ec7e0SSean Hefty struct ib_qp *qp = context; 63973c40c61SYishai Hadas unsigned long flags; 6400e0ec7e0SSean Hefty 64173c40c61SYishai Hadas spin_lock_irqsave(&qp->device->event_handler_lock, flags); 6420e0ec7e0SSean Hefty list_for_each_entry(event->element.qp, &qp->open_list, open_list) 643eec9e29fSShlomo Pongratz if (event->element.qp->event_handler) 6440e0ec7e0SSean Hefty event->element.qp->event_handler(event, event->element.qp->qp_context); 64573c40c61SYishai Hadas spin_unlock_irqrestore(&qp->device->event_handler_lock, flags); 6460e0ec7e0SSean Hefty } 6470e0ec7e0SSean Hefty 648d3d72d90SSean Hefty static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) 649d3d72d90SSean Hefty { 650d3d72d90SSean Hefty mutex_lock(&xrcd->tgt_qp_mutex); 651d3d72d90SSean Hefty list_add(&qp->xrcd_list, &xrcd->tgt_qp_list); 652d3d72d90SSean Hefty mutex_unlock(&xrcd->tgt_qp_mutex); 653d3d72d90SSean Hefty } 654d3d72d90SSean Hefty 6550e0ec7e0SSean Hefty static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp, 6560e0ec7e0SSean Hefty void (*event_handler)(struct ib_event *, void *), 6570e0ec7e0SSean Hefty void *qp_context) 658d3d72d90SSean Hefty { 6590e0ec7e0SSean Hefty struct ib_qp *qp; 6600e0ec7e0SSean Hefty unsigned long flags; 6610e0ec7e0SSean Hefty 6620e0ec7e0SSean Hefty qp = kzalloc(sizeof *qp, GFP_KERNEL); 6630e0ec7e0SSean Hefty if (!qp) 6640e0ec7e0SSean Hefty return ERR_PTR(-ENOMEM); 6650e0ec7e0SSean Hefty 6660e0ec7e0SSean Hefty qp->real_qp = real_qp; 6670e0ec7e0SSean Hefty atomic_inc(&real_qp->usecnt); 6680e0ec7e0SSean Hefty qp->device = real_qp->device; 6690e0ec7e0SSean Hefty qp->event_handler = event_handler; 6700e0ec7e0SSean Hefty qp->qp_context = qp_context; 6710e0ec7e0SSean Hefty qp->qp_num = real_qp->qp_num; 6720e0ec7e0SSean Hefty qp->qp_type = real_qp->qp_type; 6730e0ec7e0SSean Hefty 6740e0ec7e0SSean Hefty spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); 6750e0ec7e0SSean Hefty list_add(&qp->open_list, &real_qp->open_list); 6760e0ec7e0SSean Hefty spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); 6770e0ec7e0SSean Hefty 6780e0ec7e0SSean Hefty return qp; 679d3d72d90SSean Hefty } 680d3d72d90SSean Hefty 6810e0ec7e0SSean Hefty struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd, 6820e0ec7e0SSean Hefty struct ib_qp_open_attr *qp_open_attr) 6830e0ec7e0SSean Hefty { 6840e0ec7e0SSean Hefty struct ib_qp *qp, *real_qp; 6850e0ec7e0SSean Hefty 6860e0ec7e0SSean Hefty if (qp_open_attr->qp_type != IB_QPT_XRC_TGT) 6870e0ec7e0SSean Hefty return ERR_PTR(-EINVAL); 6880e0ec7e0SSean Hefty 6890e0ec7e0SSean Hefty qp = ERR_PTR(-EINVAL); 6900e0ec7e0SSean Hefty mutex_lock(&xrcd->tgt_qp_mutex); 6910e0ec7e0SSean Hefty list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) { 6920e0ec7e0SSean Hefty if (real_qp->qp_num == qp_open_attr->qp_num) { 6930e0ec7e0SSean Hefty qp = __ib_open_qp(real_qp, qp_open_attr->event_handler, 6940e0ec7e0SSean Hefty qp_open_attr->qp_context); 6950e0ec7e0SSean Hefty break; 6960e0ec7e0SSean Hefty } 6970e0ec7e0SSean Hefty } 6980e0ec7e0SSean Hefty mutex_unlock(&xrcd->tgt_qp_mutex); 6990e0ec7e0SSean Hefty return qp; 7000e0ec7e0SSean Hefty } 7010e0ec7e0SSean Hefty EXPORT_SYMBOL(ib_open_qp); 7020e0ec7e0SSean Hefty 7031da177e4SLinus Torvalds struct ib_qp *ib_create_qp(struct ib_pd *pd, 7041da177e4SLinus Torvalds struct ib_qp_init_attr *qp_init_attr) 7051da177e4SLinus Torvalds { 7060e0ec7e0SSean Hefty struct ib_qp *qp, *real_qp; 707b42b63cfSSean Hefty struct ib_device *device; 7081da177e4SLinus Torvalds 709b42b63cfSSean Hefty device = pd ? pd->device : qp_init_attr->xrcd->device; 710b42b63cfSSean Hefty qp = device->create_qp(pd, qp_init_attr, NULL); 7111da177e4SLinus Torvalds 7121da177e4SLinus Torvalds if (!IS_ERR(qp)) { 713b42b63cfSSean Hefty qp->device = device; 7140e0ec7e0SSean Hefty qp->real_qp = qp; 715b5e81bf5SRoland Dreier qp->uobject = NULL; 7160e0ec7e0SSean Hefty qp->qp_type = qp_init_attr->qp_type; 717b42b63cfSSean Hefty 718e47e321aSBernd Schubert atomic_set(&qp->usecnt, 0); 719b42b63cfSSean Hefty if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) { 7200e0ec7e0SSean Hefty qp->event_handler = __ib_shared_qp_event_handler; 7210e0ec7e0SSean Hefty qp->qp_context = qp; 722b42b63cfSSean Hefty qp->pd = NULL; 723b42b63cfSSean Hefty qp->send_cq = qp->recv_cq = NULL; 724b42b63cfSSean Hefty qp->srq = NULL; 725b42b63cfSSean Hefty qp->xrcd = qp_init_attr->xrcd; 726b42b63cfSSean Hefty atomic_inc(&qp_init_attr->xrcd->usecnt); 7270e0ec7e0SSean Hefty INIT_LIST_HEAD(&qp->open_list); 7280e0ec7e0SSean Hefty 7290e0ec7e0SSean Hefty real_qp = qp; 7300e0ec7e0SSean Hefty qp = __ib_open_qp(real_qp, qp_init_attr->event_handler, 7310e0ec7e0SSean Hefty qp_init_attr->qp_context); 7320e0ec7e0SSean Hefty if (!IS_ERR(qp)) 7330e0ec7e0SSean Hefty __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp); 7340e0ec7e0SSean Hefty else 7350e0ec7e0SSean Hefty real_qp->device->destroy_qp(real_qp); 736b42b63cfSSean Hefty } else { 7371da177e4SLinus Torvalds qp->event_handler = qp_init_attr->event_handler; 7381da177e4SLinus Torvalds qp->qp_context = qp_init_attr->qp_context; 739b42b63cfSSean Hefty if (qp_init_attr->qp_type == IB_QPT_XRC_INI) { 740b42b63cfSSean Hefty qp->recv_cq = NULL; 741b42b63cfSSean Hefty qp->srq = NULL; 742b42b63cfSSean Hefty } else { 743b42b63cfSSean Hefty qp->recv_cq = qp_init_attr->recv_cq; 744b42b63cfSSean Hefty atomic_inc(&qp_init_attr->recv_cq->usecnt); 745b42b63cfSSean Hefty qp->srq = qp_init_attr->srq; 746b42b63cfSSean Hefty if (qp->srq) 747b42b63cfSSean Hefty atomic_inc(&qp_init_attr->srq->usecnt); 748b42b63cfSSean Hefty } 749b42b63cfSSean Hefty 7501da177e4SLinus Torvalds qp->pd = pd; 7511da177e4SLinus Torvalds qp->send_cq = qp_init_attr->send_cq; 752b42b63cfSSean Hefty qp->xrcd = NULL; 753b42b63cfSSean Hefty 7541da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 7551da177e4SLinus Torvalds atomic_inc(&qp_init_attr->send_cq->usecnt); 756b42b63cfSSean Hefty } 7571da177e4SLinus Torvalds } 7581da177e4SLinus Torvalds 7591da177e4SLinus Torvalds return qp; 7601da177e4SLinus Torvalds } 7611da177e4SLinus Torvalds EXPORT_SYMBOL(ib_create_qp); 7621da177e4SLinus Torvalds 7638a51866fSRoland Dreier static const struct { 7648a51866fSRoland Dreier int valid; 765b42b63cfSSean Hefty enum ib_qp_attr_mask req_param[IB_QPT_MAX]; 766b42b63cfSSean Hefty enum ib_qp_attr_mask opt_param[IB_QPT_MAX]; 7678a51866fSRoland Dreier } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { 7688a51866fSRoland Dreier [IB_QPS_RESET] = { 7698a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 7708a51866fSRoland Dreier [IB_QPS_INIT] = { 7718a51866fSRoland Dreier .valid = 1, 7728a51866fSRoland Dreier .req_param = { 7738a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 7748a51866fSRoland Dreier IB_QP_PORT | 7758a51866fSRoland Dreier IB_QP_QKEY), 776c938a616SOr Gerlitz [IB_QPT_RAW_PACKET] = IB_QP_PORT, 7778a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 7788a51866fSRoland Dreier IB_QP_PORT | 7798a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 7808a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 7818a51866fSRoland Dreier IB_QP_PORT | 7828a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 783b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX | 784b42b63cfSSean Hefty IB_QP_PORT | 785b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 786b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX | 787b42b63cfSSean Hefty IB_QP_PORT | 788b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 7898a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 7908a51866fSRoland Dreier IB_QP_QKEY), 7918a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 7928a51866fSRoland Dreier IB_QP_QKEY), 7938a51866fSRoland Dreier } 7948a51866fSRoland Dreier }, 7958a51866fSRoland Dreier }, 7968a51866fSRoland Dreier [IB_QPS_INIT] = { 7978a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 7988a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 7998a51866fSRoland Dreier [IB_QPS_INIT] = { 8008a51866fSRoland Dreier .valid = 1, 8018a51866fSRoland Dreier .opt_param = { 8028a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 8038a51866fSRoland Dreier IB_QP_PORT | 8048a51866fSRoland Dreier IB_QP_QKEY), 8058a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 8068a51866fSRoland Dreier IB_QP_PORT | 8078a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 8088a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 8098a51866fSRoland Dreier IB_QP_PORT | 8108a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 811b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX | 812b42b63cfSSean Hefty IB_QP_PORT | 813b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 814b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX | 815b42b63cfSSean Hefty IB_QP_PORT | 816b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 8178a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 8188a51866fSRoland Dreier IB_QP_QKEY), 8198a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 8208a51866fSRoland Dreier IB_QP_QKEY), 8218a51866fSRoland Dreier } 8228a51866fSRoland Dreier }, 8238a51866fSRoland Dreier [IB_QPS_RTR] = { 8248a51866fSRoland Dreier .valid = 1, 8258a51866fSRoland Dreier .req_param = { 8268a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_AV | 8278a51866fSRoland Dreier IB_QP_PATH_MTU | 8288a51866fSRoland Dreier IB_QP_DEST_QPN | 8298a51866fSRoland Dreier IB_QP_RQ_PSN), 8308a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_AV | 8318a51866fSRoland Dreier IB_QP_PATH_MTU | 8328a51866fSRoland Dreier IB_QP_DEST_QPN | 8338a51866fSRoland Dreier IB_QP_RQ_PSN | 8348a51866fSRoland Dreier IB_QP_MAX_DEST_RD_ATOMIC | 8358a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER), 836b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_AV | 837b42b63cfSSean Hefty IB_QP_PATH_MTU | 838b42b63cfSSean Hefty IB_QP_DEST_QPN | 839b42b63cfSSean Hefty IB_QP_RQ_PSN), 840b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_AV | 841b42b63cfSSean Hefty IB_QP_PATH_MTU | 842b42b63cfSSean Hefty IB_QP_DEST_QPN | 843b42b63cfSSean Hefty IB_QP_RQ_PSN | 844b42b63cfSSean Hefty IB_QP_MAX_DEST_RD_ATOMIC | 845b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER), 8468a51866fSRoland Dreier }, 8478a51866fSRoland Dreier .opt_param = { 8488a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 8498a51866fSRoland Dreier IB_QP_QKEY), 8508a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_ALT_PATH | 8518a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 8528a51866fSRoland Dreier IB_QP_PKEY_INDEX), 8538a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_ALT_PATH | 8548a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 8558a51866fSRoland Dreier IB_QP_PKEY_INDEX), 856b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH | 857b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 858b42b63cfSSean Hefty IB_QP_PKEY_INDEX), 859b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH | 860b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 861b42b63cfSSean Hefty IB_QP_PKEY_INDEX), 8628a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 8638a51866fSRoland Dreier IB_QP_QKEY), 8648a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 8658a51866fSRoland Dreier IB_QP_QKEY), 866dd5f03beSMatan Barak }, 867dbf727deSMatan Barak }, 8688a51866fSRoland Dreier }, 8698a51866fSRoland Dreier [IB_QPS_RTR] = { 8708a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 8718a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 8728a51866fSRoland Dreier [IB_QPS_RTS] = { 8738a51866fSRoland Dreier .valid = 1, 8748a51866fSRoland Dreier .req_param = { 8758a51866fSRoland Dreier [IB_QPT_UD] = IB_QP_SQ_PSN, 8768a51866fSRoland Dreier [IB_QPT_UC] = IB_QP_SQ_PSN, 8778a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_TIMEOUT | 8788a51866fSRoland Dreier IB_QP_RETRY_CNT | 8798a51866fSRoland Dreier IB_QP_RNR_RETRY | 8808a51866fSRoland Dreier IB_QP_SQ_PSN | 8818a51866fSRoland Dreier IB_QP_MAX_QP_RD_ATOMIC), 882b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT | 883b42b63cfSSean Hefty IB_QP_RETRY_CNT | 884b42b63cfSSean Hefty IB_QP_RNR_RETRY | 885b42b63cfSSean Hefty IB_QP_SQ_PSN | 886b42b63cfSSean Hefty IB_QP_MAX_QP_RD_ATOMIC), 887b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT | 888b42b63cfSSean Hefty IB_QP_SQ_PSN), 8898a51866fSRoland Dreier [IB_QPT_SMI] = IB_QP_SQ_PSN, 8908a51866fSRoland Dreier [IB_QPT_GSI] = IB_QP_SQ_PSN, 8918a51866fSRoland Dreier }, 8928a51866fSRoland Dreier .opt_param = { 8938a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 8948a51866fSRoland Dreier IB_QP_QKEY), 8958a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_CUR_STATE | 8968a51866fSRoland Dreier IB_QP_ALT_PATH | 8978a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 8988a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 8998a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_CUR_STATE | 9008a51866fSRoland Dreier IB_QP_ALT_PATH | 9018a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 9028a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER | 9038a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 904b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 905b42b63cfSSean Hefty IB_QP_ALT_PATH | 906b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 907b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 908b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 909b42b63cfSSean Hefty IB_QP_ALT_PATH | 910b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 911b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER | 912b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 9138a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 9148a51866fSRoland Dreier IB_QP_QKEY), 9158a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 9168a51866fSRoland Dreier IB_QP_QKEY), 9178a51866fSRoland Dreier } 9188a51866fSRoland Dreier } 9198a51866fSRoland Dreier }, 9208a51866fSRoland Dreier [IB_QPS_RTS] = { 9218a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 9228a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 9238a51866fSRoland Dreier [IB_QPS_RTS] = { 9248a51866fSRoland Dreier .valid = 1, 9258a51866fSRoland Dreier .opt_param = { 9268a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 9278a51866fSRoland Dreier IB_QP_QKEY), 9284546d31dSDotan Barak [IB_QPT_UC] = (IB_QP_CUR_STATE | 9294546d31dSDotan Barak IB_QP_ACCESS_FLAGS | 9308a51866fSRoland Dreier IB_QP_ALT_PATH | 9318a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 9324546d31dSDotan Barak [IB_QPT_RC] = (IB_QP_CUR_STATE | 9334546d31dSDotan Barak IB_QP_ACCESS_FLAGS | 9348a51866fSRoland Dreier IB_QP_ALT_PATH | 9358a51866fSRoland Dreier IB_QP_PATH_MIG_STATE | 9368a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER), 937b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 938b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 939b42b63cfSSean Hefty IB_QP_ALT_PATH | 940b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 941b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 942b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 943b42b63cfSSean Hefty IB_QP_ALT_PATH | 944b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE | 945b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER), 9468a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 9478a51866fSRoland Dreier IB_QP_QKEY), 9488a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 9498a51866fSRoland Dreier IB_QP_QKEY), 9508a51866fSRoland Dreier } 9518a51866fSRoland Dreier }, 9528a51866fSRoland Dreier [IB_QPS_SQD] = { 9538a51866fSRoland Dreier .valid = 1, 9548a51866fSRoland Dreier .opt_param = { 9558a51866fSRoland Dreier [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY, 9568a51866fSRoland Dreier [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 9578a51866fSRoland Dreier [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 958b42b63cfSSean Hefty [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 959b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */ 9608a51866fSRoland Dreier [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 9618a51866fSRoland Dreier [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY 9628a51866fSRoland Dreier } 9638a51866fSRoland Dreier }, 9648a51866fSRoland Dreier }, 9658a51866fSRoland Dreier [IB_QPS_SQD] = { 9668a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 9678a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 9688a51866fSRoland Dreier [IB_QPS_RTS] = { 9698a51866fSRoland Dreier .valid = 1, 9708a51866fSRoland Dreier .opt_param = { 9718a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 9728a51866fSRoland Dreier IB_QP_QKEY), 9738a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_CUR_STATE | 9748a51866fSRoland Dreier IB_QP_ALT_PATH | 9758a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 9768a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 9778a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_CUR_STATE | 9788a51866fSRoland Dreier IB_QP_ALT_PATH | 9798a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 9808a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER | 9818a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 982b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 983b42b63cfSSean Hefty IB_QP_ALT_PATH | 984b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 985b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 986b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 987b42b63cfSSean Hefty IB_QP_ALT_PATH | 988b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 989b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER | 990b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 9918a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 9928a51866fSRoland Dreier IB_QP_QKEY), 9938a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 9948a51866fSRoland Dreier IB_QP_QKEY), 9958a51866fSRoland Dreier } 9968a51866fSRoland Dreier }, 9978a51866fSRoland Dreier [IB_QPS_SQD] = { 9988a51866fSRoland Dreier .valid = 1, 9998a51866fSRoland Dreier .opt_param = { 10008a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 10018a51866fSRoland Dreier IB_QP_QKEY), 10028a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_AV | 10038a51866fSRoland Dreier IB_QP_ALT_PATH | 10048a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 10058a51866fSRoland Dreier IB_QP_PKEY_INDEX | 10068a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 10078a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_PORT | 10088a51866fSRoland Dreier IB_QP_AV | 10098a51866fSRoland Dreier IB_QP_TIMEOUT | 10108a51866fSRoland Dreier IB_QP_RETRY_CNT | 10118a51866fSRoland Dreier IB_QP_RNR_RETRY | 10128a51866fSRoland Dreier IB_QP_MAX_QP_RD_ATOMIC | 10138a51866fSRoland Dreier IB_QP_MAX_DEST_RD_ATOMIC | 10148a51866fSRoland Dreier IB_QP_ALT_PATH | 10158a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 10168a51866fSRoland Dreier IB_QP_PKEY_INDEX | 10178a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER | 10188a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 1019b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_PORT | 1020b42b63cfSSean Hefty IB_QP_AV | 1021b42b63cfSSean Hefty IB_QP_TIMEOUT | 1022b42b63cfSSean Hefty IB_QP_RETRY_CNT | 1023b42b63cfSSean Hefty IB_QP_RNR_RETRY | 1024b42b63cfSSean Hefty IB_QP_MAX_QP_RD_ATOMIC | 1025b42b63cfSSean Hefty IB_QP_ALT_PATH | 1026b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 1027b42b63cfSSean Hefty IB_QP_PKEY_INDEX | 1028b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 1029b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_PORT | 1030b42b63cfSSean Hefty IB_QP_AV | 1031b42b63cfSSean Hefty IB_QP_TIMEOUT | 1032b42b63cfSSean Hefty IB_QP_MAX_DEST_RD_ATOMIC | 1033b42b63cfSSean Hefty IB_QP_ALT_PATH | 1034b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 1035b42b63cfSSean Hefty IB_QP_PKEY_INDEX | 1036b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER | 1037b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 10388a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 10398a51866fSRoland Dreier IB_QP_QKEY), 10408a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 10418a51866fSRoland Dreier IB_QP_QKEY), 10428a51866fSRoland Dreier } 10438a51866fSRoland Dreier } 10448a51866fSRoland Dreier }, 10458a51866fSRoland Dreier [IB_QPS_SQE] = { 10468a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 10478a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 10488a51866fSRoland Dreier [IB_QPS_RTS] = { 10498a51866fSRoland Dreier .valid = 1, 10508a51866fSRoland Dreier .opt_param = { 10518a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 10528a51866fSRoland Dreier IB_QP_QKEY), 10538a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_CUR_STATE | 10548a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 10558a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 10568a51866fSRoland Dreier IB_QP_QKEY), 10578a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 10588a51866fSRoland Dreier IB_QP_QKEY), 10598a51866fSRoland Dreier } 10608a51866fSRoland Dreier } 10618a51866fSRoland Dreier }, 10628a51866fSRoland Dreier [IB_QPS_ERR] = { 10638a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 10648a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 } 10658a51866fSRoland Dreier } 10668a51866fSRoland Dreier }; 10678a51866fSRoland Dreier 10688a51866fSRoland Dreier int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, 1069dd5f03beSMatan Barak enum ib_qp_type type, enum ib_qp_attr_mask mask, 1070dd5f03beSMatan Barak enum rdma_link_layer ll) 10718a51866fSRoland Dreier { 10728a51866fSRoland Dreier enum ib_qp_attr_mask req_param, opt_param; 10738a51866fSRoland Dreier 10748a51866fSRoland Dreier if (cur_state < 0 || cur_state > IB_QPS_ERR || 10758a51866fSRoland Dreier next_state < 0 || next_state > IB_QPS_ERR) 10768a51866fSRoland Dreier return 0; 10778a51866fSRoland Dreier 10788a51866fSRoland Dreier if (mask & IB_QP_CUR_STATE && 10798a51866fSRoland Dreier cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && 10808a51866fSRoland Dreier cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) 10818a51866fSRoland Dreier return 0; 10828a51866fSRoland Dreier 10838a51866fSRoland Dreier if (!qp_state_table[cur_state][next_state].valid) 10848a51866fSRoland Dreier return 0; 10858a51866fSRoland Dreier 10868a51866fSRoland Dreier req_param = qp_state_table[cur_state][next_state].req_param[type]; 10878a51866fSRoland Dreier opt_param = qp_state_table[cur_state][next_state].opt_param[type]; 10888a51866fSRoland Dreier 10898a51866fSRoland Dreier if ((mask & req_param) != req_param) 10908a51866fSRoland Dreier return 0; 10918a51866fSRoland Dreier 10928a51866fSRoland Dreier if (mask & ~(req_param | opt_param | IB_QP_STATE)) 10938a51866fSRoland Dreier return 0; 10948a51866fSRoland Dreier 10958a51866fSRoland Dreier return 1; 10968a51866fSRoland Dreier } 10978a51866fSRoland Dreier EXPORT_SYMBOL(ib_modify_qp_is_ok); 10988a51866fSRoland Dreier 1099dbf727deSMatan Barak int ib_resolve_eth_dmac(struct ib_qp *qp, 1100ed4c54e5SOr Gerlitz struct ib_qp_attr *qp_attr, int *qp_attr_mask) 1101ed4c54e5SOr Gerlitz { 1102ed4c54e5SOr Gerlitz int ret = 0; 1103ed4c54e5SOr Gerlitz 1104dbf727deSMatan Barak if (*qp_attr_mask & IB_QP_AV) { 1105dbf727deSMatan Barak if (qp_attr->ah_attr.port_num < rdma_start_port(qp->device) || 1106dbf727deSMatan Barak qp_attr->ah_attr.port_num > rdma_end_port(qp->device)) 1107dbf727deSMatan Barak return -EINVAL; 1108dbf727deSMatan Barak 1109dbf727deSMatan Barak if (!rdma_cap_eth_ah(qp->device, qp_attr->ah_attr.port_num)) 1110dbf727deSMatan Barak return 0; 1111dbf727deSMatan Barak 1112ed4c54e5SOr Gerlitz if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) { 1113dbf727deSMatan Barak rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw, 1114dbf727deSMatan Barak qp_attr->ah_attr.dmac); 1115ed4c54e5SOr Gerlitz } else { 1116dbf727deSMatan Barak union ib_gid sgid; 1117dbf727deSMatan Barak struct ib_gid_attr sgid_attr; 1118dbf727deSMatan Barak int ifindex; 1119dbf727deSMatan Barak 1120dbf727deSMatan Barak ret = ib_query_gid(qp->device, 1121dbf727deSMatan Barak qp_attr->ah_attr.port_num, 1122dbf727deSMatan Barak qp_attr->ah_attr.grh.sgid_index, 1123dbf727deSMatan Barak &sgid, &sgid_attr); 1124dbf727deSMatan Barak 1125dbf727deSMatan Barak if (ret || !sgid_attr.ndev) { 1126dbf727deSMatan Barak if (!ret) 1127dbf727deSMatan Barak ret = -ENXIO; 1128ed4c54e5SOr Gerlitz goto out; 1129ed4c54e5SOr Gerlitz } 1130c865f246SSomnath Kotur if (sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) 1131c865f246SSomnath Kotur /* TODO: get the hoplimit from the inet/inet6 1132c865f246SSomnath Kotur * device 1133c865f246SSomnath Kotur */ 1134c865f246SSomnath Kotur qp_attr->ah_attr.grh.hop_limit = 1135c865f246SSomnath Kotur IPV6_DEFAULT_HOPLIMIT; 1136dbf727deSMatan Barak 1137dbf727deSMatan Barak ifindex = sgid_attr.ndev->ifindex; 1138dbf727deSMatan Barak 1139dbf727deSMatan Barak ret = rdma_addr_find_dmac_by_grh(&sgid, 1140dbf727deSMatan Barak &qp_attr->ah_attr.grh.dgid, 1141dbf727deSMatan Barak qp_attr->ah_attr.dmac, 1142dbf727deSMatan Barak NULL, ifindex); 1143dbf727deSMatan Barak 1144dbf727deSMatan Barak dev_put(sgid_attr.ndev); 1145dbf727deSMatan Barak } 1146ed4c54e5SOr Gerlitz } 1147ed4c54e5SOr Gerlitz out: 1148ed4c54e5SOr Gerlitz return ret; 1149ed4c54e5SOr Gerlitz } 1150dbf727deSMatan Barak EXPORT_SYMBOL(ib_resolve_eth_dmac); 1151ed4c54e5SOr Gerlitz 1152ed4c54e5SOr Gerlitz 11531da177e4SLinus Torvalds int ib_modify_qp(struct ib_qp *qp, 11541da177e4SLinus Torvalds struct ib_qp_attr *qp_attr, 11551da177e4SLinus Torvalds int qp_attr_mask) 11561da177e4SLinus Torvalds { 1157ed4c54e5SOr Gerlitz int ret; 1158ed4c54e5SOr Gerlitz 1159dbf727deSMatan Barak ret = ib_resolve_eth_dmac(qp, qp_attr, &qp_attr_mask); 1160ed4c54e5SOr Gerlitz if (ret) 1161ed4c54e5SOr Gerlitz return ret; 1162ed4c54e5SOr Gerlitz 11630e0ec7e0SSean Hefty return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL); 11641da177e4SLinus Torvalds } 11651da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_qp); 11661da177e4SLinus Torvalds 11671da177e4SLinus Torvalds int ib_query_qp(struct ib_qp *qp, 11681da177e4SLinus Torvalds struct ib_qp_attr *qp_attr, 11691da177e4SLinus Torvalds int qp_attr_mask, 11701da177e4SLinus Torvalds struct ib_qp_init_attr *qp_init_attr) 11711da177e4SLinus Torvalds { 11721da177e4SLinus Torvalds return qp->device->query_qp ? 11730e0ec7e0SSean Hefty qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) : 11741da177e4SLinus Torvalds -ENOSYS; 11751da177e4SLinus Torvalds } 11761da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_qp); 11771da177e4SLinus Torvalds 11780e0ec7e0SSean Hefty int ib_close_qp(struct ib_qp *qp) 11790e0ec7e0SSean Hefty { 11800e0ec7e0SSean Hefty struct ib_qp *real_qp; 11810e0ec7e0SSean Hefty unsigned long flags; 11820e0ec7e0SSean Hefty 11830e0ec7e0SSean Hefty real_qp = qp->real_qp; 11840e0ec7e0SSean Hefty if (real_qp == qp) 11850e0ec7e0SSean Hefty return -EINVAL; 11860e0ec7e0SSean Hefty 11870e0ec7e0SSean Hefty spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); 11880e0ec7e0SSean Hefty list_del(&qp->open_list); 11890e0ec7e0SSean Hefty spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); 11900e0ec7e0SSean Hefty 11910e0ec7e0SSean Hefty atomic_dec(&real_qp->usecnt); 11920e0ec7e0SSean Hefty kfree(qp); 11930e0ec7e0SSean Hefty 11940e0ec7e0SSean Hefty return 0; 11950e0ec7e0SSean Hefty } 11960e0ec7e0SSean Hefty EXPORT_SYMBOL(ib_close_qp); 11970e0ec7e0SSean Hefty 11980e0ec7e0SSean Hefty static int __ib_destroy_shared_qp(struct ib_qp *qp) 11990e0ec7e0SSean Hefty { 12000e0ec7e0SSean Hefty struct ib_xrcd *xrcd; 12010e0ec7e0SSean Hefty struct ib_qp *real_qp; 12020e0ec7e0SSean Hefty int ret; 12030e0ec7e0SSean Hefty 12040e0ec7e0SSean Hefty real_qp = qp->real_qp; 12050e0ec7e0SSean Hefty xrcd = real_qp->xrcd; 12060e0ec7e0SSean Hefty 12070e0ec7e0SSean Hefty mutex_lock(&xrcd->tgt_qp_mutex); 12080e0ec7e0SSean Hefty ib_close_qp(qp); 12090e0ec7e0SSean Hefty if (atomic_read(&real_qp->usecnt) == 0) 12100e0ec7e0SSean Hefty list_del(&real_qp->xrcd_list); 12110e0ec7e0SSean Hefty else 12120e0ec7e0SSean Hefty real_qp = NULL; 12130e0ec7e0SSean Hefty mutex_unlock(&xrcd->tgt_qp_mutex); 12140e0ec7e0SSean Hefty 12150e0ec7e0SSean Hefty if (real_qp) { 12160e0ec7e0SSean Hefty ret = ib_destroy_qp(real_qp); 12170e0ec7e0SSean Hefty if (!ret) 12180e0ec7e0SSean Hefty atomic_dec(&xrcd->usecnt); 12190e0ec7e0SSean Hefty else 12200e0ec7e0SSean Hefty __ib_insert_xrcd_qp(xrcd, real_qp); 12210e0ec7e0SSean Hefty } 12220e0ec7e0SSean Hefty 12230e0ec7e0SSean Hefty return 0; 12240e0ec7e0SSean Hefty } 12250e0ec7e0SSean Hefty 12261da177e4SLinus Torvalds int ib_destroy_qp(struct ib_qp *qp) 12271da177e4SLinus Torvalds { 12281da177e4SLinus Torvalds struct ib_pd *pd; 12291da177e4SLinus Torvalds struct ib_cq *scq, *rcq; 12301da177e4SLinus Torvalds struct ib_srq *srq; 12311da177e4SLinus Torvalds int ret; 12321da177e4SLinus Torvalds 12330e0ec7e0SSean Hefty if (atomic_read(&qp->usecnt)) 12340e0ec7e0SSean Hefty return -EBUSY; 12350e0ec7e0SSean Hefty 12360e0ec7e0SSean Hefty if (qp->real_qp != qp) 12370e0ec7e0SSean Hefty return __ib_destroy_shared_qp(qp); 12380e0ec7e0SSean Hefty 12391da177e4SLinus Torvalds pd = qp->pd; 12401da177e4SLinus Torvalds scq = qp->send_cq; 12411da177e4SLinus Torvalds rcq = qp->recv_cq; 12421da177e4SLinus Torvalds srq = qp->srq; 12431da177e4SLinus Torvalds 12441da177e4SLinus Torvalds ret = qp->device->destroy_qp(qp); 12451da177e4SLinus Torvalds if (!ret) { 1246b42b63cfSSean Hefty if (pd) 12471da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 1248b42b63cfSSean Hefty if (scq) 12491da177e4SLinus Torvalds atomic_dec(&scq->usecnt); 1250b42b63cfSSean Hefty if (rcq) 12511da177e4SLinus Torvalds atomic_dec(&rcq->usecnt); 12521da177e4SLinus Torvalds if (srq) 12531da177e4SLinus Torvalds atomic_dec(&srq->usecnt); 12541da177e4SLinus Torvalds } 12551da177e4SLinus Torvalds 12561da177e4SLinus Torvalds return ret; 12571da177e4SLinus Torvalds } 12581da177e4SLinus Torvalds EXPORT_SYMBOL(ib_destroy_qp); 12591da177e4SLinus Torvalds 12601da177e4SLinus Torvalds /* Completion queues */ 12611da177e4SLinus Torvalds 12621da177e4SLinus Torvalds struct ib_cq *ib_create_cq(struct ib_device *device, 12631da177e4SLinus Torvalds ib_comp_handler comp_handler, 12641da177e4SLinus Torvalds void (*event_handler)(struct ib_event *, void *), 12658e37210bSMatan Barak void *cq_context, 12668e37210bSMatan Barak const struct ib_cq_init_attr *cq_attr) 12671da177e4SLinus Torvalds { 12681da177e4SLinus Torvalds struct ib_cq *cq; 12691da177e4SLinus Torvalds 12708e37210bSMatan Barak cq = device->create_cq(device, cq_attr, NULL, NULL); 12711da177e4SLinus Torvalds 12721da177e4SLinus Torvalds if (!IS_ERR(cq)) { 12731da177e4SLinus Torvalds cq->device = device; 1274b5e81bf5SRoland Dreier cq->uobject = NULL; 12751da177e4SLinus Torvalds cq->comp_handler = comp_handler; 12761da177e4SLinus Torvalds cq->event_handler = event_handler; 12771da177e4SLinus Torvalds cq->cq_context = cq_context; 12781da177e4SLinus Torvalds atomic_set(&cq->usecnt, 0); 12791da177e4SLinus Torvalds } 12801da177e4SLinus Torvalds 12811da177e4SLinus Torvalds return cq; 12821da177e4SLinus Torvalds } 12831da177e4SLinus Torvalds EXPORT_SYMBOL(ib_create_cq); 12841da177e4SLinus Torvalds 12852dd57162SEli Cohen int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 12862dd57162SEli Cohen { 12872dd57162SEli Cohen return cq->device->modify_cq ? 12882dd57162SEli Cohen cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS; 12892dd57162SEli Cohen } 12902dd57162SEli Cohen EXPORT_SYMBOL(ib_modify_cq); 12912dd57162SEli Cohen 12921da177e4SLinus Torvalds int ib_destroy_cq(struct ib_cq *cq) 12931da177e4SLinus Torvalds { 12941da177e4SLinus Torvalds if (atomic_read(&cq->usecnt)) 12951da177e4SLinus Torvalds return -EBUSY; 12961da177e4SLinus Torvalds 12971da177e4SLinus Torvalds return cq->device->destroy_cq(cq); 12981da177e4SLinus Torvalds } 12991da177e4SLinus Torvalds EXPORT_SYMBOL(ib_destroy_cq); 13001da177e4SLinus Torvalds 1301a74cd4afSRoland Dreier int ib_resize_cq(struct ib_cq *cq, int cqe) 13021da177e4SLinus Torvalds { 130340de2e54SRoland Dreier return cq->device->resize_cq ? 130433b9b3eeSRoland Dreier cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS; 13051da177e4SLinus Torvalds } 13061da177e4SLinus Torvalds EXPORT_SYMBOL(ib_resize_cq); 13071da177e4SLinus Torvalds 13081da177e4SLinus Torvalds /* Memory regions */ 13091da177e4SLinus Torvalds 13101da177e4SLinus Torvalds struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags) 13111da177e4SLinus Torvalds { 13121da177e4SLinus Torvalds struct ib_mr *mr; 13131c636f80SEli Cohen int err; 13141c636f80SEli Cohen 13151c636f80SEli Cohen err = ib_check_mr_access(mr_access_flags); 13161c636f80SEli Cohen if (err) 13171c636f80SEli Cohen return ERR_PTR(err); 13181da177e4SLinus Torvalds 13191da177e4SLinus Torvalds mr = pd->device->get_dma_mr(pd, mr_access_flags); 13201da177e4SLinus Torvalds 13211da177e4SLinus Torvalds if (!IS_ERR(mr)) { 13221da177e4SLinus Torvalds mr->device = pd->device; 13231da177e4SLinus Torvalds mr->pd = pd; 1324b5e81bf5SRoland Dreier mr->uobject = NULL; 13251da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 13261da177e4SLinus Torvalds atomic_set(&mr->usecnt, 0); 13271da177e4SLinus Torvalds } 13281da177e4SLinus Torvalds 13291da177e4SLinus Torvalds return mr; 13301da177e4SLinus Torvalds } 13311da177e4SLinus Torvalds EXPORT_SYMBOL(ib_get_dma_mr); 13321da177e4SLinus Torvalds 13331da177e4SLinus Torvalds int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr) 13341da177e4SLinus Torvalds { 13351da177e4SLinus Torvalds return mr->device->query_mr ? 13361da177e4SLinus Torvalds mr->device->query_mr(mr, mr_attr) : -ENOSYS; 13371da177e4SLinus Torvalds } 13381da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_mr); 13391da177e4SLinus Torvalds 13401da177e4SLinus Torvalds int ib_dereg_mr(struct ib_mr *mr) 13411da177e4SLinus Torvalds { 13421da177e4SLinus Torvalds struct ib_pd *pd; 13431da177e4SLinus Torvalds int ret; 13441da177e4SLinus Torvalds 13451da177e4SLinus Torvalds if (atomic_read(&mr->usecnt)) 13461da177e4SLinus Torvalds return -EBUSY; 13471da177e4SLinus Torvalds 13481da177e4SLinus Torvalds pd = mr->pd; 13491da177e4SLinus Torvalds ret = mr->device->dereg_mr(mr); 13501da177e4SLinus Torvalds if (!ret) 13511da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 13521da177e4SLinus Torvalds 13531da177e4SLinus Torvalds return ret; 13541da177e4SLinus Torvalds } 13551da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dereg_mr); 13561da177e4SLinus Torvalds 13579bee178bSSagi Grimberg /** 13589bee178bSSagi Grimberg * ib_alloc_mr() - Allocates a memory region 13599bee178bSSagi Grimberg * @pd: protection domain associated with the region 13609bee178bSSagi Grimberg * @mr_type: memory region type 13619bee178bSSagi Grimberg * @max_num_sg: maximum sg entries available for registration. 13629bee178bSSagi Grimberg * 13639bee178bSSagi Grimberg * Notes: 13649bee178bSSagi Grimberg * Memory registeration page/sg lists must not exceed max_num_sg. 13659bee178bSSagi Grimberg * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed 13669bee178bSSagi Grimberg * max_num_sg * used_page_size. 13679bee178bSSagi Grimberg * 13689bee178bSSagi Grimberg */ 13699bee178bSSagi Grimberg struct ib_mr *ib_alloc_mr(struct ib_pd *pd, 13709bee178bSSagi Grimberg enum ib_mr_type mr_type, 13719bee178bSSagi Grimberg u32 max_num_sg) 137217cd3a2dSSagi Grimberg { 137317cd3a2dSSagi Grimberg struct ib_mr *mr; 137417cd3a2dSSagi Grimberg 1375d9f272c5SSagi Grimberg if (!pd->device->alloc_mr) 137617cd3a2dSSagi Grimberg return ERR_PTR(-ENOSYS); 137717cd3a2dSSagi Grimberg 1378d9f272c5SSagi Grimberg mr = pd->device->alloc_mr(pd, mr_type, max_num_sg); 137917cd3a2dSSagi Grimberg if (!IS_ERR(mr)) { 138017cd3a2dSSagi Grimberg mr->device = pd->device; 138117cd3a2dSSagi Grimberg mr->pd = pd; 138217cd3a2dSSagi Grimberg mr->uobject = NULL; 138317cd3a2dSSagi Grimberg atomic_inc(&pd->usecnt); 138417cd3a2dSSagi Grimberg atomic_set(&mr->usecnt, 0); 138517cd3a2dSSagi Grimberg } 138617cd3a2dSSagi Grimberg 138717cd3a2dSSagi Grimberg return mr; 138817cd3a2dSSagi Grimberg } 13899bee178bSSagi Grimberg EXPORT_SYMBOL(ib_alloc_mr); 139000f7ec36SSteve Wise 13911da177e4SLinus Torvalds /* Memory windows */ 13921da177e4SLinus Torvalds 13937083e42eSShani Michaeli struct ib_mw *ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) 13941da177e4SLinus Torvalds { 13951da177e4SLinus Torvalds struct ib_mw *mw; 13961da177e4SLinus Torvalds 13971da177e4SLinus Torvalds if (!pd->device->alloc_mw) 13981da177e4SLinus Torvalds return ERR_PTR(-ENOSYS); 13991da177e4SLinus Torvalds 14007083e42eSShani Michaeli mw = pd->device->alloc_mw(pd, type); 14011da177e4SLinus Torvalds if (!IS_ERR(mw)) { 14021da177e4SLinus Torvalds mw->device = pd->device; 14031da177e4SLinus Torvalds mw->pd = pd; 1404b5e81bf5SRoland Dreier mw->uobject = NULL; 14057083e42eSShani Michaeli mw->type = type; 14061da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 14071da177e4SLinus Torvalds } 14081da177e4SLinus Torvalds 14091da177e4SLinus Torvalds return mw; 14101da177e4SLinus Torvalds } 14111da177e4SLinus Torvalds EXPORT_SYMBOL(ib_alloc_mw); 14121da177e4SLinus Torvalds 14131da177e4SLinus Torvalds int ib_dealloc_mw(struct ib_mw *mw) 14141da177e4SLinus Torvalds { 14151da177e4SLinus Torvalds struct ib_pd *pd; 14161da177e4SLinus Torvalds int ret; 14171da177e4SLinus Torvalds 14181da177e4SLinus Torvalds pd = mw->pd; 14191da177e4SLinus Torvalds ret = mw->device->dealloc_mw(mw); 14201da177e4SLinus Torvalds if (!ret) 14211da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 14221da177e4SLinus Torvalds 14231da177e4SLinus Torvalds return ret; 14241da177e4SLinus Torvalds } 14251da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_mw); 14261da177e4SLinus Torvalds 14271da177e4SLinus Torvalds /* "Fast" memory regions */ 14281da177e4SLinus Torvalds 14291da177e4SLinus Torvalds struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd, 14301da177e4SLinus Torvalds int mr_access_flags, 14311da177e4SLinus Torvalds struct ib_fmr_attr *fmr_attr) 14321da177e4SLinus Torvalds { 14331da177e4SLinus Torvalds struct ib_fmr *fmr; 14341da177e4SLinus Torvalds 14351da177e4SLinus Torvalds if (!pd->device->alloc_fmr) 14361da177e4SLinus Torvalds return ERR_PTR(-ENOSYS); 14371da177e4SLinus Torvalds 14381da177e4SLinus Torvalds fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr); 14391da177e4SLinus Torvalds if (!IS_ERR(fmr)) { 14401da177e4SLinus Torvalds fmr->device = pd->device; 14411da177e4SLinus Torvalds fmr->pd = pd; 14421da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 14431da177e4SLinus Torvalds } 14441da177e4SLinus Torvalds 14451da177e4SLinus Torvalds return fmr; 14461da177e4SLinus Torvalds } 14471da177e4SLinus Torvalds EXPORT_SYMBOL(ib_alloc_fmr); 14481da177e4SLinus Torvalds 14491da177e4SLinus Torvalds int ib_unmap_fmr(struct list_head *fmr_list) 14501da177e4SLinus Torvalds { 14511da177e4SLinus Torvalds struct ib_fmr *fmr; 14521da177e4SLinus Torvalds 14531da177e4SLinus Torvalds if (list_empty(fmr_list)) 14541da177e4SLinus Torvalds return 0; 14551da177e4SLinus Torvalds 14561da177e4SLinus Torvalds fmr = list_entry(fmr_list->next, struct ib_fmr, list); 14571da177e4SLinus Torvalds return fmr->device->unmap_fmr(fmr_list); 14581da177e4SLinus Torvalds } 14591da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unmap_fmr); 14601da177e4SLinus Torvalds 14611da177e4SLinus Torvalds int ib_dealloc_fmr(struct ib_fmr *fmr) 14621da177e4SLinus Torvalds { 14631da177e4SLinus Torvalds struct ib_pd *pd; 14641da177e4SLinus Torvalds int ret; 14651da177e4SLinus Torvalds 14661da177e4SLinus Torvalds pd = fmr->pd; 14671da177e4SLinus Torvalds ret = fmr->device->dealloc_fmr(fmr); 14681da177e4SLinus Torvalds if (!ret) 14691da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 14701da177e4SLinus Torvalds 14711da177e4SLinus Torvalds return ret; 14721da177e4SLinus Torvalds } 14731da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_fmr); 14741da177e4SLinus Torvalds 14751da177e4SLinus Torvalds /* Multicast groups */ 14761da177e4SLinus Torvalds 14771da177e4SLinus Torvalds int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 14781da177e4SLinus Torvalds { 1479c3bccbfbSOr Gerlitz int ret; 1480c3bccbfbSOr Gerlitz 14810c33aeedSJack Morgenstein if (!qp->device->attach_mcast) 14820c33aeedSJack Morgenstein return -ENOSYS; 14830c33aeedSJack Morgenstein if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 14840c33aeedSJack Morgenstein return -EINVAL; 14850c33aeedSJack Morgenstein 1486c3bccbfbSOr Gerlitz ret = qp->device->attach_mcast(qp, gid, lid); 1487c3bccbfbSOr Gerlitz if (!ret) 1488c3bccbfbSOr Gerlitz atomic_inc(&qp->usecnt); 1489c3bccbfbSOr Gerlitz return ret; 14901da177e4SLinus Torvalds } 14911da177e4SLinus Torvalds EXPORT_SYMBOL(ib_attach_mcast); 14921da177e4SLinus Torvalds 14931da177e4SLinus Torvalds int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 14941da177e4SLinus Torvalds { 1495c3bccbfbSOr Gerlitz int ret; 1496c3bccbfbSOr Gerlitz 14970c33aeedSJack Morgenstein if (!qp->device->detach_mcast) 14980c33aeedSJack Morgenstein return -ENOSYS; 14990c33aeedSJack Morgenstein if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 15000c33aeedSJack Morgenstein return -EINVAL; 15010c33aeedSJack Morgenstein 1502c3bccbfbSOr Gerlitz ret = qp->device->detach_mcast(qp, gid, lid); 1503c3bccbfbSOr Gerlitz if (!ret) 1504c3bccbfbSOr Gerlitz atomic_dec(&qp->usecnt); 1505c3bccbfbSOr Gerlitz return ret; 15061da177e4SLinus Torvalds } 15071da177e4SLinus Torvalds EXPORT_SYMBOL(ib_detach_mcast); 150859991f94SSean Hefty 150959991f94SSean Hefty struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device) 151059991f94SSean Hefty { 151159991f94SSean Hefty struct ib_xrcd *xrcd; 151259991f94SSean Hefty 151359991f94SSean Hefty if (!device->alloc_xrcd) 151459991f94SSean Hefty return ERR_PTR(-ENOSYS); 151559991f94SSean Hefty 151659991f94SSean Hefty xrcd = device->alloc_xrcd(device, NULL, NULL); 151759991f94SSean Hefty if (!IS_ERR(xrcd)) { 151859991f94SSean Hefty xrcd->device = device; 151953d0bd1eSSean Hefty xrcd->inode = NULL; 152059991f94SSean Hefty atomic_set(&xrcd->usecnt, 0); 1521d3d72d90SSean Hefty mutex_init(&xrcd->tgt_qp_mutex); 1522d3d72d90SSean Hefty INIT_LIST_HEAD(&xrcd->tgt_qp_list); 152359991f94SSean Hefty } 152459991f94SSean Hefty 152559991f94SSean Hefty return xrcd; 152659991f94SSean Hefty } 152759991f94SSean Hefty EXPORT_SYMBOL(ib_alloc_xrcd); 152859991f94SSean Hefty 152959991f94SSean Hefty int ib_dealloc_xrcd(struct ib_xrcd *xrcd) 153059991f94SSean Hefty { 1531d3d72d90SSean Hefty struct ib_qp *qp; 1532d3d72d90SSean Hefty int ret; 1533d3d72d90SSean Hefty 153459991f94SSean Hefty if (atomic_read(&xrcd->usecnt)) 153559991f94SSean Hefty return -EBUSY; 153659991f94SSean Hefty 1537d3d72d90SSean Hefty while (!list_empty(&xrcd->tgt_qp_list)) { 1538d3d72d90SSean Hefty qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list); 1539d3d72d90SSean Hefty ret = ib_destroy_qp(qp); 1540d3d72d90SSean Hefty if (ret) 1541d3d72d90SSean Hefty return ret; 1542d3d72d90SSean Hefty } 1543d3d72d90SSean Hefty 154459991f94SSean Hefty return xrcd->device->dealloc_xrcd(xrcd); 154559991f94SSean Hefty } 154659991f94SSean Hefty EXPORT_SYMBOL(ib_dealloc_xrcd); 1547319a441dSHadar Hen Zion 1548319a441dSHadar Hen Zion struct ib_flow *ib_create_flow(struct ib_qp *qp, 1549319a441dSHadar Hen Zion struct ib_flow_attr *flow_attr, 1550319a441dSHadar Hen Zion int domain) 1551319a441dSHadar Hen Zion { 1552319a441dSHadar Hen Zion struct ib_flow *flow_id; 1553319a441dSHadar Hen Zion if (!qp->device->create_flow) 1554319a441dSHadar Hen Zion return ERR_PTR(-ENOSYS); 1555319a441dSHadar Hen Zion 1556319a441dSHadar Hen Zion flow_id = qp->device->create_flow(qp, flow_attr, domain); 1557319a441dSHadar Hen Zion if (!IS_ERR(flow_id)) 1558319a441dSHadar Hen Zion atomic_inc(&qp->usecnt); 1559319a441dSHadar Hen Zion return flow_id; 1560319a441dSHadar Hen Zion } 1561319a441dSHadar Hen Zion EXPORT_SYMBOL(ib_create_flow); 1562319a441dSHadar Hen Zion 1563319a441dSHadar Hen Zion int ib_destroy_flow(struct ib_flow *flow_id) 1564319a441dSHadar Hen Zion { 1565319a441dSHadar Hen Zion int err; 1566319a441dSHadar Hen Zion struct ib_qp *qp = flow_id->qp; 1567319a441dSHadar Hen Zion 1568319a441dSHadar Hen Zion err = qp->device->destroy_flow(flow_id); 1569319a441dSHadar Hen Zion if (!err) 1570319a441dSHadar Hen Zion atomic_dec(&qp->usecnt); 1571319a441dSHadar Hen Zion return err; 1572319a441dSHadar Hen Zion } 1573319a441dSHadar Hen Zion EXPORT_SYMBOL(ib_destroy_flow); 15741b01d335SSagi Grimberg 15751b01d335SSagi Grimberg int ib_check_mr_status(struct ib_mr *mr, u32 check_mask, 15761b01d335SSagi Grimberg struct ib_mr_status *mr_status) 15771b01d335SSagi Grimberg { 15781b01d335SSagi Grimberg return mr->device->check_mr_status ? 15791b01d335SSagi Grimberg mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS; 15801b01d335SSagi Grimberg } 15811b01d335SSagi Grimberg EXPORT_SYMBOL(ib_check_mr_status); 15824c67e2bfSSagi Grimberg 15834c67e2bfSSagi Grimberg /** 15844c67e2bfSSagi Grimberg * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list 15854c67e2bfSSagi Grimberg * and set it the memory region. 15864c67e2bfSSagi Grimberg * @mr: memory region 15874c67e2bfSSagi Grimberg * @sg: dma mapped scatterlist 15884c67e2bfSSagi Grimberg * @sg_nents: number of entries in sg 15894c67e2bfSSagi Grimberg * @page_size: page vector desired page size 15904c67e2bfSSagi Grimberg * 15914c67e2bfSSagi Grimberg * Constraints: 15924c67e2bfSSagi Grimberg * - The first sg element is allowed to have an offset. 15934c67e2bfSSagi Grimberg * - Each sg element must be aligned to page_size (or physically 15944c67e2bfSSagi Grimberg * contiguous to the previous element). In case an sg element has a 15954c67e2bfSSagi Grimberg * non contiguous offset, the mapping prefix will not include it. 15964c67e2bfSSagi Grimberg * - The last sg element is allowed to have length less than page_size. 15974c67e2bfSSagi Grimberg * - If sg_nents total byte length exceeds the mr max_num_sge * page_size 15984c67e2bfSSagi Grimberg * then only max_num_sg entries will be mapped. 15994c67e2bfSSagi Grimberg * 16004c67e2bfSSagi Grimberg * Returns the number of sg elements that were mapped to the memory region. 16014c67e2bfSSagi Grimberg * 16024c67e2bfSSagi Grimberg * After this completes successfully, the memory region 16034c67e2bfSSagi Grimberg * is ready for registration. 16044c67e2bfSSagi Grimberg */ 16054c67e2bfSSagi Grimberg int ib_map_mr_sg(struct ib_mr *mr, 16064c67e2bfSSagi Grimberg struct scatterlist *sg, 16074c67e2bfSSagi Grimberg int sg_nents, 16084c67e2bfSSagi Grimberg unsigned int page_size) 16094c67e2bfSSagi Grimberg { 16104c67e2bfSSagi Grimberg if (unlikely(!mr->device->map_mr_sg)) 16114c67e2bfSSagi Grimberg return -ENOSYS; 16124c67e2bfSSagi Grimberg 16134c67e2bfSSagi Grimberg mr->page_size = page_size; 16144c67e2bfSSagi Grimberg 16154c67e2bfSSagi Grimberg return mr->device->map_mr_sg(mr, sg, sg_nents); 16164c67e2bfSSagi Grimberg } 16174c67e2bfSSagi Grimberg EXPORT_SYMBOL(ib_map_mr_sg); 16184c67e2bfSSagi Grimberg 16194c67e2bfSSagi Grimberg /** 16204c67e2bfSSagi Grimberg * ib_sg_to_pages() - Convert the largest prefix of a sg list 16214c67e2bfSSagi Grimberg * to a page vector 16224c67e2bfSSagi Grimberg * @mr: memory region 16234c67e2bfSSagi Grimberg * @sgl: dma mapped scatterlist 16244c67e2bfSSagi Grimberg * @sg_nents: number of entries in sg 16254c67e2bfSSagi Grimberg * @set_page: driver page assignment function pointer 16264c67e2bfSSagi Grimberg * 16278f5ba10eSBart Van Assche * Core service helper for drivers to convert the largest 16284c67e2bfSSagi Grimberg * prefix of given sg list to a page vector. The sg list 16294c67e2bfSSagi Grimberg * prefix converted is the prefix that meet the requirements 16304c67e2bfSSagi Grimberg * of ib_map_mr_sg. 16314c67e2bfSSagi Grimberg * 16324c67e2bfSSagi Grimberg * Returns the number of sg elements that were assigned to 16334c67e2bfSSagi Grimberg * a page vector. 16344c67e2bfSSagi Grimberg */ 16354c67e2bfSSagi Grimberg int ib_sg_to_pages(struct ib_mr *mr, 16364c67e2bfSSagi Grimberg struct scatterlist *sgl, 16374c67e2bfSSagi Grimberg int sg_nents, 16384c67e2bfSSagi Grimberg int (*set_page)(struct ib_mr *, u64)) 16394c67e2bfSSagi Grimberg { 16404c67e2bfSSagi Grimberg struct scatterlist *sg; 16414c67e2bfSSagi Grimberg u64 last_end_dma_addr = 0, last_page_addr = 0; 16424c67e2bfSSagi Grimberg unsigned int last_page_off = 0; 16434c67e2bfSSagi Grimberg u64 page_mask = ~((u64)mr->page_size - 1); 16448f5ba10eSBart Van Assche int i, ret; 16454c67e2bfSSagi Grimberg 16464c67e2bfSSagi Grimberg mr->iova = sg_dma_address(&sgl[0]); 16474c67e2bfSSagi Grimberg mr->length = 0; 16484c67e2bfSSagi Grimberg 16494c67e2bfSSagi Grimberg for_each_sg(sgl, sg, sg_nents, i) { 16504c67e2bfSSagi Grimberg u64 dma_addr = sg_dma_address(sg); 16514c67e2bfSSagi Grimberg unsigned int dma_len = sg_dma_len(sg); 16524c67e2bfSSagi Grimberg u64 end_dma_addr = dma_addr + dma_len; 16534c67e2bfSSagi Grimberg u64 page_addr = dma_addr & page_mask; 16544c67e2bfSSagi Grimberg 16558f5ba10eSBart Van Assche /* 16568f5ba10eSBart Van Assche * For the second and later elements, check whether either the 16578f5ba10eSBart Van Assche * end of element i-1 or the start of element i is not aligned 16588f5ba10eSBart Van Assche * on a page boundary. 16598f5ba10eSBart Van Assche */ 16608f5ba10eSBart Van Assche if (i && (last_page_off != 0 || page_addr != dma_addr)) { 16618f5ba10eSBart Van Assche /* Stop mapping if there is a gap. */ 16628f5ba10eSBart Van Assche if (last_end_dma_addr != dma_addr) 16638f5ba10eSBart Van Assche break; 16644c67e2bfSSagi Grimberg 16658f5ba10eSBart Van Assche /* 16668f5ba10eSBart Van Assche * Coalesce this element with the last. If it is small 16678f5ba10eSBart Van Assche * enough just update mr->length. Otherwise start 16688f5ba10eSBart Van Assche * mapping from the next page. 16698f5ba10eSBart Van Assche */ 16708f5ba10eSBart Van Assche goto next_page; 16714c67e2bfSSagi Grimberg } 16724c67e2bfSSagi Grimberg 16734c67e2bfSSagi Grimberg do { 16748f5ba10eSBart Van Assche ret = set_page(mr, page_addr); 16758f5ba10eSBart Van Assche if (unlikely(ret < 0)) 16768f5ba10eSBart Van Assche return i ? : ret; 16778f5ba10eSBart Van Assche next_page: 16784c67e2bfSSagi Grimberg page_addr += mr->page_size; 16794c67e2bfSSagi Grimberg } while (page_addr < end_dma_addr); 16804c67e2bfSSagi Grimberg 16814c67e2bfSSagi Grimberg mr->length += dma_len; 16824c67e2bfSSagi Grimberg last_end_dma_addr = end_dma_addr; 16834c67e2bfSSagi Grimberg last_page_addr = end_dma_addr & page_mask; 16844c67e2bfSSagi Grimberg last_page_off = end_dma_addr & ~page_mask; 16854c67e2bfSSagi Grimberg } 16864c67e2bfSSagi Grimberg 16874c67e2bfSSagi Grimberg return i; 16884c67e2bfSSagi Grimberg } 16894c67e2bfSSagi Grimberg EXPORT_SYMBOL(ib_sg_to_pages); 1690