11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. 31da177e4SLinus Torvalds * Copyright (c) 2004 Infinicon Corporation. All rights reserved. 41da177e4SLinus Torvalds * Copyright (c) 2004 Intel Corporation. All rights reserved. 51da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Corporation. All rights reserved. 61da177e4SLinus Torvalds * Copyright (c) 2004 Voltaire Corporation. All rights reserved. 72a1d9b7fSRoland Dreier * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 833b9b3eeSRoland Dreier * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 91da177e4SLinus Torvalds * 101da177e4SLinus Torvalds * This software is available to you under a choice of one of two 111da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 121da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 131da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 141da177e4SLinus Torvalds * OpenIB.org BSD license below: 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 171da177e4SLinus Torvalds * without modification, are permitted provided that the following 181da177e4SLinus Torvalds * conditions are met: 191da177e4SLinus Torvalds * 201da177e4SLinus Torvalds * - Redistributions of source code must retain the above 211da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 221da177e4SLinus Torvalds * disclaimer. 231da177e4SLinus Torvalds * 241da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 251da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 261da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 271da177e4SLinus Torvalds * provided with the distribution. 281da177e4SLinus Torvalds * 291da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 301da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 311da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 321da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 331da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 341da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 351da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 361da177e4SLinus Torvalds * SOFTWARE. 371da177e4SLinus Torvalds */ 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds #include <linux/errno.h> 401da177e4SLinus Torvalds #include <linux/err.h> 41b108d976SPaul Gortmaker #include <linux/export.h> 428c65b4a6STim Schmielau #include <linux/string.h> 430e0ec7e0SSean Hefty #include <linux/slab.h> 44dbf727deSMatan Barak #include <linux/in.h> 45dbf727deSMatan Barak #include <linux/in6.h> 46dbf727deSMatan Barak #include <net/addrconf.h> 471da177e4SLinus Torvalds 48a4d61e84SRoland Dreier #include <rdma/ib_verbs.h> 49a4d61e84SRoland Dreier #include <rdma/ib_cache.h> 50dd5f03beSMatan Barak #include <rdma/ib_addr.h> 511da177e4SLinus Torvalds 52ed4c54e5SOr Gerlitz #include "core_priv.h" 531da177e4SLinus Torvalds 542b1b5b60SSagi Grimberg static const char * const ib_events[] = { 552b1b5b60SSagi Grimberg [IB_EVENT_CQ_ERR] = "CQ error", 562b1b5b60SSagi Grimberg [IB_EVENT_QP_FATAL] = "QP fatal error", 572b1b5b60SSagi Grimberg [IB_EVENT_QP_REQ_ERR] = "QP request error", 582b1b5b60SSagi Grimberg [IB_EVENT_QP_ACCESS_ERR] = "QP access error", 592b1b5b60SSagi Grimberg [IB_EVENT_COMM_EST] = "communication established", 602b1b5b60SSagi Grimberg [IB_EVENT_SQ_DRAINED] = "send queue drained", 612b1b5b60SSagi Grimberg [IB_EVENT_PATH_MIG] = "path migration successful", 622b1b5b60SSagi Grimberg [IB_EVENT_PATH_MIG_ERR] = "path migration error", 632b1b5b60SSagi Grimberg [IB_EVENT_DEVICE_FATAL] = "device fatal error", 642b1b5b60SSagi Grimberg [IB_EVENT_PORT_ACTIVE] = "port active", 652b1b5b60SSagi Grimberg [IB_EVENT_PORT_ERR] = "port error", 662b1b5b60SSagi Grimberg [IB_EVENT_LID_CHANGE] = "LID change", 672b1b5b60SSagi Grimberg [IB_EVENT_PKEY_CHANGE] = "P_key change", 682b1b5b60SSagi Grimberg [IB_EVENT_SM_CHANGE] = "SM change", 692b1b5b60SSagi Grimberg [IB_EVENT_SRQ_ERR] = "SRQ error", 702b1b5b60SSagi Grimberg [IB_EVENT_SRQ_LIMIT_REACHED] = "SRQ limit reached", 712b1b5b60SSagi Grimberg [IB_EVENT_QP_LAST_WQE_REACHED] = "last WQE reached", 722b1b5b60SSagi Grimberg [IB_EVENT_CLIENT_REREGISTER] = "client reregister", 732b1b5b60SSagi Grimberg [IB_EVENT_GID_CHANGE] = "GID changed", 742b1b5b60SSagi Grimberg }; 752b1b5b60SSagi Grimberg 76db7489e0SBart Van Assche const char *__attribute_const__ ib_event_msg(enum ib_event_type event) 772b1b5b60SSagi Grimberg { 782b1b5b60SSagi Grimberg size_t index = event; 792b1b5b60SSagi Grimberg 802b1b5b60SSagi Grimberg return (index < ARRAY_SIZE(ib_events) && ib_events[index]) ? 812b1b5b60SSagi Grimberg ib_events[index] : "unrecognized event"; 822b1b5b60SSagi Grimberg } 832b1b5b60SSagi Grimberg EXPORT_SYMBOL(ib_event_msg); 842b1b5b60SSagi Grimberg 852b1b5b60SSagi Grimberg static const char * const wc_statuses[] = { 862b1b5b60SSagi Grimberg [IB_WC_SUCCESS] = "success", 872b1b5b60SSagi Grimberg [IB_WC_LOC_LEN_ERR] = "local length error", 882b1b5b60SSagi Grimberg [IB_WC_LOC_QP_OP_ERR] = "local QP operation error", 892b1b5b60SSagi Grimberg [IB_WC_LOC_EEC_OP_ERR] = "local EE context operation error", 902b1b5b60SSagi Grimberg [IB_WC_LOC_PROT_ERR] = "local protection error", 912b1b5b60SSagi Grimberg [IB_WC_WR_FLUSH_ERR] = "WR flushed", 922b1b5b60SSagi Grimberg [IB_WC_MW_BIND_ERR] = "memory management operation error", 932b1b5b60SSagi Grimberg [IB_WC_BAD_RESP_ERR] = "bad response error", 942b1b5b60SSagi Grimberg [IB_WC_LOC_ACCESS_ERR] = "local access error", 952b1b5b60SSagi Grimberg [IB_WC_REM_INV_REQ_ERR] = "invalid request error", 962b1b5b60SSagi Grimberg [IB_WC_REM_ACCESS_ERR] = "remote access error", 972b1b5b60SSagi Grimberg [IB_WC_REM_OP_ERR] = "remote operation error", 982b1b5b60SSagi Grimberg [IB_WC_RETRY_EXC_ERR] = "transport retry counter exceeded", 992b1b5b60SSagi Grimberg [IB_WC_RNR_RETRY_EXC_ERR] = "RNR retry counter exceeded", 1002b1b5b60SSagi Grimberg [IB_WC_LOC_RDD_VIOL_ERR] = "local RDD violation error", 1012b1b5b60SSagi Grimberg [IB_WC_REM_INV_RD_REQ_ERR] = "remote invalid RD request", 1022b1b5b60SSagi Grimberg [IB_WC_REM_ABORT_ERR] = "operation aborted", 1032b1b5b60SSagi Grimberg [IB_WC_INV_EECN_ERR] = "invalid EE context number", 1042b1b5b60SSagi Grimberg [IB_WC_INV_EEC_STATE_ERR] = "invalid EE context state", 1052b1b5b60SSagi Grimberg [IB_WC_FATAL_ERR] = "fatal error", 1062b1b5b60SSagi Grimberg [IB_WC_RESP_TIMEOUT_ERR] = "response timeout error", 1072b1b5b60SSagi Grimberg [IB_WC_GENERAL_ERR] = "general error", 1082b1b5b60SSagi Grimberg }; 1092b1b5b60SSagi Grimberg 110db7489e0SBart Van Assche const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status) 1112b1b5b60SSagi Grimberg { 1122b1b5b60SSagi Grimberg size_t index = status; 1132b1b5b60SSagi Grimberg 1142b1b5b60SSagi Grimberg return (index < ARRAY_SIZE(wc_statuses) && wc_statuses[index]) ? 1152b1b5b60SSagi Grimberg wc_statuses[index] : "unrecognized status"; 1162b1b5b60SSagi Grimberg } 1172b1b5b60SSagi Grimberg EXPORT_SYMBOL(ib_wc_status_msg); 1182b1b5b60SSagi Grimberg 1198385fd84SRoland Dreier __attribute_const__ int ib_rate_to_mult(enum ib_rate rate) 120bf6a9e31SJack Morgenstein { 121bf6a9e31SJack Morgenstein switch (rate) { 122bf6a9e31SJack Morgenstein case IB_RATE_2_5_GBPS: return 1; 123bf6a9e31SJack Morgenstein case IB_RATE_5_GBPS: return 2; 124bf6a9e31SJack Morgenstein case IB_RATE_10_GBPS: return 4; 125bf6a9e31SJack Morgenstein case IB_RATE_20_GBPS: return 8; 126bf6a9e31SJack Morgenstein case IB_RATE_30_GBPS: return 12; 127bf6a9e31SJack Morgenstein case IB_RATE_40_GBPS: return 16; 128bf6a9e31SJack Morgenstein case IB_RATE_60_GBPS: return 24; 129bf6a9e31SJack Morgenstein case IB_RATE_80_GBPS: return 32; 130bf6a9e31SJack Morgenstein case IB_RATE_120_GBPS: return 48; 131bf6a9e31SJack Morgenstein default: return -1; 132bf6a9e31SJack Morgenstein } 133bf6a9e31SJack Morgenstein } 134bf6a9e31SJack Morgenstein EXPORT_SYMBOL(ib_rate_to_mult); 135bf6a9e31SJack Morgenstein 1368385fd84SRoland Dreier __attribute_const__ enum ib_rate mult_to_ib_rate(int mult) 137bf6a9e31SJack Morgenstein { 138bf6a9e31SJack Morgenstein switch (mult) { 139bf6a9e31SJack Morgenstein case 1: return IB_RATE_2_5_GBPS; 140bf6a9e31SJack Morgenstein case 2: return IB_RATE_5_GBPS; 141bf6a9e31SJack Morgenstein case 4: return IB_RATE_10_GBPS; 142bf6a9e31SJack Morgenstein case 8: return IB_RATE_20_GBPS; 143bf6a9e31SJack Morgenstein case 12: return IB_RATE_30_GBPS; 144bf6a9e31SJack Morgenstein case 16: return IB_RATE_40_GBPS; 145bf6a9e31SJack Morgenstein case 24: return IB_RATE_60_GBPS; 146bf6a9e31SJack Morgenstein case 32: return IB_RATE_80_GBPS; 147bf6a9e31SJack Morgenstein case 48: return IB_RATE_120_GBPS; 148bf6a9e31SJack Morgenstein default: return IB_RATE_PORT_CURRENT; 149bf6a9e31SJack Morgenstein } 150bf6a9e31SJack Morgenstein } 151bf6a9e31SJack Morgenstein EXPORT_SYMBOL(mult_to_ib_rate); 152bf6a9e31SJack Morgenstein 1538385fd84SRoland Dreier __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate) 15471eeba16SMarcel Apfelbaum { 15571eeba16SMarcel Apfelbaum switch (rate) { 15671eeba16SMarcel Apfelbaum case IB_RATE_2_5_GBPS: return 2500; 15771eeba16SMarcel Apfelbaum case IB_RATE_5_GBPS: return 5000; 15871eeba16SMarcel Apfelbaum case IB_RATE_10_GBPS: return 10000; 15971eeba16SMarcel Apfelbaum case IB_RATE_20_GBPS: return 20000; 16071eeba16SMarcel Apfelbaum case IB_RATE_30_GBPS: return 30000; 16171eeba16SMarcel Apfelbaum case IB_RATE_40_GBPS: return 40000; 16271eeba16SMarcel Apfelbaum case IB_RATE_60_GBPS: return 60000; 16371eeba16SMarcel Apfelbaum case IB_RATE_80_GBPS: return 80000; 16471eeba16SMarcel Apfelbaum case IB_RATE_120_GBPS: return 120000; 16571eeba16SMarcel Apfelbaum case IB_RATE_14_GBPS: return 14062; 16671eeba16SMarcel Apfelbaum case IB_RATE_56_GBPS: return 56250; 16771eeba16SMarcel Apfelbaum case IB_RATE_112_GBPS: return 112500; 16871eeba16SMarcel Apfelbaum case IB_RATE_168_GBPS: return 168750; 16971eeba16SMarcel Apfelbaum case IB_RATE_25_GBPS: return 25781; 17071eeba16SMarcel Apfelbaum case IB_RATE_100_GBPS: return 103125; 17171eeba16SMarcel Apfelbaum case IB_RATE_200_GBPS: return 206250; 17271eeba16SMarcel Apfelbaum case IB_RATE_300_GBPS: return 309375; 17371eeba16SMarcel Apfelbaum default: return -1; 17471eeba16SMarcel Apfelbaum } 17571eeba16SMarcel Apfelbaum } 17671eeba16SMarcel Apfelbaum EXPORT_SYMBOL(ib_rate_to_mbps); 17771eeba16SMarcel Apfelbaum 1788385fd84SRoland Dreier __attribute_const__ enum rdma_transport_type 17907ebafbaSTom Tucker rdma_node_get_transport(enum rdma_node_type node_type) 18007ebafbaSTom Tucker { 18107ebafbaSTom Tucker switch (node_type) { 18207ebafbaSTom Tucker case RDMA_NODE_IB_CA: 18307ebafbaSTom Tucker case RDMA_NODE_IB_SWITCH: 18407ebafbaSTom Tucker case RDMA_NODE_IB_ROUTER: 18507ebafbaSTom Tucker return RDMA_TRANSPORT_IB; 18607ebafbaSTom Tucker case RDMA_NODE_RNIC: 18707ebafbaSTom Tucker return RDMA_TRANSPORT_IWARP; 188180771a3SUpinder Malhi \(umalhi\) case RDMA_NODE_USNIC: 1895db5765eSUpinder Malhi return RDMA_TRANSPORT_USNIC; 1905db5765eSUpinder Malhi case RDMA_NODE_USNIC_UDP: 191248567f7SUpinder Malhi return RDMA_TRANSPORT_USNIC_UDP; 19207ebafbaSTom Tucker default: 19307ebafbaSTom Tucker BUG(); 19407ebafbaSTom Tucker return 0; 19507ebafbaSTom Tucker } 19607ebafbaSTom Tucker } 19707ebafbaSTom Tucker EXPORT_SYMBOL(rdma_node_get_transport); 19807ebafbaSTom Tucker 199a3f5adafSEli Cohen enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num) 200a3f5adafSEli Cohen { 201a3f5adafSEli Cohen if (device->get_link_layer) 202a3f5adafSEli Cohen return device->get_link_layer(device, port_num); 203a3f5adafSEli Cohen 204a3f5adafSEli Cohen switch (rdma_node_get_transport(device->node_type)) { 205a3f5adafSEli Cohen case RDMA_TRANSPORT_IB: 206a3f5adafSEli Cohen return IB_LINK_LAYER_INFINIBAND; 207a3f5adafSEli Cohen case RDMA_TRANSPORT_IWARP: 208180771a3SUpinder Malhi \(umalhi\) case RDMA_TRANSPORT_USNIC: 209248567f7SUpinder Malhi case RDMA_TRANSPORT_USNIC_UDP: 210a3f5adafSEli Cohen return IB_LINK_LAYER_ETHERNET; 211a3f5adafSEli Cohen default: 212a3f5adafSEli Cohen return IB_LINK_LAYER_UNSPECIFIED; 213a3f5adafSEli Cohen } 214a3f5adafSEli Cohen } 215a3f5adafSEli Cohen EXPORT_SYMBOL(rdma_port_get_link_layer); 216a3f5adafSEli Cohen 2171da177e4SLinus Torvalds /* Protection domains */ 2181da177e4SLinus Torvalds 21996249d70SJason Gunthorpe /** 22096249d70SJason Gunthorpe * ib_alloc_pd - Allocates an unused protection domain. 22196249d70SJason Gunthorpe * @device: The device on which to allocate the protection domain. 22296249d70SJason Gunthorpe * 22396249d70SJason Gunthorpe * A protection domain object provides an association between QPs, shared 22496249d70SJason Gunthorpe * receive queues, address handles, memory regions, and memory windows. 22596249d70SJason Gunthorpe * 22696249d70SJason Gunthorpe * Every PD has a local_dma_lkey which can be used as the lkey value for local 22796249d70SJason Gunthorpe * memory operations. 22896249d70SJason Gunthorpe */ 2291da177e4SLinus Torvalds struct ib_pd *ib_alloc_pd(struct ib_device *device) 2301da177e4SLinus Torvalds { 2311da177e4SLinus Torvalds struct ib_pd *pd; 2321da177e4SLinus Torvalds 233b5e81bf5SRoland Dreier pd = device->alloc_pd(device, NULL, NULL); 23496249d70SJason Gunthorpe if (IS_ERR(pd)) 23596249d70SJason Gunthorpe return pd; 2361da177e4SLinus Torvalds 2371da177e4SLinus Torvalds pd->device = device; 238b5e81bf5SRoland Dreier pd->uobject = NULL; 23996249d70SJason Gunthorpe pd->local_mr = NULL; 2401da177e4SLinus Torvalds atomic_set(&pd->usecnt, 0); 24196249d70SJason Gunthorpe 24286bee4c9SOr Gerlitz if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) 24396249d70SJason Gunthorpe pd->local_dma_lkey = device->local_dma_lkey; 24496249d70SJason Gunthorpe else { 24596249d70SJason Gunthorpe struct ib_mr *mr; 24696249d70SJason Gunthorpe 24796249d70SJason Gunthorpe mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE); 24896249d70SJason Gunthorpe if (IS_ERR(mr)) { 24996249d70SJason Gunthorpe ib_dealloc_pd(pd); 25096249d70SJason Gunthorpe return (struct ib_pd *)mr; 2511da177e4SLinus Torvalds } 2521da177e4SLinus Torvalds 25396249d70SJason Gunthorpe pd->local_mr = mr; 25496249d70SJason Gunthorpe pd->local_dma_lkey = pd->local_mr->lkey; 25596249d70SJason Gunthorpe } 2561da177e4SLinus Torvalds return pd; 2571da177e4SLinus Torvalds } 2581da177e4SLinus Torvalds EXPORT_SYMBOL(ib_alloc_pd); 2591da177e4SLinus Torvalds 2607dd78647SJason Gunthorpe /** 2617dd78647SJason Gunthorpe * ib_dealloc_pd - Deallocates a protection domain. 2627dd78647SJason Gunthorpe * @pd: The protection domain to deallocate. 2637dd78647SJason Gunthorpe * 2647dd78647SJason Gunthorpe * It is an error to call this function while any resources in the pd still 2657dd78647SJason Gunthorpe * exist. The caller is responsible to synchronously destroy them and 2667dd78647SJason Gunthorpe * guarantee no new allocations will happen. 2677dd78647SJason Gunthorpe */ 2687dd78647SJason Gunthorpe void ib_dealloc_pd(struct ib_pd *pd) 2691da177e4SLinus Torvalds { 2707dd78647SJason Gunthorpe int ret; 2711da177e4SLinus Torvalds 27296249d70SJason Gunthorpe if (pd->local_mr) { 2737dd78647SJason Gunthorpe ret = ib_dereg_mr(pd->local_mr); 2747dd78647SJason Gunthorpe WARN_ON(ret); 27596249d70SJason Gunthorpe pd->local_mr = NULL; 27696249d70SJason Gunthorpe } 27796249d70SJason Gunthorpe 2787dd78647SJason Gunthorpe /* uverbs manipulates usecnt with proper locking, while the kabi 2797dd78647SJason Gunthorpe requires the caller to guarantee we can't race here. */ 2807dd78647SJason Gunthorpe WARN_ON(atomic_read(&pd->usecnt)); 2811da177e4SLinus Torvalds 2827dd78647SJason Gunthorpe /* Making delalloc_pd a void return is a WIP, no driver should return 2837dd78647SJason Gunthorpe an error here. */ 2847dd78647SJason Gunthorpe ret = pd->device->dealloc_pd(pd); 2857dd78647SJason Gunthorpe WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd"); 2861da177e4SLinus Torvalds } 2871da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_pd); 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds /* Address handles */ 2901da177e4SLinus Torvalds 2911da177e4SLinus Torvalds struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) 2921da177e4SLinus Torvalds { 2931da177e4SLinus Torvalds struct ib_ah *ah; 2941da177e4SLinus Torvalds 2951da177e4SLinus Torvalds ah = pd->device->create_ah(pd, ah_attr); 2961da177e4SLinus Torvalds 2971da177e4SLinus Torvalds if (!IS_ERR(ah)) { 2981da177e4SLinus Torvalds ah->device = pd->device; 2991da177e4SLinus Torvalds ah->pd = pd; 300b5e81bf5SRoland Dreier ah->uobject = NULL; 3011da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 3021da177e4SLinus Torvalds } 3031da177e4SLinus Torvalds 3041da177e4SLinus Torvalds return ah; 3051da177e4SLinus Torvalds } 3061da177e4SLinus Torvalds EXPORT_SYMBOL(ib_create_ah); 3071da177e4SLinus Torvalds 308c865f246SSomnath Kotur static int ib_get_header_version(const union rdma_network_hdr *hdr) 309c865f246SSomnath Kotur { 310c865f246SSomnath Kotur const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh; 311c865f246SSomnath Kotur struct iphdr ip4h_checked; 312c865f246SSomnath Kotur const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh; 313c865f246SSomnath Kotur 314c865f246SSomnath Kotur /* If it's IPv6, the version must be 6, otherwise, the first 315c865f246SSomnath Kotur * 20 bytes (before the IPv4 header) are garbled. 316c865f246SSomnath Kotur */ 317c865f246SSomnath Kotur if (ip6h->version != 6) 318c865f246SSomnath Kotur return (ip4h->version == 4) ? 4 : 0; 319c865f246SSomnath Kotur /* version may be 6 or 4 because the first 20 bytes could be garbled */ 320c865f246SSomnath Kotur 321c865f246SSomnath Kotur /* RoCE v2 requires no options, thus header length 322c865f246SSomnath Kotur * must be 5 words 323c865f246SSomnath Kotur */ 324c865f246SSomnath Kotur if (ip4h->ihl != 5) 325c865f246SSomnath Kotur return 6; 326c865f246SSomnath Kotur 327c865f246SSomnath Kotur /* Verify checksum. 328c865f246SSomnath Kotur * We can't write on scattered buffers so we need to copy to 329c865f246SSomnath Kotur * temp buffer. 330c865f246SSomnath Kotur */ 331c865f246SSomnath Kotur memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked)); 332c865f246SSomnath Kotur ip4h_checked.check = 0; 333c865f246SSomnath Kotur ip4h_checked.check = ip_fast_csum((u8 *)&ip4h_checked, 5); 334c865f246SSomnath Kotur /* if IPv4 header checksum is OK, believe it */ 335c865f246SSomnath Kotur if (ip4h->check == ip4h_checked.check) 336c865f246SSomnath Kotur return 4; 337c865f246SSomnath Kotur return 6; 338c865f246SSomnath Kotur } 339c865f246SSomnath Kotur 340c865f246SSomnath Kotur static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device, 341c865f246SSomnath Kotur u8 port_num, 342c865f246SSomnath Kotur const struct ib_grh *grh) 343c865f246SSomnath Kotur { 344c865f246SSomnath Kotur int grh_version; 345c865f246SSomnath Kotur 346c865f246SSomnath Kotur if (rdma_protocol_ib(device, port_num)) 347c865f246SSomnath Kotur return RDMA_NETWORK_IB; 348c865f246SSomnath Kotur 349c865f246SSomnath Kotur grh_version = ib_get_header_version((union rdma_network_hdr *)grh); 350c865f246SSomnath Kotur 351c865f246SSomnath Kotur if (grh_version == 4) 352c865f246SSomnath Kotur return RDMA_NETWORK_IPV4; 353c865f246SSomnath Kotur 354c865f246SSomnath Kotur if (grh->next_hdr == IPPROTO_UDP) 355c865f246SSomnath Kotur return RDMA_NETWORK_IPV6; 356c865f246SSomnath Kotur 357c865f246SSomnath Kotur return RDMA_NETWORK_ROCE_V1; 358c865f246SSomnath Kotur } 359c865f246SSomnath Kotur 360dbf727deSMatan Barak struct find_gid_index_context { 361dbf727deSMatan Barak u16 vlan_id; 362c865f246SSomnath Kotur enum ib_gid_type gid_type; 363dbf727deSMatan Barak }; 364dbf727deSMatan Barak 365dbf727deSMatan Barak static bool find_gid_index(const union ib_gid *gid, 366dbf727deSMatan Barak const struct ib_gid_attr *gid_attr, 367dbf727deSMatan Barak void *context) 368dbf727deSMatan Barak { 369dbf727deSMatan Barak struct find_gid_index_context *ctx = 370dbf727deSMatan Barak (struct find_gid_index_context *)context; 371dbf727deSMatan Barak 372c865f246SSomnath Kotur if (ctx->gid_type != gid_attr->gid_type) 373c865f246SSomnath Kotur return false; 374c865f246SSomnath Kotur 375dbf727deSMatan Barak if ((!!(ctx->vlan_id != 0xffff) == !is_vlan_dev(gid_attr->ndev)) || 376dbf727deSMatan Barak (is_vlan_dev(gid_attr->ndev) && 377dbf727deSMatan Barak vlan_dev_vlan_id(gid_attr->ndev) != ctx->vlan_id)) 378dbf727deSMatan Barak return false; 379dbf727deSMatan Barak 380dbf727deSMatan Barak return true; 381dbf727deSMatan Barak } 382dbf727deSMatan Barak 383dbf727deSMatan Barak static int get_sgid_index_from_eth(struct ib_device *device, u8 port_num, 384dbf727deSMatan Barak u16 vlan_id, const union ib_gid *sgid, 385c865f246SSomnath Kotur enum ib_gid_type gid_type, 386dbf727deSMatan Barak u16 *gid_index) 387dbf727deSMatan Barak { 388c865f246SSomnath Kotur struct find_gid_index_context context = {.vlan_id = vlan_id, 389c865f246SSomnath Kotur .gid_type = gid_type}; 390dbf727deSMatan Barak 391dbf727deSMatan Barak return ib_find_gid_by_filter(device, sgid, port_num, find_gid_index, 392dbf727deSMatan Barak &context, gid_index); 393dbf727deSMatan Barak } 394dbf727deSMatan Barak 395c865f246SSomnath Kotur static int get_gids_from_rdma_hdr(union rdma_network_hdr *hdr, 396c865f246SSomnath Kotur enum rdma_network_type net_type, 397c865f246SSomnath Kotur union ib_gid *sgid, union ib_gid *dgid) 398c865f246SSomnath Kotur { 399c865f246SSomnath Kotur struct sockaddr_in src_in; 400c865f246SSomnath Kotur struct sockaddr_in dst_in; 401c865f246SSomnath Kotur __be32 src_saddr, dst_saddr; 402c865f246SSomnath Kotur 403c865f246SSomnath Kotur if (!sgid || !dgid) 404c865f246SSomnath Kotur return -EINVAL; 405c865f246SSomnath Kotur 406c865f246SSomnath Kotur if (net_type == RDMA_NETWORK_IPV4) { 407c865f246SSomnath Kotur memcpy(&src_in.sin_addr.s_addr, 408c865f246SSomnath Kotur &hdr->roce4grh.saddr, 4); 409c865f246SSomnath Kotur memcpy(&dst_in.sin_addr.s_addr, 410c865f246SSomnath Kotur &hdr->roce4grh.daddr, 4); 411c865f246SSomnath Kotur src_saddr = src_in.sin_addr.s_addr; 412c865f246SSomnath Kotur dst_saddr = dst_in.sin_addr.s_addr; 413c865f246SSomnath Kotur ipv6_addr_set_v4mapped(src_saddr, 414c865f246SSomnath Kotur (struct in6_addr *)sgid); 415c865f246SSomnath Kotur ipv6_addr_set_v4mapped(dst_saddr, 416c865f246SSomnath Kotur (struct in6_addr *)dgid); 417c865f246SSomnath Kotur return 0; 418c865f246SSomnath Kotur } else if (net_type == RDMA_NETWORK_IPV6 || 419c865f246SSomnath Kotur net_type == RDMA_NETWORK_IB) { 420c865f246SSomnath Kotur *dgid = hdr->ibgrh.dgid; 421c865f246SSomnath Kotur *sgid = hdr->ibgrh.sgid; 422c865f246SSomnath Kotur return 0; 423c865f246SSomnath Kotur } else { 424c865f246SSomnath Kotur return -EINVAL; 425c865f246SSomnath Kotur } 426c865f246SSomnath Kotur } 427c865f246SSomnath Kotur 42873cdaaeeSIra Weiny int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, 42973cdaaeeSIra Weiny const struct ib_wc *wc, const struct ib_grh *grh, 43073cdaaeeSIra Weiny struct ib_ah_attr *ah_attr) 431513789edSHal Rosenstock { 432513789edSHal Rosenstock u32 flow_class; 433513789edSHal Rosenstock u16 gid_index; 434513789edSHal Rosenstock int ret; 435c865f246SSomnath Kotur enum rdma_network_type net_type = RDMA_NETWORK_IB; 436c865f246SSomnath Kotur enum ib_gid_type gid_type = IB_GID_TYPE_IB; 437c865f246SSomnath Kotur union ib_gid dgid; 438c865f246SSomnath Kotur union ib_gid sgid; 439513789edSHal Rosenstock 4404e00d694SSean Hefty memset(ah_attr, 0, sizeof *ah_attr); 441227128fcSMichael Wang if (rdma_cap_eth_ah(device, port_num)) { 442c865f246SSomnath Kotur if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE) 443c865f246SSomnath Kotur net_type = wc->network_hdr_type; 444c865f246SSomnath Kotur else 445c865f246SSomnath Kotur net_type = ib_get_net_type_by_grh(device, port_num, grh); 446c865f246SSomnath Kotur gid_type = ib_network_to_gid_type(net_type); 447c865f246SSomnath Kotur } 448c865f246SSomnath Kotur ret = get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type, 449c865f246SSomnath Kotur &sgid, &dgid); 450c865f246SSomnath Kotur if (ret) 451c865f246SSomnath Kotur return ret; 452c865f246SSomnath Kotur 453c865f246SSomnath Kotur if (rdma_protocol_roce(device, port_num)) { 45420029832SMatan Barak int if_index = 0; 455dbf727deSMatan Barak u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ? 456dbf727deSMatan Barak wc->vlan_id : 0xffff; 45720029832SMatan Barak struct net_device *idev; 45820029832SMatan Barak struct net_device *resolved_dev; 459dbf727deSMatan Barak 460dd5f03beSMatan Barak if (!(wc->wc_flags & IB_WC_GRH)) 461dd5f03beSMatan Barak return -EPROTOTYPE; 462dd5f03beSMatan Barak 46320029832SMatan Barak if (!device->get_netdev) 46420029832SMatan Barak return -EOPNOTSUPP; 46520029832SMatan Barak 46620029832SMatan Barak idev = device->get_netdev(device, port_num); 46720029832SMatan Barak if (!idev) 46820029832SMatan Barak return -ENODEV; 46920029832SMatan Barak 470f7f4b23eSMatan Barak ret = rdma_addr_find_l2_eth_by_grh(&dgid, &sgid, 471dbf727deSMatan Barak ah_attr->dmac, 472dbf727deSMatan Barak wc->wc_flags & IB_WC_WITH_VLAN ? 473dbf727deSMatan Barak NULL : &vlan_id, 47420029832SMatan Barak &if_index); 47520029832SMatan Barak if (ret) { 47620029832SMatan Barak dev_put(idev); 477dd5f03beSMatan Barak return ret; 478dd5f03beSMatan Barak } 479dbf727deSMatan Barak 48020029832SMatan Barak resolved_dev = dev_get_by_index(&init_net, if_index); 48120029832SMatan Barak if (resolved_dev->flags & IFF_LOOPBACK) { 48220029832SMatan Barak dev_put(resolved_dev); 48320029832SMatan Barak resolved_dev = idev; 48420029832SMatan Barak dev_hold(resolved_dev); 48520029832SMatan Barak } 48620029832SMatan Barak rcu_read_lock(); 48720029832SMatan Barak if (resolved_dev != idev && !rdma_is_upper_dev_rcu(idev, 48820029832SMatan Barak resolved_dev)) 48920029832SMatan Barak ret = -EHOSTUNREACH; 49020029832SMatan Barak rcu_read_unlock(); 49120029832SMatan Barak dev_put(idev); 49220029832SMatan Barak dev_put(resolved_dev); 49320029832SMatan Barak if (ret) 49420029832SMatan Barak return ret; 49520029832SMatan Barak 496dbf727deSMatan Barak ret = get_sgid_index_from_eth(device, port_num, vlan_id, 497c865f246SSomnath Kotur &dgid, gid_type, &gid_index); 498dbf727deSMatan Barak if (ret) 499dbf727deSMatan Barak return ret; 500dd5f03beSMatan Barak } 501dd5f03beSMatan Barak 5024e00d694SSean Hefty ah_attr->dlid = wc->slid; 5034e00d694SSean Hefty ah_attr->sl = wc->sl; 5044e00d694SSean Hefty ah_attr->src_path_bits = wc->dlid_path_bits; 5054e00d694SSean Hefty ah_attr->port_num = port_num; 506513789edSHal Rosenstock 507513789edSHal Rosenstock if (wc->wc_flags & IB_WC_GRH) { 5084e00d694SSean Hefty ah_attr->ah_flags = IB_AH_GRH; 509c865f246SSomnath Kotur ah_attr->grh.dgid = sgid; 510513789edSHal Rosenstock 511dbf727deSMatan Barak if (!rdma_cap_eth_ah(device, port_num)) { 512c865f246SSomnath Kotur ret = ib_find_cached_gid_by_port(device, &dgid, 513b39ffa1dSMatan Barak IB_GID_TYPE_IB, 514dbf727deSMatan Barak port_num, NULL, 515dbf727deSMatan Barak &gid_index); 516513789edSHal Rosenstock if (ret) 5174e00d694SSean Hefty return ret; 518dbf727deSMatan Barak } 519513789edSHal Rosenstock 5204e00d694SSean Hefty ah_attr->grh.sgid_index = (u8) gid_index; 521497677abSHal Rosenstock flow_class = be32_to_cpu(grh->version_tclass_flow); 5224e00d694SSean Hefty ah_attr->grh.flow_label = flow_class & 0xFFFFF; 52347645d8dSSean Hefty ah_attr->grh.hop_limit = 0xFF; 5244e00d694SSean Hefty ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF; 525513789edSHal Rosenstock } 5264e00d694SSean Hefty return 0; 5274e00d694SSean Hefty } 5284e00d694SSean Hefty EXPORT_SYMBOL(ib_init_ah_from_wc); 5294e00d694SSean Hefty 53073cdaaeeSIra Weiny struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc, 53173cdaaeeSIra Weiny const struct ib_grh *grh, u8 port_num) 5324e00d694SSean Hefty { 5334e00d694SSean Hefty struct ib_ah_attr ah_attr; 5344e00d694SSean Hefty int ret; 5354e00d694SSean Hefty 5364e00d694SSean Hefty ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr); 5374e00d694SSean Hefty if (ret) 5384e00d694SSean Hefty return ERR_PTR(ret); 539513789edSHal Rosenstock 540513789edSHal Rosenstock return ib_create_ah(pd, &ah_attr); 541513789edSHal Rosenstock } 542513789edSHal Rosenstock EXPORT_SYMBOL(ib_create_ah_from_wc); 543513789edSHal Rosenstock 5441da177e4SLinus Torvalds int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 5451da177e4SLinus Torvalds { 5461da177e4SLinus Torvalds return ah->device->modify_ah ? 5471da177e4SLinus Torvalds ah->device->modify_ah(ah, ah_attr) : 5481da177e4SLinus Torvalds -ENOSYS; 5491da177e4SLinus Torvalds } 5501da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_ah); 5511da177e4SLinus Torvalds 5521da177e4SLinus Torvalds int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 5531da177e4SLinus Torvalds { 5541da177e4SLinus Torvalds return ah->device->query_ah ? 5551da177e4SLinus Torvalds ah->device->query_ah(ah, ah_attr) : 5561da177e4SLinus Torvalds -ENOSYS; 5571da177e4SLinus Torvalds } 5581da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_ah); 5591da177e4SLinus Torvalds 5601da177e4SLinus Torvalds int ib_destroy_ah(struct ib_ah *ah) 5611da177e4SLinus Torvalds { 5621da177e4SLinus Torvalds struct ib_pd *pd; 5631da177e4SLinus Torvalds int ret; 5641da177e4SLinus Torvalds 5651da177e4SLinus Torvalds pd = ah->pd; 5661da177e4SLinus Torvalds ret = ah->device->destroy_ah(ah); 5671da177e4SLinus Torvalds if (!ret) 5681da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 5691da177e4SLinus Torvalds 5701da177e4SLinus Torvalds return ret; 5711da177e4SLinus Torvalds } 5721da177e4SLinus Torvalds EXPORT_SYMBOL(ib_destroy_ah); 5731da177e4SLinus Torvalds 574d41fcc67SRoland Dreier /* Shared receive queues */ 575d41fcc67SRoland Dreier 576d41fcc67SRoland Dreier struct ib_srq *ib_create_srq(struct ib_pd *pd, 577d41fcc67SRoland Dreier struct ib_srq_init_attr *srq_init_attr) 578d41fcc67SRoland Dreier { 579d41fcc67SRoland Dreier struct ib_srq *srq; 580d41fcc67SRoland Dreier 581d41fcc67SRoland Dreier if (!pd->device->create_srq) 582d41fcc67SRoland Dreier return ERR_PTR(-ENOSYS); 583d41fcc67SRoland Dreier 584d41fcc67SRoland Dreier srq = pd->device->create_srq(pd, srq_init_attr, NULL); 585d41fcc67SRoland Dreier 586d41fcc67SRoland Dreier if (!IS_ERR(srq)) { 587d41fcc67SRoland Dreier srq->device = pd->device; 588d41fcc67SRoland Dreier srq->pd = pd; 589d41fcc67SRoland Dreier srq->uobject = NULL; 590d41fcc67SRoland Dreier srq->event_handler = srq_init_attr->event_handler; 591d41fcc67SRoland Dreier srq->srq_context = srq_init_attr->srq_context; 59296104edaSSean Hefty srq->srq_type = srq_init_attr->srq_type; 593418d5130SSean Hefty if (srq->srq_type == IB_SRQT_XRC) { 594418d5130SSean Hefty srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd; 595418d5130SSean Hefty srq->ext.xrc.cq = srq_init_attr->ext.xrc.cq; 596418d5130SSean Hefty atomic_inc(&srq->ext.xrc.xrcd->usecnt); 597418d5130SSean Hefty atomic_inc(&srq->ext.xrc.cq->usecnt); 598418d5130SSean Hefty } 599d41fcc67SRoland Dreier atomic_inc(&pd->usecnt); 600d41fcc67SRoland Dreier atomic_set(&srq->usecnt, 0); 601d41fcc67SRoland Dreier } 602d41fcc67SRoland Dreier 603d41fcc67SRoland Dreier return srq; 604d41fcc67SRoland Dreier } 605d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_create_srq); 606d41fcc67SRoland Dreier 607d41fcc67SRoland Dreier int ib_modify_srq(struct ib_srq *srq, 608d41fcc67SRoland Dreier struct ib_srq_attr *srq_attr, 609d41fcc67SRoland Dreier enum ib_srq_attr_mask srq_attr_mask) 610d41fcc67SRoland Dreier { 6117ce5eacbSDotan Barak return srq->device->modify_srq ? 6127ce5eacbSDotan Barak srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) : 6137ce5eacbSDotan Barak -ENOSYS; 614d41fcc67SRoland Dreier } 615d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_modify_srq); 616d41fcc67SRoland Dreier 617d41fcc67SRoland Dreier int ib_query_srq(struct ib_srq *srq, 618d41fcc67SRoland Dreier struct ib_srq_attr *srq_attr) 619d41fcc67SRoland Dreier { 620d41fcc67SRoland Dreier return srq->device->query_srq ? 621d41fcc67SRoland Dreier srq->device->query_srq(srq, srq_attr) : -ENOSYS; 622d41fcc67SRoland Dreier } 623d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_query_srq); 624d41fcc67SRoland Dreier 625d41fcc67SRoland Dreier int ib_destroy_srq(struct ib_srq *srq) 626d41fcc67SRoland Dreier { 627d41fcc67SRoland Dreier struct ib_pd *pd; 628418d5130SSean Hefty enum ib_srq_type srq_type; 629418d5130SSean Hefty struct ib_xrcd *uninitialized_var(xrcd); 630418d5130SSean Hefty struct ib_cq *uninitialized_var(cq); 631d41fcc67SRoland Dreier int ret; 632d41fcc67SRoland Dreier 633d41fcc67SRoland Dreier if (atomic_read(&srq->usecnt)) 634d41fcc67SRoland Dreier return -EBUSY; 635d41fcc67SRoland Dreier 636d41fcc67SRoland Dreier pd = srq->pd; 637418d5130SSean Hefty srq_type = srq->srq_type; 638418d5130SSean Hefty if (srq_type == IB_SRQT_XRC) { 639418d5130SSean Hefty xrcd = srq->ext.xrc.xrcd; 640418d5130SSean Hefty cq = srq->ext.xrc.cq; 641418d5130SSean Hefty } 642d41fcc67SRoland Dreier 643d41fcc67SRoland Dreier ret = srq->device->destroy_srq(srq); 644418d5130SSean Hefty if (!ret) { 645d41fcc67SRoland Dreier atomic_dec(&pd->usecnt); 646418d5130SSean Hefty if (srq_type == IB_SRQT_XRC) { 647418d5130SSean Hefty atomic_dec(&xrcd->usecnt); 648418d5130SSean Hefty atomic_dec(&cq->usecnt); 649418d5130SSean Hefty } 650418d5130SSean Hefty } 651d41fcc67SRoland Dreier 652d41fcc67SRoland Dreier return ret; 653d41fcc67SRoland Dreier } 654d41fcc67SRoland Dreier EXPORT_SYMBOL(ib_destroy_srq); 655d41fcc67SRoland Dreier 6561da177e4SLinus Torvalds /* Queue pairs */ 6571da177e4SLinus Torvalds 6580e0ec7e0SSean Hefty static void __ib_shared_qp_event_handler(struct ib_event *event, void *context) 6590e0ec7e0SSean Hefty { 6600e0ec7e0SSean Hefty struct ib_qp *qp = context; 66173c40c61SYishai Hadas unsigned long flags; 6620e0ec7e0SSean Hefty 66373c40c61SYishai Hadas spin_lock_irqsave(&qp->device->event_handler_lock, flags); 6640e0ec7e0SSean Hefty list_for_each_entry(event->element.qp, &qp->open_list, open_list) 665eec9e29fSShlomo Pongratz if (event->element.qp->event_handler) 6660e0ec7e0SSean Hefty event->element.qp->event_handler(event, event->element.qp->qp_context); 66773c40c61SYishai Hadas spin_unlock_irqrestore(&qp->device->event_handler_lock, flags); 6680e0ec7e0SSean Hefty } 6690e0ec7e0SSean Hefty 670d3d72d90SSean Hefty static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) 671d3d72d90SSean Hefty { 672d3d72d90SSean Hefty mutex_lock(&xrcd->tgt_qp_mutex); 673d3d72d90SSean Hefty list_add(&qp->xrcd_list, &xrcd->tgt_qp_list); 674d3d72d90SSean Hefty mutex_unlock(&xrcd->tgt_qp_mutex); 675d3d72d90SSean Hefty } 676d3d72d90SSean Hefty 6770e0ec7e0SSean Hefty static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp, 6780e0ec7e0SSean Hefty void (*event_handler)(struct ib_event *, void *), 6790e0ec7e0SSean Hefty void *qp_context) 680d3d72d90SSean Hefty { 6810e0ec7e0SSean Hefty struct ib_qp *qp; 6820e0ec7e0SSean Hefty unsigned long flags; 6830e0ec7e0SSean Hefty 6840e0ec7e0SSean Hefty qp = kzalloc(sizeof *qp, GFP_KERNEL); 6850e0ec7e0SSean Hefty if (!qp) 6860e0ec7e0SSean Hefty return ERR_PTR(-ENOMEM); 6870e0ec7e0SSean Hefty 6880e0ec7e0SSean Hefty qp->real_qp = real_qp; 6890e0ec7e0SSean Hefty atomic_inc(&real_qp->usecnt); 6900e0ec7e0SSean Hefty qp->device = real_qp->device; 6910e0ec7e0SSean Hefty qp->event_handler = event_handler; 6920e0ec7e0SSean Hefty qp->qp_context = qp_context; 6930e0ec7e0SSean Hefty qp->qp_num = real_qp->qp_num; 6940e0ec7e0SSean Hefty qp->qp_type = real_qp->qp_type; 6950e0ec7e0SSean Hefty 6960e0ec7e0SSean Hefty spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); 6970e0ec7e0SSean Hefty list_add(&qp->open_list, &real_qp->open_list); 6980e0ec7e0SSean Hefty spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); 6990e0ec7e0SSean Hefty 7000e0ec7e0SSean Hefty return qp; 701d3d72d90SSean Hefty } 702d3d72d90SSean Hefty 7030e0ec7e0SSean Hefty struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd, 7040e0ec7e0SSean Hefty struct ib_qp_open_attr *qp_open_attr) 7050e0ec7e0SSean Hefty { 7060e0ec7e0SSean Hefty struct ib_qp *qp, *real_qp; 7070e0ec7e0SSean Hefty 7080e0ec7e0SSean Hefty if (qp_open_attr->qp_type != IB_QPT_XRC_TGT) 7090e0ec7e0SSean Hefty return ERR_PTR(-EINVAL); 7100e0ec7e0SSean Hefty 7110e0ec7e0SSean Hefty qp = ERR_PTR(-EINVAL); 7120e0ec7e0SSean Hefty mutex_lock(&xrcd->tgt_qp_mutex); 7130e0ec7e0SSean Hefty list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) { 7140e0ec7e0SSean Hefty if (real_qp->qp_num == qp_open_attr->qp_num) { 7150e0ec7e0SSean Hefty qp = __ib_open_qp(real_qp, qp_open_attr->event_handler, 7160e0ec7e0SSean Hefty qp_open_attr->qp_context); 7170e0ec7e0SSean Hefty break; 7180e0ec7e0SSean Hefty } 7190e0ec7e0SSean Hefty } 7200e0ec7e0SSean Hefty mutex_unlock(&xrcd->tgt_qp_mutex); 7210e0ec7e0SSean Hefty return qp; 7220e0ec7e0SSean Hefty } 7230e0ec7e0SSean Hefty EXPORT_SYMBOL(ib_open_qp); 7240e0ec7e0SSean Hefty 7251da177e4SLinus Torvalds struct ib_qp *ib_create_qp(struct ib_pd *pd, 7261da177e4SLinus Torvalds struct ib_qp_init_attr *qp_init_attr) 7271da177e4SLinus Torvalds { 7280e0ec7e0SSean Hefty struct ib_qp *qp, *real_qp; 729b42b63cfSSean Hefty struct ib_device *device; 7301da177e4SLinus Torvalds 731b42b63cfSSean Hefty device = pd ? pd->device : qp_init_attr->xrcd->device; 732b42b63cfSSean Hefty qp = device->create_qp(pd, qp_init_attr, NULL); 7331da177e4SLinus Torvalds 7341da177e4SLinus Torvalds if (!IS_ERR(qp)) { 735b42b63cfSSean Hefty qp->device = device; 7360e0ec7e0SSean Hefty qp->real_qp = qp; 737b5e81bf5SRoland Dreier qp->uobject = NULL; 7380e0ec7e0SSean Hefty qp->qp_type = qp_init_attr->qp_type; 739b42b63cfSSean Hefty 740e47e321aSBernd Schubert atomic_set(&qp->usecnt, 0); 741b42b63cfSSean Hefty if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) { 7420e0ec7e0SSean Hefty qp->event_handler = __ib_shared_qp_event_handler; 7430e0ec7e0SSean Hefty qp->qp_context = qp; 744b42b63cfSSean Hefty qp->pd = NULL; 745b42b63cfSSean Hefty qp->send_cq = qp->recv_cq = NULL; 746b42b63cfSSean Hefty qp->srq = NULL; 747b42b63cfSSean Hefty qp->xrcd = qp_init_attr->xrcd; 748b42b63cfSSean Hefty atomic_inc(&qp_init_attr->xrcd->usecnt); 7490e0ec7e0SSean Hefty INIT_LIST_HEAD(&qp->open_list); 7500e0ec7e0SSean Hefty 7510e0ec7e0SSean Hefty real_qp = qp; 7520e0ec7e0SSean Hefty qp = __ib_open_qp(real_qp, qp_init_attr->event_handler, 7530e0ec7e0SSean Hefty qp_init_attr->qp_context); 7540e0ec7e0SSean Hefty if (!IS_ERR(qp)) 7550e0ec7e0SSean Hefty __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp); 7560e0ec7e0SSean Hefty else 7570e0ec7e0SSean Hefty real_qp->device->destroy_qp(real_qp); 758b42b63cfSSean Hefty } else { 7591da177e4SLinus Torvalds qp->event_handler = qp_init_attr->event_handler; 7601da177e4SLinus Torvalds qp->qp_context = qp_init_attr->qp_context; 761b42b63cfSSean Hefty if (qp_init_attr->qp_type == IB_QPT_XRC_INI) { 762b42b63cfSSean Hefty qp->recv_cq = NULL; 763b42b63cfSSean Hefty qp->srq = NULL; 764b42b63cfSSean Hefty } else { 765b42b63cfSSean Hefty qp->recv_cq = qp_init_attr->recv_cq; 766b42b63cfSSean Hefty atomic_inc(&qp_init_attr->recv_cq->usecnt); 767b42b63cfSSean Hefty qp->srq = qp_init_attr->srq; 768b42b63cfSSean Hefty if (qp->srq) 769b42b63cfSSean Hefty atomic_inc(&qp_init_attr->srq->usecnt); 770b42b63cfSSean Hefty } 771b42b63cfSSean Hefty 7721da177e4SLinus Torvalds qp->pd = pd; 7731da177e4SLinus Torvalds qp->send_cq = qp_init_attr->send_cq; 774b42b63cfSSean Hefty qp->xrcd = NULL; 775b42b63cfSSean Hefty 7761da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 7771da177e4SLinus Torvalds atomic_inc(&qp_init_attr->send_cq->usecnt); 778b42b63cfSSean Hefty } 7791da177e4SLinus Torvalds } 7801da177e4SLinus Torvalds 7811da177e4SLinus Torvalds return qp; 7821da177e4SLinus Torvalds } 7831da177e4SLinus Torvalds EXPORT_SYMBOL(ib_create_qp); 7841da177e4SLinus Torvalds 7858a51866fSRoland Dreier static const struct { 7868a51866fSRoland Dreier int valid; 787b42b63cfSSean Hefty enum ib_qp_attr_mask req_param[IB_QPT_MAX]; 788b42b63cfSSean Hefty enum ib_qp_attr_mask opt_param[IB_QPT_MAX]; 7898a51866fSRoland Dreier } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { 7908a51866fSRoland Dreier [IB_QPS_RESET] = { 7918a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 7928a51866fSRoland Dreier [IB_QPS_INIT] = { 7938a51866fSRoland Dreier .valid = 1, 7948a51866fSRoland Dreier .req_param = { 7958a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 7968a51866fSRoland Dreier IB_QP_PORT | 7978a51866fSRoland Dreier IB_QP_QKEY), 798c938a616SOr Gerlitz [IB_QPT_RAW_PACKET] = IB_QP_PORT, 7998a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 8008a51866fSRoland Dreier IB_QP_PORT | 8018a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 8028a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 8038a51866fSRoland Dreier IB_QP_PORT | 8048a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 805b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX | 806b42b63cfSSean Hefty IB_QP_PORT | 807b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 808b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX | 809b42b63cfSSean Hefty IB_QP_PORT | 810b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 8118a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 8128a51866fSRoland Dreier IB_QP_QKEY), 8138a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 8148a51866fSRoland Dreier IB_QP_QKEY), 8158a51866fSRoland Dreier } 8168a51866fSRoland Dreier }, 8178a51866fSRoland Dreier }, 8188a51866fSRoland Dreier [IB_QPS_INIT] = { 8198a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 8208a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 8218a51866fSRoland Dreier [IB_QPS_INIT] = { 8228a51866fSRoland Dreier .valid = 1, 8238a51866fSRoland Dreier .opt_param = { 8248a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 8258a51866fSRoland Dreier IB_QP_PORT | 8268a51866fSRoland Dreier IB_QP_QKEY), 8278a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 8288a51866fSRoland Dreier IB_QP_PORT | 8298a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 8308a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 8318a51866fSRoland Dreier IB_QP_PORT | 8328a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 833b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX | 834b42b63cfSSean Hefty IB_QP_PORT | 835b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 836b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX | 837b42b63cfSSean Hefty IB_QP_PORT | 838b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS), 8398a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 8408a51866fSRoland Dreier IB_QP_QKEY), 8418a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 8428a51866fSRoland Dreier IB_QP_QKEY), 8438a51866fSRoland Dreier } 8448a51866fSRoland Dreier }, 8458a51866fSRoland Dreier [IB_QPS_RTR] = { 8468a51866fSRoland Dreier .valid = 1, 8478a51866fSRoland Dreier .req_param = { 8488a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_AV | 8498a51866fSRoland Dreier IB_QP_PATH_MTU | 8508a51866fSRoland Dreier IB_QP_DEST_QPN | 8518a51866fSRoland Dreier IB_QP_RQ_PSN), 8528a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_AV | 8538a51866fSRoland Dreier IB_QP_PATH_MTU | 8548a51866fSRoland Dreier IB_QP_DEST_QPN | 8558a51866fSRoland Dreier IB_QP_RQ_PSN | 8568a51866fSRoland Dreier IB_QP_MAX_DEST_RD_ATOMIC | 8578a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER), 858b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_AV | 859b42b63cfSSean Hefty IB_QP_PATH_MTU | 860b42b63cfSSean Hefty IB_QP_DEST_QPN | 861b42b63cfSSean Hefty IB_QP_RQ_PSN), 862b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_AV | 863b42b63cfSSean Hefty IB_QP_PATH_MTU | 864b42b63cfSSean Hefty IB_QP_DEST_QPN | 865b42b63cfSSean Hefty IB_QP_RQ_PSN | 866b42b63cfSSean Hefty IB_QP_MAX_DEST_RD_ATOMIC | 867b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER), 8688a51866fSRoland Dreier }, 8698a51866fSRoland Dreier .opt_param = { 8708a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 8718a51866fSRoland Dreier IB_QP_QKEY), 8728a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_ALT_PATH | 8738a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 8748a51866fSRoland Dreier IB_QP_PKEY_INDEX), 8758a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_ALT_PATH | 8768a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 8778a51866fSRoland Dreier IB_QP_PKEY_INDEX), 878b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH | 879b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 880b42b63cfSSean Hefty IB_QP_PKEY_INDEX), 881b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH | 882b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 883b42b63cfSSean Hefty IB_QP_PKEY_INDEX), 8848a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 8858a51866fSRoland Dreier IB_QP_QKEY), 8868a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 8878a51866fSRoland Dreier IB_QP_QKEY), 888dd5f03beSMatan Barak }, 889dbf727deSMatan Barak }, 8908a51866fSRoland Dreier }, 8918a51866fSRoland Dreier [IB_QPS_RTR] = { 8928a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 8938a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 8948a51866fSRoland Dreier [IB_QPS_RTS] = { 8958a51866fSRoland Dreier .valid = 1, 8968a51866fSRoland Dreier .req_param = { 8978a51866fSRoland Dreier [IB_QPT_UD] = IB_QP_SQ_PSN, 8988a51866fSRoland Dreier [IB_QPT_UC] = IB_QP_SQ_PSN, 8998a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_TIMEOUT | 9008a51866fSRoland Dreier IB_QP_RETRY_CNT | 9018a51866fSRoland Dreier IB_QP_RNR_RETRY | 9028a51866fSRoland Dreier IB_QP_SQ_PSN | 9038a51866fSRoland Dreier IB_QP_MAX_QP_RD_ATOMIC), 904b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT | 905b42b63cfSSean Hefty IB_QP_RETRY_CNT | 906b42b63cfSSean Hefty IB_QP_RNR_RETRY | 907b42b63cfSSean Hefty IB_QP_SQ_PSN | 908b42b63cfSSean Hefty IB_QP_MAX_QP_RD_ATOMIC), 909b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT | 910b42b63cfSSean Hefty IB_QP_SQ_PSN), 9118a51866fSRoland Dreier [IB_QPT_SMI] = IB_QP_SQ_PSN, 9128a51866fSRoland Dreier [IB_QPT_GSI] = IB_QP_SQ_PSN, 9138a51866fSRoland Dreier }, 9148a51866fSRoland Dreier .opt_param = { 9158a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 9168a51866fSRoland Dreier IB_QP_QKEY), 9178a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_CUR_STATE | 9188a51866fSRoland Dreier IB_QP_ALT_PATH | 9198a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 9208a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 9218a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_CUR_STATE | 9228a51866fSRoland Dreier IB_QP_ALT_PATH | 9238a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 9248a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER | 9258a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 926b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 927b42b63cfSSean Hefty IB_QP_ALT_PATH | 928b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 929b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 930b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 931b42b63cfSSean Hefty IB_QP_ALT_PATH | 932b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 933b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER | 934b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 9358a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 9368a51866fSRoland Dreier IB_QP_QKEY), 9378a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 9388a51866fSRoland Dreier IB_QP_QKEY), 9398a51866fSRoland Dreier } 9408a51866fSRoland Dreier } 9418a51866fSRoland Dreier }, 9428a51866fSRoland Dreier [IB_QPS_RTS] = { 9438a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 9448a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 9458a51866fSRoland Dreier [IB_QPS_RTS] = { 9468a51866fSRoland Dreier .valid = 1, 9478a51866fSRoland Dreier .opt_param = { 9488a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 9498a51866fSRoland Dreier IB_QP_QKEY), 9504546d31dSDotan Barak [IB_QPT_UC] = (IB_QP_CUR_STATE | 9514546d31dSDotan Barak IB_QP_ACCESS_FLAGS | 9528a51866fSRoland Dreier IB_QP_ALT_PATH | 9538a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 9544546d31dSDotan Barak [IB_QPT_RC] = (IB_QP_CUR_STATE | 9554546d31dSDotan Barak IB_QP_ACCESS_FLAGS | 9568a51866fSRoland Dreier IB_QP_ALT_PATH | 9578a51866fSRoland Dreier IB_QP_PATH_MIG_STATE | 9588a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER), 959b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 960b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 961b42b63cfSSean Hefty IB_QP_ALT_PATH | 962b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 963b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 964b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 965b42b63cfSSean Hefty IB_QP_ALT_PATH | 966b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE | 967b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER), 9688a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 9698a51866fSRoland Dreier IB_QP_QKEY), 9708a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 9718a51866fSRoland Dreier IB_QP_QKEY), 9728a51866fSRoland Dreier } 9738a51866fSRoland Dreier }, 9748a51866fSRoland Dreier [IB_QPS_SQD] = { 9758a51866fSRoland Dreier .valid = 1, 9768a51866fSRoland Dreier .opt_param = { 9778a51866fSRoland Dreier [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY, 9788a51866fSRoland Dreier [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 9798a51866fSRoland Dreier [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 980b42b63cfSSean Hefty [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 981b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */ 9828a51866fSRoland Dreier [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 9838a51866fSRoland Dreier [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY 9848a51866fSRoland Dreier } 9858a51866fSRoland Dreier }, 9868a51866fSRoland Dreier }, 9878a51866fSRoland Dreier [IB_QPS_SQD] = { 9888a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 9898a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 9908a51866fSRoland Dreier [IB_QPS_RTS] = { 9918a51866fSRoland Dreier .valid = 1, 9928a51866fSRoland Dreier .opt_param = { 9938a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 9948a51866fSRoland Dreier IB_QP_QKEY), 9958a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_CUR_STATE | 9968a51866fSRoland Dreier IB_QP_ALT_PATH | 9978a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 9988a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 9998a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_CUR_STATE | 10008a51866fSRoland Dreier IB_QP_ALT_PATH | 10018a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 10028a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER | 10038a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 1004b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE | 1005b42b63cfSSean Hefty IB_QP_ALT_PATH | 1006b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 1007b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 1008b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE | 1009b42b63cfSSean Hefty IB_QP_ALT_PATH | 1010b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 1011b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER | 1012b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 10138a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 10148a51866fSRoland Dreier IB_QP_QKEY), 10158a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 10168a51866fSRoland Dreier IB_QP_QKEY), 10178a51866fSRoland Dreier } 10188a51866fSRoland Dreier }, 10198a51866fSRoland Dreier [IB_QPS_SQD] = { 10208a51866fSRoland Dreier .valid = 1, 10218a51866fSRoland Dreier .opt_param = { 10228a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 10238a51866fSRoland Dreier IB_QP_QKEY), 10248a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_AV | 10258a51866fSRoland Dreier IB_QP_ALT_PATH | 10268a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 10278a51866fSRoland Dreier IB_QP_PKEY_INDEX | 10288a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 10298a51866fSRoland Dreier [IB_QPT_RC] = (IB_QP_PORT | 10308a51866fSRoland Dreier IB_QP_AV | 10318a51866fSRoland Dreier IB_QP_TIMEOUT | 10328a51866fSRoland Dreier IB_QP_RETRY_CNT | 10338a51866fSRoland Dreier IB_QP_RNR_RETRY | 10348a51866fSRoland Dreier IB_QP_MAX_QP_RD_ATOMIC | 10358a51866fSRoland Dreier IB_QP_MAX_DEST_RD_ATOMIC | 10368a51866fSRoland Dreier IB_QP_ALT_PATH | 10378a51866fSRoland Dreier IB_QP_ACCESS_FLAGS | 10388a51866fSRoland Dreier IB_QP_PKEY_INDEX | 10398a51866fSRoland Dreier IB_QP_MIN_RNR_TIMER | 10408a51866fSRoland Dreier IB_QP_PATH_MIG_STATE), 1041b42b63cfSSean Hefty [IB_QPT_XRC_INI] = (IB_QP_PORT | 1042b42b63cfSSean Hefty IB_QP_AV | 1043b42b63cfSSean Hefty IB_QP_TIMEOUT | 1044b42b63cfSSean Hefty IB_QP_RETRY_CNT | 1045b42b63cfSSean Hefty IB_QP_RNR_RETRY | 1046b42b63cfSSean Hefty IB_QP_MAX_QP_RD_ATOMIC | 1047b42b63cfSSean Hefty IB_QP_ALT_PATH | 1048b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 1049b42b63cfSSean Hefty IB_QP_PKEY_INDEX | 1050b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 1051b42b63cfSSean Hefty [IB_QPT_XRC_TGT] = (IB_QP_PORT | 1052b42b63cfSSean Hefty IB_QP_AV | 1053b42b63cfSSean Hefty IB_QP_TIMEOUT | 1054b42b63cfSSean Hefty IB_QP_MAX_DEST_RD_ATOMIC | 1055b42b63cfSSean Hefty IB_QP_ALT_PATH | 1056b42b63cfSSean Hefty IB_QP_ACCESS_FLAGS | 1057b42b63cfSSean Hefty IB_QP_PKEY_INDEX | 1058b42b63cfSSean Hefty IB_QP_MIN_RNR_TIMER | 1059b42b63cfSSean Hefty IB_QP_PATH_MIG_STATE), 10608a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 10618a51866fSRoland Dreier IB_QP_QKEY), 10628a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 10638a51866fSRoland Dreier IB_QP_QKEY), 10648a51866fSRoland Dreier } 10658a51866fSRoland Dreier } 10668a51866fSRoland Dreier }, 10678a51866fSRoland Dreier [IB_QPS_SQE] = { 10688a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 10698a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 }, 10708a51866fSRoland Dreier [IB_QPS_RTS] = { 10718a51866fSRoland Dreier .valid = 1, 10728a51866fSRoland Dreier .opt_param = { 10738a51866fSRoland Dreier [IB_QPT_UD] = (IB_QP_CUR_STATE | 10748a51866fSRoland Dreier IB_QP_QKEY), 10758a51866fSRoland Dreier [IB_QPT_UC] = (IB_QP_CUR_STATE | 10768a51866fSRoland Dreier IB_QP_ACCESS_FLAGS), 10778a51866fSRoland Dreier [IB_QPT_SMI] = (IB_QP_CUR_STATE | 10788a51866fSRoland Dreier IB_QP_QKEY), 10798a51866fSRoland Dreier [IB_QPT_GSI] = (IB_QP_CUR_STATE | 10808a51866fSRoland Dreier IB_QP_QKEY), 10818a51866fSRoland Dreier } 10828a51866fSRoland Dreier } 10838a51866fSRoland Dreier }, 10848a51866fSRoland Dreier [IB_QPS_ERR] = { 10858a51866fSRoland Dreier [IB_QPS_RESET] = { .valid = 1 }, 10868a51866fSRoland Dreier [IB_QPS_ERR] = { .valid = 1 } 10878a51866fSRoland Dreier } 10888a51866fSRoland Dreier }; 10898a51866fSRoland Dreier 10908a51866fSRoland Dreier int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, 1091dd5f03beSMatan Barak enum ib_qp_type type, enum ib_qp_attr_mask mask, 1092dd5f03beSMatan Barak enum rdma_link_layer ll) 10938a51866fSRoland Dreier { 10948a51866fSRoland Dreier enum ib_qp_attr_mask req_param, opt_param; 10958a51866fSRoland Dreier 10968a51866fSRoland Dreier if (cur_state < 0 || cur_state > IB_QPS_ERR || 10978a51866fSRoland Dreier next_state < 0 || next_state > IB_QPS_ERR) 10988a51866fSRoland Dreier return 0; 10998a51866fSRoland Dreier 11008a51866fSRoland Dreier if (mask & IB_QP_CUR_STATE && 11018a51866fSRoland Dreier cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && 11028a51866fSRoland Dreier cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) 11038a51866fSRoland Dreier return 0; 11048a51866fSRoland Dreier 11058a51866fSRoland Dreier if (!qp_state_table[cur_state][next_state].valid) 11068a51866fSRoland Dreier return 0; 11078a51866fSRoland Dreier 11088a51866fSRoland Dreier req_param = qp_state_table[cur_state][next_state].req_param[type]; 11098a51866fSRoland Dreier opt_param = qp_state_table[cur_state][next_state].opt_param[type]; 11108a51866fSRoland Dreier 11118a51866fSRoland Dreier if ((mask & req_param) != req_param) 11128a51866fSRoland Dreier return 0; 11138a51866fSRoland Dreier 11148a51866fSRoland Dreier if (mask & ~(req_param | opt_param | IB_QP_STATE)) 11158a51866fSRoland Dreier return 0; 11168a51866fSRoland Dreier 11178a51866fSRoland Dreier return 1; 11188a51866fSRoland Dreier } 11198a51866fSRoland Dreier EXPORT_SYMBOL(ib_modify_qp_is_ok); 11208a51866fSRoland Dreier 1121dbf727deSMatan Barak int ib_resolve_eth_dmac(struct ib_qp *qp, 1122ed4c54e5SOr Gerlitz struct ib_qp_attr *qp_attr, int *qp_attr_mask) 1123ed4c54e5SOr Gerlitz { 1124ed4c54e5SOr Gerlitz int ret = 0; 1125ed4c54e5SOr Gerlitz 1126dbf727deSMatan Barak if (*qp_attr_mask & IB_QP_AV) { 1127dbf727deSMatan Barak if (qp_attr->ah_attr.port_num < rdma_start_port(qp->device) || 1128dbf727deSMatan Barak qp_attr->ah_attr.port_num > rdma_end_port(qp->device)) 1129dbf727deSMatan Barak return -EINVAL; 1130dbf727deSMatan Barak 1131dbf727deSMatan Barak if (!rdma_cap_eth_ah(qp->device, qp_attr->ah_attr.port_num)) 1132dbf727deSMatan Barak return 0; 1133dbf727deSMatan Barak 1134ed4c54e5SOr Gerlitz if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) { 1135dbf727deSMatan Barak rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw, 1136dbf727deSMatan Barak qp_attr->ah_attr.dmac); 1137ed4c54e5SOr Gerlitz } else { 1138dbf727deSMatan Barak union ib_gid sgid; 1139dbf727deSMatan Barak struct ib_gid_attr sgid_attr; 1140dbf727deSMatan Barak int ifindex; 1141dbf727deSMatan Barak 1142dbf727deSMatan Barak ret = ib_query_gid(qp->device, 1143dbf727deSMatan Barak qp_attr->ah_attr.port_num, 1144dbf727deSMatan Barak qp_attr->ah_attr.grh.sgid_index, 1145dbf727deSMatan Barak &sgid, &sgid_attr); 1146dbf727deSMatan Barak 1147dbf727deSMatan Barak if (ret || !sgid_attr.ndev) { 1148dbf727deSMatan Barak if (!ret) 1149dbf727deSMatan Barak ret = -ENXIO; 1150ed4c54e5SOr Gerlitz goto out; 1151ed4c54e5SOr Gerlitz } 1152c865f246SSomnath Kotur if (sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) 1153c865f246SSomnath Kotur /* TODO: get the hoplimit from the inet/inet6 1154c865f246SSomnath Kotur * device 1155c865f246SSomnath Kotur */ 1156c865f246SSomnath Kotur qp_attr->ah_attr.grh.hop_limit = 1157c865f246SSomnath Kotur IPV6_DEFAULT_HOPLIMIT; 1158dbf727deSMatan Barak 1159dbf727deSMatan Barak ifindex = sgid_attr.ndev->ifindex; 1160dbf727deSMatan Barak 1161f7f4b23eSMatan Barak ret = rdma_addr_find_l2_eth_by_grh(&sgid, 1162dbf727deSMatan Barak &qp_attr->ah_attr.grh.dgid, 1163dbf727deSMatan Barak qp_attr->ah_attr.dmac, 116420029832SMatan Barak NULL, &ifindex); 1165dbf727deSMatan Barak 1166dbf727deSMatan Barak dev_put(sgid_attr.ndev); 1167dbf727deSMatan Barak } 1168ed4c54e5SOr Gerlitz } 1169ed4c54e5SOr Gerlitz out: 1170ed4c54e5SOr Gerlitz return ret; 1171ed4c54e5SOr Gerlitz } 1172dbf727deSMatan Barak EXPORT_SYMBOL(ib_resolve_eth_dmac); 1173ed4c54e5SOr Gerlitz 1174ed4c54e5SOr Gerlitz 11751da177e4SLinus Torvalds int ib_modify_qp(struct ib_qp *qp, 11761da177e4SLinus Torvalds struct ib_qp_attr *qp_attr, 11771da177e4SLinus Torvalds int qp_attr_mask) 11781da177e4SLinus Torvalds { 1179ed4c54e5SOr Gerlitz int ret; 1180ed4c54e5SOr Gerlitz 1181dbf727deSMatan Barak ret = ib_resolve_eth_dmac(qp, qp_attr, &qp_attr_mask); 1182ed4c54e5SOr Gerlitz if (ret) 1183ed4c54e5SOr Gerlitz return ret; 1184ed4c54e5SOr Gerlitz 11850e0ec7e0SSean Hefty return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL); 11861da177e4SLinus Torvalds } 11871da177e4SLinus Torvalds EXPORT_SYMBOL(ib_modify_qp); 11881da177e4SLinus Torvalds 11891da177e4SLinus Torvalds int ib_query_qp(struct ib_qp *qp, 11901da177e4SLinus Torvalds struct ib_qp_attr *qp_attr, 11911da177e4SLinus Torvalds int qp_attr_mask, 11921da177e4SLinus Torvalds struct ib_qp_init_attr *qp_init_attr) 11931da177e4SLinus Torvalds { 11941da177e4SLinus Torvalds return qp->device->query_qp ? 11950e0ec7e0SSean Hefty qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) : 11961da177e4SLinus Torvalds -ENOSYS; 11971da177e4SLinus Torvalds } 11981da177e4SLinus Torvalds EXPORT_SYMBOL(ib_query_qp); 11991da177e4SLinus Torvalds 12000e0ec7e0SSean Hefty int ib_close_qp(struct ib_qp *qp) 12010e0ec7e0SSean Hefty { 12020e0ec7e0SSean Hefty struct ib_qp *real_qp; 12030e0ec7e0SSean Hefty unsigned long flags; 12040e0ec7e0SSean Hefty 12050e0ec7e0SSean Hefty real_qp = qp->real_qp; 12060e0ec7e0SSean Hefty if (real_qp == qp) 12070e0ec7e0SSean Hefty return -EINVAL; 12080e0ec7e0SSean Hefty 12090e0ec7e0SSean Hefty spin_lock_irqsave(&real_qp->device->event_handler_lock, flags); 12100e0ec7e0SSean Hefty list_del(&qp->open_list); 12110e0ec7e0SSean Hefty spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags); 12120e0ec7e0SSean Hefty 12130e0ec7e0SSean Hefty atomic_dec(&real_qp->usecnt); 12140e0ec7e0SSean Hefty kfree(qp); 12150e0ec7e0SSean Hefty 12160e0ec7e0SSean Hefty return 0; 12170e0ec7e0SSean Hefty } 12180e0ec7e0SSean Hefty EXPORT_SYMBOL(ib_close_qp); 12190e0ec7e0SSean Hefty 12200e0ec7e0SSean Hefty static int __ib_destroy_shared_qp(struct ib_qp *qp) 12210e0ec7e0SSean Hefty { 12220e0ec7e0SSean Hefty struct ib_xrcd *xrcd; 12230e0ec7e0SSean Hefty struct ib_qp *real_qp; 12240e0ec7e0SSean Hefty int ret; 12250e0ec7e0SSean Hefty 12260e0ec7e0SSean Hefty real_qp = qp->real_qp; 12270e0ec7e0SSean Hefty xrcd = real_qp->xrcd; 12280e0ec7e0SSean Hefty 12290e0ec7e0SSean Hefty mutex_lock(&xrcd->tgt_qp_mutex); 12300e0ec7e0SSean Hefty ib_close_qp(qp); 12310e0ec7e0SSean Hefty if (atomic_read(&real_qp->usecnt) == 0) 12320e0ec7e0SSean Hefty list_del(&real_qp->xrcd_list); 12330e0ec7e0SSean Hefty else 12340e0ec7e0SSean Hefty real_qp = NULL; 12350e0ec7e0SSean Hefty mutex_unlock(&xrcd->tgt_qp_mutex); 12360e0ec7e0SSean Hefty 12370e0ec7e0SSean Hefty if (real_qp) { 12380e0ec7e0SSean Hefty ret = ib_destroy_qp(real_qp); 12390e0ec7e0SSean Hefty if (!ret) 12400e0ec7e0SSean Hefty atomic_dec(&xrcd->usecnt); 12410e0ec7e0SSean Hefty else 12420e0ec7e0SSean Hefty __ib_insert_xrcd_qp(xrcd, real_qp); 12430e0ec7e0SSean Hefty } 12440e0ec7e0SSean Hefty 12450e0ec7e0SSean Hefty return 0; 12460e0ec7e0SSean Hefty } 12470e0ec7e0SSean Hefty 12481da177e4SLinus Torvalds int ib_destroy_qp(struct ib_qp *qp) 12491da177e4SLinus Torvalds { 12501da177e4SLinus Torvalds struct ib_pd *pd; 12511da177e4SLinus Torvalds struct ib_cq *scq, *rcq; 12521da177e4SLinus Torvalds struct ib_srq *srq; 12531da177e4SLinus Torvalds int ret; 12541da177e4SLinus Torvalds 12550e0ec7e0SSean Hefty if (atomic_read(&qp->usecnt)) 12560e0ec7e0SSean Hefty return -EBUSY; 12570e0ec7e0SSean Hefty 12580e0ec7e0SSean Hefty if (qp->real_qp != qp) 12590e0ec7e0SSean Hefty return __ib_destroy_shared_qp(qp); 12600e0ec7e0SSean Hefty 12611da177e4SLinus Torvalds pd = qp->pd; 12621da177e4SLinus Torvalds scq = qp->send_cq; 12631da177e4SLinus Torvalds rcq = qp->recv_cq; 12641da177e4SLinus Torvalds srq = qp->srq; 12651da177e4SLinus Torvalds 12661da177e4SLinus Torvalds ret = qp->device->destroy_qp(qp); 12671da177e4SLinus Torvalds if (!ret) { 1268b42b63cfSSean Hefty if (pd) 12691da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 1270b42b63cfSSean Hefty if (scq) 12711da177e4SLinus Torvalds atomic_dec(&scq->usecnt); 1272b42b63cfSSean Hefty if (rcq) 12731da177e4SLinus Torvalds atomic_dec(&rcq->usecnt); 12741da177e4SLinus Torvalds if (srq) 12751da177e4SLinus Torvalds atomic_dec(&srq->usecnt); 12761da177e4SLinus Torvalds } 12771da177e4SLinus Torvalds 12781da177e4SLinus Torvalds return ret; 12791da177e4SLinus Torvalds } 12801da177e4SLinus Torvalds EXPORT_SYMBOL(ib_destroy_qp); 12811da177e4SLinus Torvalds 12821da177e4SLinus Torvalds /* Completion queues */ 12831da177e4SLinus Torvalds 12841da177e4SLinus Torvalds struct ib_cq *ib_create_cq(struct ib_device *device, 12851da177e4SLinus Torvalds ib_comp_handler comp_handler, 12861da177e4SLinus Torvalds void (*event_handler)(struct ib_event *, void *), 12878e37210bSMatan Barak void *cq_context, 12888e37210bSMatan Barak const struct ib_cq_init_attr *cq_attr) 12891da177e4SLinus Torvalds { 12901da177e4SLinus Torvalds struct ib_cq *cq; 12911da177e4SLinus Torvalds 12928e37210bSMatan Barak cq = device->create_cq(device, cq_attr, NULL, NULL); 12931da177e4SLinus Torvalds 12941da177e4SLinus Torvalds if (!IS_ERR(cq)) { 12951da177e4SLinus Torvalds cq->device = device; 1296b5e81bf5SRoland Dreier cq->uobject = NULL; 12971da177e4SLinus Torvalds cq->comp_handler = comp_handler; 12981da177e4SLinus Torvalds cq->event_handler = event_handler; 12991da177e4SLinus Torvalds cq->cq_context = cq_context; 13001da177e4SLinus Torvalds atomic_set(&cq->usecnt, 0); 13011da177e4SLinus Torvalds } 13021da177e4SLinus Torvalds 13031da177e4SLinus Torvalds return cq; 13041da177e4SLinus Torvalds } 13051da177e4SLinus Torvalds EXPORT_SYMBOL(ib_create_cq); 13061da177e4SLinus Torvalds 13072dd57162SEli Cohen int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 13082dd57162SEli Cohen { 13092dd57162SEli Cohen return cq->device->modify_cq ? 13102dd57162SEli Cohen cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS; 13112dd57162SEli Cohen } 13122dd57162SEli Cohen EXPORT_SYMBOL(ib_modify_cq); 13132dd57162SEli Cohen 13141da177e4SLinus Torvalds int ib_destroy_cq(struct ib_cq *cq) 13151da177e4SLinus Torvalds { 13161da177e4SLinus Torvalds if (atomic_read(&cq->usecnt)) 13171da177e4SLinus Torvalds return -EBUSY; 13181da177e4SLinus Torvalds 13191da177e4SLinus Torvalds return cq->device->destroy_cq(cq); 13201da177e4SLinus Torvalds } 13211da177e4SLinus Torvalds EXPORT_SYMBOL(ib_destroy_cq); 13221da177e4SLinus Torvalds 1323a74cd4afSRoland Dreier int ib_resize_cq(struct ib_cq *cq, int cqe) 13241da177e4SLinus Torvalds { 132540de2e54SRoland Dreier return cq->device->resize_cq ? 132633b9b3eeSRoland Dreier cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS; 13271da177e4SLinus Torvalds } 13281da177e4SLinus Torvalds EXPORT_SYMBOL(ib_resize_cq); 13291da177e4SLinus Torvalds 13301da177e4SLinus Torvalds /* Memory regions */ 13311da177e4SLinus Torvalds 13321da177e4SLinus Torvalds struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags) 13331da177e4SLinus Torvalds { 13341da177e4SLinus Torvalds struct ib_mr *mr; 13351c636f80SEli Cohen int err; 13361c636f80SEli Cohen 13371c636f80SEli Cohen err = ib_check_mr_access(mr_access_flags); 13381c636f80SEli Cohen if (err) 13391c636f80SEli Cohen return ERR_PTR(err); 13401da177e4SLinus Torvalds 13411da177e4SLinus Torvalds mr = pd->device->get_dma_mr(pd, mr_access_flags); 13421da177e4SLinus Torvalds 13431da177e4SLinus Torvalds if (!IS_ERR(mr)) { 13441da177e4SLinus Torvalds mr->device = pd->device; 13451da177e4SLinus Torvalds mr->pd = pd; 1346b5e81bf5SRoland Dreier mr->uobject = NULL; 13471da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 13481da177e4SLinus Torvalds } 13491da177e4SLinus Torvalds 13501da177e4SLinus Torvalds return mr; 13511da177e4SLinus Torvalds } 13521da177e4SLinus Torvalds EXPORT_SYMBOL(ib_get_dma_mr); 13531da177e4SLinus Torvalds 13541da177e4SLinus Torvalds int ib_dereg_mr(struct ib_mr *mr) 13551da177e4SLinus Torvalds { 1356ab67ed8dSChristoph Hellwig struct ib_pd *pd = mr->pd; 13571da177e4SLinus Torvalds int ret; 13581da177e4SLinus Torvalds 13591da177e4SLinus Torvalds ret = mr->device->dereg_mr(mr); 13601da177e4SLinus Torvalds if (!ret) 13611da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 13621da177e4SLinus Torvalds 13631da177e4SLinus Torvalds return ret; 13641da177e4SLinus Torvalds } 13651da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dereg_mr); 13661da177e4SLinus Torvalds 13679bee178bSSagi Grimberg /** 13689bee178bSSagi Grimberg * ib_alloc_mr() - Allocates a memory region 13699bee178bSSagi Grimberg * @pd: protection domain associated with the region 13709bee178bSSagi Grimberg * @mr_type: memory region type 13719bee178bSSagi Grimberg * @max_num_sg: maximum sg entries available for registration. 13729bee178bSSagi Grimberg * 13739bee178bSSagi Grimberg * Notes: 13749bee178bSSagi Grimberg * Memory registeration page/sg lists must not exceed max_num_sg. 13759bee178bSSagi Grimberg * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed 13769bee178bSSagi Grimberg * max_num_sg * used_page_size. 13779bee178bSSagi Grimberg * 13789bee178bSSagi Grimberg */ 13799bee178bSSagi Grimberg struct ib_mr *ib_alloc_mr(struct ib_pd *pd, 13809bee178bSSagi Grimberg enum ib_mr_type mr_type, 13819bee178bSSagi Grimberg u32 max_num_sg) 138217cd3a2dSSagi Grimberg { 138317cd3a2dSSagi Grimberg struct ib_mr *mr; 138417cd3a2dSSagi Grimberg 1385d9f272c5SSagi Grimberg if (!pd->device->alloc_mr) 138617cd3a2dSSagi Grimberg return ERR_PTR(-ENOSYS); 138717cd3a2dSSagi Grimberg 1388d9f272c5SSagi Grimberg mr = pd->device->alloc_mr(pd, mr_type, max_num_sg); 138917cd3a2dSSagi Grimberg if (!IS_ERR(mr)) { 139017cd3a2dSSagi Grimberg mr->device = pd->device; 139117cd3a2dSSagi Grimberg mr->pd = pd; 139217cd3a2dSSagi Grimberg mr->uobject = NULL; 139317cd3a2dSSagi Grimberg atomic_inc(&pd->usecnt); 139417cd3a2dSSagi Grimberg } 139517cd3a2dSSagi Grimberg 139617cd3a2dSSagi Grimberg return mr; 139717cd3a2dSSagi Grimberg } 13989bee178bSSagi Grimberg EXPORT_SYMBOL(ib_alloc_mr); 139900f7ec36SSteve Wise 14001da177e4SLinus Torvalds /* "Fast" memory regions */ 14011da177e4SLinus Torvalds 14021da177e4SLinus Torvalds struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd, 14031da177e4SLinus Torvalds int mr_access_flags, 14041da177e4SLinus Torvalds struct ib_fmr_attr *fmr_attr) 14051da177e4SLinus Torvalds { 14061da177e4SLinus Torvalds struct ib_fmr *fmr; 14071da177e4SLinus Torvalds 14081da177e4SLinus Torvalds if (!pd->device->alloc_fmr) 14091da177e4SLinus Torvalds return ERR_PTR(-ENOSYS); 14101da177e4SLinus Torvalds 14111da177e4SLinus Torvalds fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr); 14121da177e4SLinus Torvalds if (!IS_ERR(fmr)) { 14131da177e4SLinus Torvalds fmr->device = pd->device; 14141da177e4SLinus Torvalds fmr->pd = pd; 14151da177e4SLinus Torvalds atomic_inc(&pd->usecnt); 14161da177e4SLinus Torvalds } 14171da177e4SLinus Torvalds 14181da177e4SLinus Torvalds return fmr; 14191da177e4SLinus Torvalds } 14201da177e4SLinus Torvalds EXPORT_SYMBOL(ib_alloc_fmr); 14211da177e4SLinus Torvalds 14221da177e4SLinus Torvalds int ib_unmap_fmr(struct list_head *fmr_list) 14231da177e4SLinus Torvalds { 14241da177e4SLinus Torvalds struct ib_fmr *fmr; 14251da177e4SLinus Torvalds 14261da177e4SLinus Torvalds if (list_empty(fmr_list)) 14271da177e4SLinus Torvalds return 0; 14281da177e4SLinus Torvalds 14291da177e4SLinus Torvalds fmr = list_entry(fmr_list->next, struct ib_fmr, list); 14301da177e4SLinus Torvalds return fmr->device->unmap_fmr(fmr_list); 14311da177e4SLinus Torvalds } 14321da177e4SLinus Torvalds EXPORT_SYMBOL(ib_unmap_fmr); 14331da177e4SLinus Torvalds 14341da177e4SLinus Torvalds int ib_dealloc_fmr(struct ib_fmr *fmr) 14351da177e4SLinus Torvalds { 14361da177e4SLinus Torvalds struct ib_pd *pd; 14371da177e4SLinus Torvalds int ret; 14381da177e4SLinus Torvalds 14391da177e4SLinus Torvalds pd = fmr->pd; 14401da177e4SLinus Torvalds ret = fmr->device->dealloc_fmr(fmr); 14411da177e4SLinus Torvalds if (!ret) 14421da177e4SLinus Torvalds atomic_dec(&pd->usecnt); 14431da177e4SLinus Torvalds 14441da177e4SLinus Torvalds return ret; 14451da177e4SLinus Torvalds } 14461da177e4SLinus Torvalds EXPORT_SYMBOL(ib_dealloc_fmr); 14471da177e4SLinus Torvalds 14481da177e4SLinus Torvalds /* Multicast groups */ 14491da177e4SLinus Torvalds 14501da177e4SLinus Torvalds int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 14511da177e4SLinus Torvalds { 1452c3bccbfbSOr Gerlitz int ret; 1453c3bccbfbSOr Gerlitz 14540c33aeedSJack Morgenstein if (!qp->device->attach_mcast) 14550c33aeedSJack Morgenstein return -ENOSYS; 14560c33aeedSJack Morgenstein if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 14570c33aeedSJack Morgenstein return -EINVAL; 14580c33aeedSJack Morgenstein 1459c3bccbfbSOr Gerlitz ret = qp->device->attach_mcast(qp, gid, lid); 1460c3bccbfbSOr Gerlitz if (!ret) 1461c3bccbfbSOr Gerlitz atomic_inc(&qp->usecnt); 1462c3bccbfbSOr Gerlitz return ret; 14631da177e4SLinus Torvalds } 14641da177e4SLinus Torvalds EXPORT_SYMBOL(ib_attach_mcast); 14651da177e4SLinus Torvalds 14661da177e4SLinus Torvalds int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 14671da177e4SLinus Torvalds { 1468c3bccbfbSOr Gerlitz int ret; 1469c3bccbfbSOr Gerlitz 14700c33aeedSJack Morgenstein if (!qp->device->detach_mcast) 14710c33aeedSJack Morgenstein return -ENOSYS; 14720c33aeedSJack Morgenstein if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 14730c33aeedSJack Morgenstein return -EINVAL; 14740c33aeedSJack Morgenstein 1475c3bccbfbSOr Gerlitz ret = qp->device->detach_mcast(qp, gid, lid); 1476c3bccbfbSOr Gerlitz if (!ret) 1477c3bccbfbSOr Gerlitz atomic_dec(&qp->usecnt); 1478c3bccbfbSOr Gerlitz return ret; 14791da177e4SLinus Torvalds } 14801da177e4SLinus Torvalds EXPORT_SYMBOL(ib_detach_mcast); 148159991f94SSean Hefty 148259991f94SSean Hefty struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device) 148359991f94SSean Hefty { 148459991f94SSean Hefty struct ib_xrcd *xrcd; 148559991f94SSean Hefty 148659991f94SSean Hefty if (!device->alloc_xrcd) 148759991f94SSean Hefty return ERR_PTR(-ENOSYS); 148859991f94SSean Hefty 148959991f94SSean Hefty xrcd = device->alloc_xrcd(device, NULL, NULL); 149059991f94SSean Hefty if (!IS_ERR(xrcd)) { 149159991f94SSean Hefty xrcd->device = device; 149253d0bd1eSSean Hefty xrcd->inode = NULL; 149359991f94SSean Hefty atomic_set(&xrcd->usecnt, 0); 1494d3d72d90SSean Hefty mutex_init(&xrcd->tgt_qp_mutex); 1495d3d72d90SSean Hefty INIT_LIST_HEAD(&xrcd->tgt_qp_list); 149659991f94SSean Hefty } 149759991f94SSean Hefty 149859991f94SSean Hefty return xrcd; 149959991f94SSean Hefty } 150059991f94SSean Hefty EXPORT_SYMBOL(ib_alloc_xrcd); 150159991f94SSean Hefty 150259991f94SSean Hefty int ib_dealloc_xrcd(struct ib_xrcd *xrcd) 150359991f94SSean Hefty { 1504d3d72d90SSean Hefty struct ib_qp *qp; 1505d3d72d90SSean Hefty int ret; 1506d3d72d90SSean Hefty 150759991f94SSean Hefty if (atomic_read(&xrcd->usecnt)) 150859991f94SSean Hefty return -EBUSY; 150959991f94SSean Hefty 1510d3d72d90SSean Hefty while (!list_empty(&xrcd->tgt_qp_list)) { 1511d3d72d90SSean Hefty qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list); 1512d3d72d90SSean Hefty ret = ib_destroy_qp(qp); 1513d3d72d90SSean Hefty if (ret) 1514d3d72d90SSean Hefty return ret; 1515d3d72d90SSean Hefty } 1516d3d72d90SSean Hefty 151759991f94SSean Hefty return xrcd->device->dealloc_xrcd(xrcd); 151859991f94SSean Hefty } 151959991f94SSean Hefty EXPORT_SYMBOL(ib_dealloc_xrcd); 1520319a441dSHadar Hen Zion 1521319a441dSHadar Hen Zion struct ib_flow *ib_create_flow(struct ib_qp *qp, 1522319a441dSHadar Hen Zion struct ib_flow_attr *flow_attr, 1523319a441dSHadar Hen Zion int domain) 1524319a441dSHadar Hen Zion { 1525319a441dSHadar Hen Zion struct ib_flow *flow_id; 1526319a441dSHadar Hen Zion if (!qp->device->create_flow) 1527319a441dSHadar Hen Zion return ERR_PTR(-ENOSYS); 1528319a441dSHadar Hen Zion 1529319a441dSHadar Hen Zion flow_id = qp->device->create_flow(qp, flow_attr, domain); 1530319a441dSHadar Hen Zion if (!IS_ERR(flow_id)) 1531319a441dSHadar Hen Zion atomic_inc(&qp->usecnt); 1532319a441dSHadar Hen Zion return flow_id; 1533319a441dSHadar Hen Zion } 1534319a441dSHadar Hen Zion EXPORT_SYMBOL(ib_create_flow); 1535319a441dSHadar Hen Zion 1536319a441dSHadar Hen Zion int ib_destroy_flow(struct ib_flow *flow_id) 1537319a441dSHadar Hen Zion { 1538319a441dSHadar Hen Zion int err; 1539319a441dSHadar Hen Zion struct ib_qp *qp = flow_id->qp; 1540319a441dSHadar Hen Zion 1541319a441dSHadar Hen Zion err = qp->device->destroy_flow(flow_id); 1542319a441dSHadar Hen Zion if (!err) 1543319a441dSHadar Hen Zion atomic_dec(&qp->usecnt); 1544319a441dSHadar Hen Zion return err; 1545319a441dSHadar Hen Zion } 1546319a441dSHadar Hen Zion EXPORT_SYMBOL(ib_destroy_flow); 15471b01d335SSagi Grimberg 15481b01d335SSagi Grimberg int ib_check_mr_status(struct ib_mr *mr, u32 check_mask, 15491b01d335SSagi Grimberg struct ib_mr_status *mr_status) 15501b01d335SSagi Grimberg { 15511b01d335SSagi Grimberg return mr->device->check_mr_status ? 15521b01d335SSagi Grimberg mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS; 15531b01d335SSagi Grimberg } 15541b01d335SSagi Grimberg EXPORT_SYMBOL(ib_check_mr_status); 15554c67e2bfSSagi Grimberg 15564c67e2bfSSagi Grimberg /** 15574c67e2bfSSagi Grimberg * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list 15584c67e2bfSSagi Grimberg * and set it the memory region. 15594c67e2bfSSagi Grimberg * @mr: memory region 15604c67e2bfSSagi Grimberg * @sg: dma mapped scatterlist 15614c67e2bfSSagi Grimberg * @sg_nents: number of entries in sg 15624c67e2bfSSagi Grimberg * @page_size: page vector desired page size 15634c67e2bfSSagi Grimberg * 15644c67e2bfSSagi Grimberg * Constraints: 15654c67e2bfSSagi Grimberg * - The first sg element is allowed to have an offset. 15664c67e2bfSSagi Grimberg * - Each sg element must be aligned to page_size (or physically 15674c67e2bfSSagi Grimberg * contiguous to the previous element). In case an sg element has a 15684c67e2bfSSagi Grimberg * non contiguous offset, the mapping prefix will not include it. 15694c67e2bfSSagi Grimberg * - The last sg element is allowed to have length less than page_size. 15704c67e2bfSSagi Grimberg * - If sg_nents total byte length exceeds the mr max_num_sge * page_size 15714c67e2bfSSagi Grimberg * then only max_num_sg entries will be mapped. 15724c67e2bfSSagi Grimberg * 15734c67e2bfSSagi Grimberg * Returns the number of sg elements that were mapped to the memory region. 15744c67e2bfSSagi Grimberg * 15754c67e2bfSSagi Grimberg * After this completes successfully, the memory region 15764c67e2bfSSagi Grimberg * is ready for registration. 15774c67e2bfSSagi Grimberg */ 15784c67e2bfSSagi Grimberg int ib_map_mr_sg(struct ib_mr *mr, 15794c67e2bfSSagi Grimberg struct scatterlist *sg, 15804c67e2bfSSagi Grimberg int sg_nents, 15814c67e2bfSSagi Grimberg unsigned int page_size) 15824c67e2bfSSagi Grimberg { 15834c67e2bfSSagi Grimberg if (unlikely(!mr->device->map_mr_sg)) 15844c67e2bfSSagi Grimberg return -ENOSYS; 15854c67e2bfSSagi Grimberg 15864c67e2bfSSagi Grimberg mr->page_size = page_size; 15874c67e2bfSSagi Grimberg 15884c67e2bfSSagi Grimberg return mr->device->map_mr_sg(mr, sg, sg_nents); 15894c67e2bfSSagi Grimberg } 15904c67e2bfSSagi Grimberg EXPORT_SYMBOL(ib_map_mr_sg); 15914c67e2bfSSagi Grimberg 15924c67e2bfSSagi Grimberg /** 15934c67e2bfSSagi Grimberg * ib_sg_to_pages() - Convert the largest prefix of a sg list 15944c67e2bfSSagi Grimberg * to a page vector 15954c67e2bfSSagi Grimberg * @mr: memory region 15964c67e2bfSSagi Grimberg * @sgl: dma mapped scatterlist 15974c67e2bfSSagi Grimberg * @sg_nents: number of entries in sg 15984c67e2bfSSagi Grimberg * @set_page: driver page assignment function pointer 15994c67e2bfSSagi Grimberg * 16008f5ba10eSBart Van Assche * Core service helper for drivers to convert the largest 16014c67e2bfSSagi Grimberg * prefix of given sg list to a page vector. The sg list 16024c67e2bfSSagi Grimberg * prefix converted is the prefix that meet the requirements 16034c67e2bfSSagi Grimberg * of ib_map_mr_sg. 16044c67e2bfSSagi Grimberg * 16054c67e2bfSSagi Grimberg * Returns the number of sg elements that were assigned to 16064c67e2bfSSagi Grimberg * a page vector. 16074c67e2bfSSagi Grimberg */ 16084c67e2bfSSagi Grimberg int ib_sg_to_pages(struct ib_mr *mr, 16094c67e2bfSSagi Grimberg struct scatterlist *sgl, 16104c67e2bfSSagi Grimberg int sg_nents, 16114c67e2bfSSagi Grimberg int (*set_page)(struct ib_mr *, u64)) 16124c67e2bfSSagi Grimberg { 16134c67e2bfSSagi Grimberg struct scatterlist *sg; 1614b6aeb980SBart Van Assche u64 last_end_dma_addr = 0; 16154c67e2bfSSagi Grimberg unsigned int last_page_off = 0; 16164c67e2bfSSagi Grimberg u64 page_mask = ~((u64)mr->page_size - 1); 16178f5ba10eSBart Van Assche int i, ret; 16184c67e2bfSSagi Grimberg 16194c67e2bfSSagi Grimberg mr->iova = sg_dma_address(&sgl[0]); 16204c67e2bfSSagi Grimberg mr->length = 0; 16214c67e2bfSSagi Grimberg 16224c67e2bfSSagi Grimberg for_each_sg(sgl, sg, sg_nents, i) { 16234c67e2bfSSagi Grimberg u64 dma_addr = sg_dma_address(sg); 16244c67e2bfSSagi Grimberg unsigned int dma_len = sg_dma_len(sg); 16254c67e2bfSSagi Grimberg u64 end_dma_addr = dma_addr + dma_len; 16264c67e2bfSSagi Grimberg u64 page_addr = dma_addr & page_mask; 16274c67e2bfSSagi Grimberg 16288f5ba10eSBart Van Assche /* 16298f5ba10eSBart Van Assche * For the second and later elements, check whether either the 16308f5ba10eSBart Van Assche * end of element i-1 or the start of element i is not aligned 16318f5ba10eSBart Van Assche * on a page boundary. 16328f5ba10eSBart Van Assche */ 16338f5ba10eSBart Van Assche if (i && (last_page_off != 0 || page_addr != dma_addr)) { 16348f5ba10eSBart Van Assche /* Stop mapping if there is a gap. */ 16358f5ba10eSBart Van Assche if (last_end_dma_addr != dma_addr) 16368f5ba10eSBart Van Assche break; 16374c67e2bfSSagi Grimberg 16388f5ba10eSBart Van Assche /* 16398f5ba10eSBart Van Assche * Coalesce this element with the last. If it is small 16408f5ba10eSBart Van Assche * enough just update mr->length. Otherwise start 16418f5ba10eSBart Van Assche * mapping from the next page. 16428f5ba10eSBart Van Assche */ 16438f5ba10eSBart Van Assche goto next_page; 16444c67e2bfSSagi Grimberg } 16454c67e2bfSSagi Grimberg 16464c67e2bfSSagi Grimberg do { 16478f5ba10eSBart Van Assche ret = set_page(mr, page_addr); 16488f5ba10eSBart Van Assche if (unlikely(ret < 0)) 16498f5ba10eSBart Van Assche return i ? : ret; 16508f5ba10eSBart Van Assche next_page: 16514c67e2bfSSagi Grimberg page_addr += mr->page_size; 16524c67e2bfSSagi Grimberg } while (page_addr < end_dma_addr); 16534c67e2bfSSagi Grimberg 16544c67e2bfSSagi Grimberg mr->length += dma_len; 16554c67e2bfSSagi Grimberg last_end_dma_addr = end_dma_addr; 16564c67e2bfSSagi Grimberg last_page_off = end_dma_addr & ~page_mask; 16574c67e2bfSSagi Grimberg } 16584c67e2bfSSagi Grimberg 16594c67e2bfSSagi Grimberg return i; 16604c67e2bfSSagi Grimberg } 16614c67e2bfSSagi Grimberg EXPORT_SYMBOL(ib_sg_to_pages); 1662