11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Communications. All rights reserved. 32a1d9b7fSRoland Dreier * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 42a1d9b7fSRoland Dreier * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This software is available to you under a choice of one of two 71da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 81da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 91da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 101da177e4SLinus Torvalds * OpenIB.org BSD license below: 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 131da177e4SLinus Torvalds * without modification, are permitted provided that the following 141da177e4SLinus Torvalds * conditions are met: 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * - Redistributions of source code must retain the above 171da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 181da177e4SLinus Torvalds * disclaimer. 191da177e4SLinus Torvalds * 201da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 211da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 221da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 231da177e4SLinus Torvalds * provided with the distribution. 241da177e4SLinus Torvalds * 251da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 261da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 271da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 281da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 291da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 301da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 311da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 321da177e4SLinus Torvalds * SOFTWARE. 331da177e4SLinus Torvalds * 341da177e4SLinus Torvalds * $Id: mthca_mad.c 1349 2004-12-16 21:09:43Z roland $ 351da177e4SLinus Torvalds */ 361da177e4SLinus Torvalds 374e57b681STim Schmielau #include <linux/string.h> 384e57b681STim Schmielau #include <linux/slab.h> 394e57b681STim Schmielau 40a4d61e84SRoland Dreier #include <rdma/ib_verbs.h> 41a4d61e84SRoland Dreier #include <rdma/ib_mad.h> 42a4d61e84SRoland Dreier #include <rdma/ib_smi.h> 431da177e4SLinus Torvalds 441da177e4SLinus Torvalds #include "mthca_dev.h" 451da177e4SLinus Torvalds #include "mthca_cmd.h" 461da177e4SLinus Torvalds 471da177e4SLinus Torvalds enum { 481da177e4SLinus Torvalds MTHCA_VENDOR_CLASS1 = 0x9, 491da177e4SLinus Torvalds MTHCA_VENDOR_CLASS2 = 0xa 501da177e4SLinus Torvalds }; 511da177e4SLinus Torvalds 52bf6a9e31SJack Morgenstein int mthca_update_rate(struct mthca_dev *dev, u8 port_num) 53bf6a9e31SJack Morgenstein { 54bf6a9e31SJack Morgenstein struct ib_port_attr *tprops = NULL; 55bf6a9e31SJack Morgenstein int ret; 56bf6a9e31SJack Morgenstein 57bf6a9e31SJack Morgenstein tprops = kmalloc(sizeof *tprops, GFP_KERNEL); 58bf6a9e31SJack Morgenstein if (!tprops) 59bf6a9e31SJack Morgenstein return -ENOMEM; 60bf6a9e31SJack Morgenstein 61bf6a9e31SJack Morgenstein ret = ib_query_port(&dev->ib_dev, port_num, tprops); 62bf6a9e31SJack Morgenstein if (ret) { 63bf6a9e31SJack Morgenstein printk(KERN_WARNING "ib_query_port failed (%d) for %s port %d\n", 64bf6a9e31SJack Morgenstein ret, dev->ib_dev.name, port_num); 65bf6a9e31SJack Morgenstein goto out; 66bf6a9e31SJack Morgenstein } 67bf6a9e31SJack Morgenstein 68bf6a9e31SJack Morgenstein dev->rate[port_num - 1] = tprops->active_speed * 69bf6a9e31SJack Morgenstein ib_width_enum_to_int(tprops->active_width); 70bf6a9e31SJack Morgenstein 71bf6a9e31SJack Morgenstein out: 72bf6a9e31SJack Morgenstein kfree(tprops); 73bf6a9e31SJack Morgenstein return ret; 74bf6a9e31SJack Morgenstein } 75bf6a9e31SJack Morgenstein 761da177e4SLinus Torvalds static void update_sm_ah(struct mthca_dev *dev, 771da177e4SLinus Torvalds u8 port_num, u16 lid, u8 sl) 781da177e4SLinus Torvalds { 791da177e4SLinus Torvalds struct ib_ah *new_ah; 801da177e4SLinus Torvalds struct ib_ah_attr ah_attr; 811da177e4SLinus Torvalds unsigned long flags; 821da177e4SLinus Torvalds 831da177e4SLinus Torvalds if (!dev->send_agent[port_num - 1][0]) 841da177e4SLinus Torvalds return; 851da177e4SLinus Torvalds 861da177e4SLinus Torvalds memset(&ah_attr, 0, sizeof ah_attr); 871da177e4SLinus Torvalds ah_attr.dlid = lid; 881da177e4SLinus Torvalds ah_attr.sl = sl; 891da177e4SLinus Torvalds ah_attr.port_num = port_num; 901da177e4SLinus Torvalds 911da177e4SLinus Torvalds new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, 921da177e4SLinus Torvalds &ah_attr); 931da177e4SLinus Torvalds if (IS_ERR(new_ah)) 941da177e4SLinus Torvalds return; 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds spin_lock_irqsave(&dev->sm_lock, flags); 971da177e4SLinus Torvalds if (dev->sm_ah[port_num - 1]) 981da177e4SLinus Torvalds ib_destroy_ah(dev->sm_ah[port_num - 1]); 991da177e4SLinus Torvalds dev->sm_ah[port_num - 1] = new_ah; 1001da177e4SLinus Torvalds spin_unlock_irqrestore(&dev->sm_lock, flags); 1011da177e4SLinus Torvalds } 1021da177e4SLinus Torvalds 1031da177e4SLinus Torvalds /* 1041da177e4SLinus Torvalds * Snoop SM MADs for port info and P_Key table sets, so we can 1051da177e4SLinus Torvalds * synthesize LID change and P_Key change events. 1061da177e4SLinus Torvalds */ 1071da177e4SLinus Torvalds static void smp_snoop(struct ib_device *ibdev, 1081da177e4SLinus Torvalds u8 port_num, 1091da177e4SLinus Torvalds struct ib_mad *mad) 1101da177e4SLinus Torvalds { 1111da177e4SLinus Torvalds struct ib_event event; 1121da177e4SLinus Torvalds 1131da177e4SLinus Torvalds if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 1141da177e4SLinus Torvalds mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && 1151da177e4SLinus Torvalds mad->mad_hdr.method == IB_MGMT_METHOD_SET) { 1161da177e4SLinus Torvalds if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO) { 117bf6a9e31SJack Morgenstein mthca_update_rate(to_mdev(ibdev), port_num); 1181da177e4SLinus Torvalds update_sm_ah(to_mdev(ibdev), port_num, 1191da177e4SLinus Torvalds be16_to_cpup((__be16 *) (mad->data + 58)), 1201da177e4SLinus Torvalds (*(u8 *) (mad->data + 76)) & 0xf); 1211da177e4SLinus Torvalds 1221da177e4SLinus Torvalds event.device = ibdev; 1231da177e4SLinus Torvalds event.event = IB_EVENT_LID_CHANGE; 1241da177e4SLinus Torvalds event.element.port_num = port_num; 1251da177e4SLinus Torvalds ib_dispatch_event(&event); 1261da177e4SLinus Torvalds } 1271da177e4SLinus Torvalds 1281da177e4SLinus Torvalds if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PKEY_TABLE) { 1291da177e4SLinus Torvalds event.device = ibdev; 1301da177e4SLinus Torvalds event.event = IB_EVENT_PKEY_CHANGE; 1311da177e4SLinus Torvalds event.element.port_num = port_num; 1321da177e4SLinus Torvalds ib_dispatch_event(&event); 1331da177e4SLinus Torvalds } 1341da177e4SLinus Torvalds } 1351da177e4SLinus Torvalds } 1361da177e4SLinus Torvalds 1376dfc3901SRoland Dreier static void node_desc_override(struct ib_device *dev, 1386dfc3901SRoland Dreier struct ib_mad *mad) 1396dfc3901SRoland Dreier { 1406dfc3901SRoland Dreier if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 1416dfc3901SRoland Dreier mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && 1426dfc3901SRoland Dreier mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP && 1436dfc3901SRoland Dreier mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) { 1446dfc3901SRoland Dreier mutex_lock(&to_mdev(dev)->cap_mask_mutex); 1456dfc3901SRoland Dreier memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64); 1466dfc3901SRoland Dreier mutex_unlock(&to_mdev(dev)->cap_mask_mutex); 1476dfc3901SRoland Dreier } 1486dfc3901SRoland Dreier } 1496dfc3901SRoland Dreier 1501da177e4SLinus Torvalds static void forward_trap(struct mthca_dev *dev, 1511da177e4SLinus Torvalds u8 port_num, 1521da177e4SLinus Torvalds struct ib_mad *mad) 1531da177e4SLinus Torvalds { 1541da177e4SLinus Torvalds int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; 15534816ad9SSean Hefty struct ib_mad_send_buf *send_buf; 1561da177e4SLinus Torvalds struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn]; 1571da177e4SLinus Torvalds int ret; 1581da177e4SLinus Torvalds unsigned long flags; 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds if (agent) { 16134816ad9SSean Hefty send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR, 16234816ad9SSean Hefty IB_MGMT_MAD_DATA, GFP_ATOMIC); 1631da177e4SLinus Torvalds /* 1641da177e4SLinus Torvalds * We rely here on the fact that MLX QPs don't use the 1651da177e4SLinus Torvalds * address handle after the send is posted (this is 1661da177e4SLinus Torvalds * wrong following the IB spec strictly, but we know 1671da177e4SLinus Torvalds * it's OK for our devices). 1681da177e4SLinus Torvalds */ 1691da177e4SLinus Torvalds spin_lock_irqsave(&dev->sm_lock, flags); 17034816ad9SSean Hefty memcpy(send_buf->mad, mad, sizeof *mad); 17134816ad9SSean Hefty if ((send_buf->ah = dev->sm_ah[port_num - 1])) 17234816ad9SSean Hefty ret = ib_post_send_mad(send_buf, NULL); 1731da177e4SLinus Torvalds else 1741da177e4SLinus Torvalds ret = -EINVAL; 1751da177e4SLinus Torvalds spin_unlock_irqrestore(&dev->sm_lock, flags); 1761da177e4SLinus Torvalds 17734816ad9SSean Hefty if (ret) 17834816ad9SSean Hefty ib_free_send_mad(send_buf); 1791da177e4SLinus Torvalds } 1801da177e4SLinus Torvalds } 1811da177e4SLinus Torvalds 1821da177e4SLinus Torvalds int mthca_process_mad(struct ib_device *ibdev, 1831da177e4SLinus Torvalds int mad_flags, 1841da177e4SLinus Torvalds u8 port_num, 1851da177e4SLinus Torvalds struct ib_wc *in_wc, 1861da177e4SLinus Torvalds struct ib_grh *in_grh, 1871da177e4SLinus Torvalds struct ib_mad *in_mad, 1881da177e4SLinus Torvalds struct ib_mad *out_mad) 1891da177e4SLinus Torvalds { 1901da177e4SLinus Torvalds int err; 1911da177e4SLinus Torvalds u8 status; 19297f52eb4SSean Hefty u16 slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); 1931da177e4SLinus Torvalds 1941da177e4SLinus Torvalds /* Forward locally generated traps to the SM */ 1951da177e4SLinus Torvalds if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && 1961da177e4SLinus Torvalds slid == 0) { 1971da177e4SLinus Torvalds forward_trap(to_mdev(ibdev), port_num, in_mad); 1981da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; 1991da177e4SLinus Torvalds } 2001da177e4SLinus Torvalds 2011da177e4SLinus Torvalds /* 2021da177e4SLinus Torvalds * Only handle SM gets, sets and trap represses for SM class 2031da177e4SLinus Torvalds * 2041da177e4SLinus Torvalds * Only handle PMA and Mellanox vendor-specific class gets and 2051da177e4SLinus Torvalds * sets for other classes. 2061da177e4SLinus Torvalds */ 2071da177e4SLinus Torvalds if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 2081da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 2091da177e4SLinus Torvalds if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && 2101da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && 2111da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) 2121da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2131da177e4SLinus Torvalds 2141da177e4SLinus Torvalds /* 2151da177e4SLinus Torvalds * Don't process SMInfo queries or vendor-specific 2161da177e4SLinus Torvalds * MADs -- the SMA can't handle them. 2171da177e4SLinus Torvalds */ 2181da177e4SLinus Torvalds if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO || 2191da177e4SLinus Torvalds ((in_mad->mad_hdr.attr_id & IB_SMP_ATTR_VENDOR_MASK) == 2201da177e4SLinus Torvalds IB_SMP_ATTR_VENDOR_MASK)) 2211da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2221da177e4SLinus Torvalds } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || 2231da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS1 || 2241da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS2) { 2251da177e4SLinus Torvalds if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && 2261da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) 2271da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2281da177e4SLinus Torvalds } else 2291da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2301da177e4SLinus Torvalds 2311da177e4SLinus Torvalds err = mthca_MAD_IFC(to_mdev(ibdev), 2321da177e4SLinus Torvalds mad_flags & IB_MAD_IGNORE_MKEY, 2331da177e4SLinus Torvalds mad_flags & IB_MAD_IGNORE_BKEY, 2341da177e4SLinus Torvalds port_num, in_wc, in_grh, in_mad, out_mad, 2351da177e4SLinus Torvalds &status); 2361da177e4SLinus Torvalds if (err) { 2371da177e4SLinus Torvalds mthca_err(to_mdev(ibdev), "MAD_IFC failed\n"); 2381da177e4SLinus Torvalds return IB_MAD_RESULT_FAILURE; 2391da177e4SLinus Torvalds } 2401da177e4SLinus Torvalds if (status == MTHCA_CMD_STAT_BAD_PKT) 2411da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2421da177e4SLinus Torvalds if (status) { 2431da177e4SLinus Torvalds mthca_err(to_mdev(ibdev), "MAD_IFC returned status %02x\n", 2441da177e4SLinus Torvalds status); 2451da177e4SLinus Torvalds return IB_MAD_RESULT_FAILURE; 2461da177e4SLinus Torvalds } 2471da177e4SLinus Torvalds 2486dfc3901SRoland Dreier if (!out_mad->mad_hdr.status) { 2491da177e4SLinus Torvalds smp_snoop(ibdev, port_num, in_mad); 2506dfc3901SRoland Dreier node_desc_override(ibdev, out_mad); 2516dfc3901SRoland Dreier } 2521da177e4SLinus Torvalds 2531da177e4SLinus Torvalds /* set return bit in status of directed route responses */ 2541da177e4SLinus Torvalds if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 2551da177e4SLinus Torvalds out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); 2561da177e4SLinus Torvalds 2571da177e4SLinus Torvalds if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) 2581da177e4SLinus Torvalds /* no response for trap repress */ 2591da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; 2601da177e4SLinus Torvalds 2611da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; 2621da177e4SLinus Torvalds } 2631da177e4SLinus Torvalds 2641da177e4SLinus Torvalds static void send_handler(struct ib_mad_agent *agent, 2651da177e4SLinus Torvalds struct ib_mad_send_wc *mad_send_wc) 2661da177e4SLinus Torvalds { 26734816ad9SSean Hefty ib_free_send_mad(mad_send_wc->send_buf); 2681da177e4SLinus Torvalds } 2691da177e4SLinus Torvalds 2701da177e4SLinus Torvalds int mthca_create_agents(struct mthca_dev *dev) 2711da177e4SLinus Torvalds { 2721da177e4SLinus Torvalds struct ib_mad_agent *agent; 2731da177e4SLinus Torvalds int p, q; 274bf6a9e31SJack Morgenstein int ret; 2751da177e4SLinus Torvalds 2761da177e4SLinus Torvalds spin_lock_init(&dev->sm_lock); 2771da177e4SLinus Torvalds 2781da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) 2791da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) { 2801da177e4SLinus Torvalds agent = ib_register_mad_agent(&dev->ib_dev, p + 1, 2811da177e4SLinus Torvalds q ? IB_QPT_GSI : IB_QPT_SMI, 2821da177e4SLinus Torvalds NULL, 0, send_handler, 2831da177e4SLinus Torvalds NULL, NULL); 284bf6a9e31SJack Morgenstein if (IS_ERR(agent)) { 285bf6a9e31SJack Morgenstein ret = PTR_ERR(agent); 2861da177e4SLinus Torvalds goto err; 287bf6a9e31SJack Morgenstein } 2881da177e4SLinus Torvalds dev->send_agent[p][q] = agent; 2891da177e4SLinus Torvalds } 2901da177e4SLinus Torvalds 291bf6a9e31SJack Morgenstein 292bf6a9e31SJack Morgenstein for (p = 1; p <= dev->limits.num_ports; ++p) { 293bf6a9e31SJack Morgenstein ret = mthca_update_rate(dev, p); 294bf6a9e31SJack Morgenstein if (ret) { 295bf6a9e31SJack Morgenstein mthca_err(dev, "Failed to obtain port %d rate." 296bf6a9e31SJack Morgenstein " aborting.\n", p); 297bf6a9e31SJack Morgenstein goto err; 298bf6a9e31SJack Morgenstein } 299bf6a9e31SJack Morgenstein } 300bf6a9e31SJack Morgenstein 3011da177e4SLinus Torvalds return 0; 3021da177e4SLinus Torvalds 3031da177e4SLinus Torvalds err: 3041da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) 3051da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) 3061da177e4SLinus Torvalds if (dev->send_agent[p][q]) 3071da177e4SLinus Torvalds ib_unregister_mad_agent(dev->send_agent[p][q]); 3081da177e4SLinus Torvalds 309bf6a9e31SJack Morgenstein return ret; 3101da177e4SLinus Torvalds } 3111da177e4SLinus Torvalds 312e1f7868cSRoland Dreier void __devexit mthca_free_agents(struct mthca_dev *dev) 3131da177e4SLinus Torvalds { 3141da177e4SLinus Torvalds struct ib_mad_agent *agent; 3151da177e4SLinus Torvalds int p, q; 3161da177e4SLinus Torvalds 3171da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) { 3181da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) { 3191da177e4SLinus Torvalds agent = dev->send_agent[p][q]; 3201da177e4SLinus Torvalds dev->send_agent[p][q] = NULL; 3211da177e4SLinus Torvalds ib_unregister_mad_agent(agent); 3221da177e4SLinus Torvalds } 3231da177e4SLinus Torvalds 3241da177e4SLinus Torvalds if (dev->sm_ah[p]) 3251da177e4SLinus Torvalds ib_destroy_ah(dev->sm_ah[p]); 3261da177e4SLinus Torvalds } 3271da177e4SLinus Torvalds } 328