11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Communications. All rights reserved. 32a1d9b7fSRoland Dreier * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 42a1d9b7fSRoland Dreier * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This software is available to you under a choice of one of two 71da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 81da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 91da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 101da177e4SLinus Torvalds * OpenIB.org BSD license below: 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 131da177e4SLinus Torvalds * without modification, are permitted provided that the following 141da177e4SLinus Torvalds * conditions are met: 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * - Redistributions of source code must retain the above 171da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 181da177e4SLinus Torvalds * disclaimer. 191da177e4SLinus Torvalds * 201da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 211da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 221da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 231da177e4SLinus Torvalds * provided with the distribution. 241da177e4SLinus Torvalds * 251da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 261da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 271da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 281da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 291da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 301da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 311da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 321da177e4SLinus Torvalds * SOFTWARE. 331da177e4SLinus Torvalds * 341da177e4SLinus Torvalds * $Id: mthca_mad.c 1349 2004-12-16 21:09:43Z roland $ 351da177e4SLinus Torvalds */ 361da177e4SLinus Torvalds 374e57b681STim Schmielau #include <linux/string.h> 384e57b681STim Schmielau #include <linux/slab.h> 394e57b681STim Schmielau 40a4d61e84SRoland Dreier #include <rdma/ib_verbs.h> 41a4d61e84SRoland Dreier #include <rdma/ib_mad.h> 42a4d61e84SRoland Dreier #include <rdma/ib_smi.h> 431da177e4SLinus Torvalds 441da177e4SLinus Torvalds #include "mthca_dev.h" 451da177e4SLinus Torvalds #include "mthca_cmd.h" 461da177e4SLinus Torvalds 471da177e4SLinus Torvalds enum { 481da177e4SLinus Torvalds MTHCA_VENDOR_CLASS1 = 0x9, 491da177e4SLinus Torvalds MTHCA_VENDOR_CLASS2 = 0xa 501da177e4SLinus Torvalds }; 511da177e4SLinus Torvalds 521da177e4SLinus Torvalds static void update_sm_ah(struct mthca_dev *dev, 531da177e4SLinus Torvalds u8 port_num, u16 lid, u8 sl) 541da177e4SLinus Torvalds { 551da177e4SLinus Torvalds struct ib_ah *new_ah; 561da177e4SLinus Torvalds struct ib_ah_attr ah_attr; 571da177e4SLinus Torvalds unsigned long flags; 581da177e4SLinus Torvalds 591da177e4SLinus Torvalds if (!dev->send_agent[port_num - 1][0]) 601da177e4SLinus Torvalds return; 611da177e4SLinus Torvalds 621da177e4SLinus Torvalds memset(&ah_attr, 0, sizeof ah_attr); 631da177e4SLinus Torvalds ah_attr.dlid = lid; 641da177e4SLinus Torvalds ah_attr.sl = sl; 651da177e4SLinus Torvalds ah_attr.port_num = port_num; 661da177e4SLinus Torvalds 671da177e4SLinus Torvalds new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, 681da177e4SLinus Torvalds &ah_attr); 691da177e4SLinus Torvalds if (IS_ERR(new_ah)) 701da177e4SLinus Torvalds return; 711da177e4SLinus Torvalds 721da177e4SLinus Torvalds spin_lock_irqsave(&dev->sm_lock, flags); 731da177e4SLinus Torvalds if (dev->sm_ah[port_num - 1]) 741da177e4SLinus Torvalds ib_destroy_ah(dev->sm_ah[port_num - 1]); 751da177e4SLinus Torvalds dev->sm_ah[port_num - 1] = new_ah; 761da177e4SLinus Torvalds spin_unlock_irqrestore(&dev->sm_lock, flags); 771da177e4SLinus Torvalds } 781da177e4SLinus Torvalds 791da177e4SLinus Torvalds /* 801da177e4SLinus Torvalds * Snoop SM MADs for port info and P_Key table sets, so we can 811da177e4SLinus Torvalds * synthesize LID change and P_Key change events. 821da177e4SLinus Torvalds */ 831da177e4SLinus Torvalds static void smp_snoop(struct ib_device *ibdev, 841da177e4SLinus Torvalds u8 port_num, 851da177e4SLinus Torvalds struct ib_mad *mad) 861da177e4SLinus Torvalds { 871da177e4SLinus Torvalds struct ib_event event; 881da177e4SLinus Torvalds 891da177e4SLinus Torvalds if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 901da177e4SLinus Torvalds mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && 911da177e4SLinus Torvalds mad->mad_hdr.method == IB_MGMT_METHOD_SET) { 921da177e4SLinus Torvalds if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO) { 931da177e4SLinus Torvalds update_sm_ah(to_mdev(ibdev), port_num, 941da177e4SLinus Torvalds be16_to_cpup((__be16 *) (mad->data + 58)), 951da177e4SLinus Torvalds (*(u8 *) (mad->data + 76)) & 0xf); 961da177e4SLinus Torvalds 971da177e4SLinus Torvalds event.device = ibdev; 981da177e4SLinus Torvalds event.event = IB_EVENT_LID_CHANGE; 991da177e4SLinus Torvalds event.element.port_num = port_num; 1001da177e4SLinus Torvalds ib_dispatch_event(&event); 1011da177e4SLinus Torvalds } 1021da177e4SLinus Torvalds 1031da177e4SLinus Torvalds if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PKEY_TABLE) { 1041da177e4SLinus Torvalds event.device = ibdev; 1051da177e4SLinus Torvalds event.event = IB_EVENT_PKEY_CHANGE; 1061da177e4SLinus Torvalds event.element.port_num = port_num; 1071da177e4SLinus Torvalds ib_dispatch_event(&event); 1081da177e4SLinus Torvalds } 1091da177e4SLinus Torvalds } 1101da177e4SLinus Torvalds } 1111da177e4SLinus Torvalds 1126dfc3901SRoland Dreier static void node_desc_override(struct ib_device *dev, 1136dfc3901SRoland Dreier struct ib_mad *mad) 1146dfc3901SRoland Dreier { 1156dfc3901SRoland Dreier if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 1166dfc3901SRoland Dreier mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && 1176dfc3901SRoland Dreier mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP && 1186dfc3901SRoland Dreier mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) { 1196dfc3901SRoland Dreier mutex_lock(&to_mdev(dev)->cap_mask_mutex); 1206dfc3901SRoland Dreier memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64); 1216dfc3901SRoland Dreier mutex_unlock(&to_mdev(dev)->cap_mask_mutex); 1226dfc3901SRoland Dreier } 1236dfc3901SRoland Dreier } 1246dfc3901SRoland Dreier 1251da177e4SLinus Torvalds static void forward_trap(struct mthca_dev *dev, 1261da177e4SLinus Torvalds u8 port_num, 1271da177e4SLinus Torvalds struct ib_mad *mad) 1281da177e4SLinus Torvalds { 1291da177e4SLinus Torvalds int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; 13034816ad9SSean Hefty struct ib_mad_send_buf *send_buf; 1311da177e4SLinus Torvalds struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn]; 1321da177e4SLinus Torvalds int ret; 1331da177e4SLinus Torvalds unsigned long flags; 1341da177e4SLinus Torvalds 1351da177e4SLinus Torvalds if (agent) { 13634816ad9SSean Hefty send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR, 13734816ad9SSean Hefty IB_MGMT_MAD_DATA, GFP_ATOMIC); 1381da177e4SLinus Torvalds /* 1391da177e4SLinus Torvalds * We rely here on the fact that MLX QPs don't use the 1401da177e4SLinus Torvalds * address handle after the send is posted (this is 1411da177e4SLinus Torvalds * wrong following the IB spec strictly, but we know 1421da177e4SLinus Torvalds * it's OK for our devices). 1431da177e4SLinus Torvalds */ 1441da177e4SLinus Torvalds spin_lock_irqsave(&dev->sm_lock, flags); 14534816ad9SSean Hefty memcpy(send_buf->mad, mad, sizeof *mad); 14634816ad9SSean Hefty if ((send_buf->ah = dev->sm_ah[port_num - 1])) 14734816ad9SSean Hefty ret = ib_post_send_mad(send_buf, NULL); 1481da177e4SLinus Torvalds else 1491da177e4SLinus Torvalds ret = -EINVAL; 1501da177e4SLinus Torvalds spin_unlock_irqrestore(&dev->sm_lock, flags); 1511da177e4SLinus Torvalds 15234816ad9SSean Hefty if (ret) 15334816ad9SSean Hefty ib_free_send_mad(send_buf); 1541da177e4SLinus Torvalds } 1551da177e4SLinus Torvalds } 1561da177e4SLinus Torvalds 1571da177e4SLinus Torvalds int mthca_process_mad(struct ib_device *ibdev, 1581da177e4SLinus Torvalds int mad_flags, 1591da177e4SLinus Torvalds u8 port_num, 1601da177e4SLinus Torvalds struct ib_wc *in_wc, 1611da177e4SLinus Torvalds struct ib_grh *in_grh, 1621da177e4SLinus Torvalds struct ib_mad *in_mad, 1631da177e4SLinus Torvalds struct ib_mad *out_mad) 1641da177e4SLinus Torvalds { 1651da177e4SLinus Torvalds int err; 1661da177e4SLinus Torvalds u8 status; 16797f52eb4SSean Hefty u16 slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); 1681da177e4SLinus Torvalds 1691da177e4SLinus Torvalds /* Forward locally generated traps to the SM */ 1701da177e4SLinus Torvalds if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && 1711da177e4SLinus Torvalds slid == 0) { 1721da177e4SLinus Torvalds forward_trap(to_mdev(ibdev), port_num, in_mad); 1731da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; 1741da177e4SLinus Torvalds } 1751da177e4SLinus Torvalds 1761da177e4SLinus Torvalds /* 1771da177e4SLinus Torvalds * Only handle SM gets, sets and trap represses for SM class 1781da177e4SLinus Torvalds * 1791da177e4SLinus Torvalds * Only handle PMA and Mellanox vendor-specific class gets and 1801da177e4SLinus Torvalds * sets for other classes. 1811da177e4SLinus Torvalds */ 1821da177e4SLinus Torvalds if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 1831da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 1841da177e4SLinus Torvalds if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && 1851da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && 1861da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) 1871da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 1881da177e4SLinus Torvalds 1891da177e4SLinus Torvalds /* 1901da177e4SLinus Torvalds * Don't process SMInfo queries or vendor-specific 1911da177e4SLinus Torvalds * MADs -- the SMA can't handle them. 1921da177e4SLinus Torvalds */ 1931da177e4SLinus Torvalds if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO || 1941da177e4SLinus Torvalds ((in_mad->mad_hdr.attr_id & IB_SMP_ATTR_VENDOR_MASK) == 1951da177e4SLinus Torvalds IB_SMP_ATTR_VENDOR_MASK)) 1961da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 1971da177e4SLinus Torvalds } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || 1981da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS1 || 1991da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS2) { 2001da177e4SLinus Torvalds if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && 2011da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) 2021da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2031da177e4SLinus Torvalds } else 2041da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2051da177e4SLinus Torvalds 2061da177e4SLinus Torvalds err = mthca_MAD_IFC(to_mdev(ibdev), 2071da177e4SLinus Torvalds mad_flags & IB_MAD_IGNORE_MKEY, 2081da177e4SLinus Torvalds mad_flags & IB_MAD_IGNORE_BKEY, 2091da177e4SLinus Torvalds port_num, in_wc, in_grh, in_mad, out_mad, 2101da177e4SLinus Torvalds &status); 2111da177e4SLinus Torvalds if (err) { 2121da177e4SLinus Torvalds mthca_err(to_mdev(ibdev), "MAD_IFC failed\n"); 2131da177e4SLinus Torvalds return IB_MAD_RESULT_FAILURE; 2141da177e4SLinus Torvalds } 2151da177e4SLinus Torvalds if (status == MTHCA_CMD_STAT_BAD_PKT) 2161da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2171da177e4SLinus Torvalds if (status) { 2181da177e4SLinus Torvalds mthca_err(to_mdev(ibdev), "MAD_IFC returned status %02x\n", 2191da177e4SLinus Torvalds status); 2201da177e4SLinus Torvalds return IB_MAD_RESULT_FAILURE; 2211da177e4SLinus Torvalds } 2221da177e4SLinus Torvalds 2236dfc3901SRoland Dreier if (!out_mad->mad_hdr.status) { 2241da177e4SLinus Torvalds smp_snoop(ibdev, port_num, in_mad); 2256dfc3901SRoland Dreier node_desc_override(ibdev, out_mad); 2266dfc3901SRoland Dreier } 2271da177e4SLinus Torvalds 2281da177e4SLinus Torvalds /* set return bit in status of directed route responses */ 2291da177e4SLinus Torvalds if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 2301da177e4SLinus Torvalds out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); 2311da177e4SLinus Torvalds 2321da177e4SLinus Torvalds if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) 2331da177e4SLinus Torvalds /* no response for trap repress */ 2341da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; 2351da177e4SLinus Torvalds 2361da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; 2371da177e4SLinus Torvalds } 2381da177e4SLinus Torvalds 2391da177e4SLinus Torvalds static void send_handler(struct ib_mad_agent *agent, 2401da177e4SLinus Torvalds struct ib_mad_send_wc *mad_send_wc) 2411da177e4SLinus Torvalds { 24234816ad9SSean Hefty ib_free_send_mad(mad_send_wc->send_buf); 2431da177e4SLinus Torvalds } 2441da177e4SLinus Torvalds 2451da177e4SLinus Torvalds int mthca_create_agents(struct mthca_dev *dev) 2461da177e4SLinus Torvalds { 2471da177e4SLinus Torvalds struct ib_mad_agent *agent; 2481da177e4SLinus Torvalds int p, q; 2491da177e4SLinus Torvalds 2501da177e4SLinus Torvalds spin_lock_init(&dev->sm_lock); 2511da177e4SLinus Torvalds 2521da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) 2531da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) { 2541da177e4SLinus Torvalds agent = ib_register_mad_agent(&dev->ib_dev, p + 1, 2551da177e4SLinus Torvalds q ? IB_QPT_GSI : IB_QPT_SMI, 2561da177e4SLinus Torvalds NULL, 0, send_handler, 2571da177e4SLinus Torvalds NULL, NULL); 2581da177e4SLinus Torvalds if (IS_ERR(agent)) 2591da177e4SLinus Torvalds goto err; 2601da177e4SLinus Torvalds dev->send_agent[p][q] = agent; 2611da177e4SLinus Torvalds } 2621da177e4SLinus Torvalds 2631da177e4SLinus Torvalds return 0; 2641da177e4SLinus Torvalds 2651da177e4SLinus Torvalds err: 2661da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) 2671da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) 2681da177e4SLinus Torvalds if (dev->send_agent[p][q]) 2691da177e4SLinus Torvalds ib_unregister_mad_agent(dev->send_agent[p][q]); 2701da177e4SLinus Torvalds 2711da177e4SLinus Torvalds return PTR_ERR(agent); 2721da177e4SLinus Torvalds } 2731da177e4SLinus Torvalds 274e1f7868cSRoland Dreier void __devexit mthca_free_agents(struct mthca_dev *dev) 2751da177e4SLinus Torvalds { 2761da177e4SLinus Torvalds struct ib_mad_agent *agent; 2771da177e4SLinus Torvalds int p, q; 2781da177e4SLinus Torvalds 2791da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) { 2801da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) { 2811da177e4SLinus Torvalds agent = dev->send_agent[p][q]; 2821da177e4SLinus Torvalds dev->send_agent[p][q] = NULL; 2831da177e4SLinus Torvalds ib_unregister_mad_agent(agent); 2841da177e4SLinus Torvalds } 2851da177e4SLinus Torvalds 2861da177e4SLinus Torvalds if (dev->sm_ah[p]) 2871da177e4SLinus Torvalds ib_destroy_ah(dev->sm_ah[p]); 2881da177e4SLinus Torvalds } 2891da177e4SLinus Torvalds } 290