11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * Copyright (c) 2004 Topspin Communications. All rights reserved. 32a1d9b7fSRoland Dreier * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 42a1d9b7fSRoland Dreier * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * This software is available to you under a choice of one of two 71da177e4SLinus Torvalds * licenses. You may choose to be licensed under the terms of the GNU 81da177e4SLinus Torvalds * General Public License (GPL) Version 2, available from the file 91da177e4SLinus Torvalds * COPYING in the main directory of this source tree, or the 101da177e4SLinus Torvalds * OpenIB.org BSD license below: 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Redistribution and use in source and binary forms, with or 131da177e4SLinus Torvalds * without modification, are permitted provided that the following 141da177e4SLinus Torvalds * conditions are met: 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * - Redistributions of source code must retain the above 171da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 181da177e4SLinus Torvalds * disclaimer. 191da177e4SLinus Torvalds * 201da177e4SLinus Torvalds * - Redistributions in binary form must reproduce the above 211da177e4SLinus Torvalds * copyright notice, this list of conditions and the following 221da177e4SLinus Torvalds * disclaimer in the documentation and/or other materials 231da177e4SLinus Torvalds * provided with the distribution. 241da177e4SLinus Torvalds * 251da177e4SLinus Torvalds * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 261da177e4SLinus Torvalds * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 271da177e4SLinus Torvalds * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 281da177e4SLinus Torvalds * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 291da177e4SLinus Torvalds * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 301da177e4SLinus Torvalds * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 311da177e4SLinus Torvalds * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 321da177e4SLinus Torvalds * SOFTWARE. 331da177e4SLinus Torvalds * 341da177e4SLinus Torvalds * $Id: mthca_mad.c 1349 2004-12-16 21:09:43Z roland $ 351da177e4SLinus Torvalds */ 361da177e4SLinus Torvalds 371da177e4SLinus Torvalds #include <ib_verbs.h> 381da177e4SLinus Torvalds #include <ib_mad.h> 391da177e4SLinus Torvalds #include <ib_smi.h> 401da177e4SLinus Torvalds 411da177e4SLinus Torvalds #include "mthca_dev.h" 421da177e4SLinus Torvalds #include "mthca_cmd.h" 431da177e4SLinus Torvalds 441da177e4SLinus Torvalds enum { 451da177e4SLinus Torvalds MTHCA_VENDOR_CLASS1 = 0x9, 461da177e4SLinus Torvalds MTHCA_VENDOR_CLASS2 = 0xa 471da177e4SLinus Torvalds }; 481da177e4SLinus Torvalds 491da177e4SLinus Torvalds struct mthca_trap_mad { 501da177e4SLinus Torvalds struct ib_mad *mad; 511da177e4SLinus Torvalds DECLARE_PCI_UNMAP_ADDR(mapping) 521da177e4SLinus Torvalds }; 531da177e4SLinus Torvalds 541da177e4SLinus Torvalds static void update_sm_ah(struct mthca_dev *dev, 551da177e4SLinus Torvalds u8 port_num, u16 lid, u8 sl) 561da177e4SLinus Torvalds { 571da177e4SLinus Torvalds struct ib_ah *new_ah; 581da177e4SLinus Torvalds struct ib_ah_attr ah_attr; 591da177e4SLinus Torvalds unsigned long flags; 601da177e4SLinus Torvalds 611da177e4SLinus Torvalds if (!dev->send_agent[port_num - 1][0]) 621da177e4SLinus Torvalds return; 631da177e4SLinus Torvalds 641da177e4SLinus Torvalds memset(&ah_attr, 0, sizeof ah_attr); 651da177e4SLinus Torvalds ah_attr.dlid = lid; 661da177e4SLinus Torvalds ah_attr.sl = sl; 671da177e4SLinus Torvalds ah_attr.port_num = port_num; 681da177e4SLinus Torvalds 691da177e4SLinus Torvalds new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, 701da177e4SLinus Torvalds &ah_attr); 711da177e4SLinus Torvalds if (IS_ERR(new_ah)) 721da177e4SLinus Torvalds return; 731da177e4SLinus Torvalds 741da177e4SLinus Torvalds spin_lock_irqsave(&dev->sm_lock, flags); 751da177e4SLinus Torvalds if (dev->sm_ah[port_num - 1]) 761da177e4SLinus Torvalds ib_destroy_ah(dev->sm_ah[port_num - 1]); 771da177e4SLinus Torvalds dev->sm_ah[port_num - 1] = new_ah; 781da177e4SLinus Torvalds spin_unlock_irqrestore(&dev->sm_lock, flags); 791da177e4SLinus Torvalds } 801da177e4SLinus Torvalds 811da177e4SLinus Torvalds /* 821da177e4SLinus Torvalds * Snoop SM MADs for port info and P_Key table sets, so we can 831da177e4SLinus Torvalds * synthesize LID change and P_Key change events. 841da177e4SLinus Torvalds */ 851da177e4SLinus Torvalds static void smp_snoop(struct ib_device *ibdev, 861da177e4SLinus Torvalds u8 port_num, 871da177e4SLinus Torvalds struct ib_mad *mad) 881da177e4SLinus Torvalds { 891da177e4SLinus Torvalds struct ib_event event; 901da177e4SLinus Torvalds 911da177e4SLinus Torvalds if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 921da177e4SLinus Torvalds mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && 931da177e4SLinus Torvalds mad->mad_hdr.method == IB_MGMT_METHOD_SET) { 941da177e4SLinus Torvalds if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO) { 951da177e4SLinus Torvalds update_sm_ah(to_mdev(ibdev), port_num, 961da177e4SLinus Torvalds be16_to_cpup((__be16 *) (mad->data + 58)), 971da177e4SLinus Torvalds (*(u8 *) (mad->data + 76)) & 0xf); 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds event.device = ibdev; 1001da177e4SLinus Torvalds event.event = IB_EVENT_LID_CHANGE; 1011da177e4SLinus Torvalds event.element.port_num = port_num; 1021da177e4SLinus Torvalds ib_dispatch_event(&event); 1031da177e4SLinus Torvalds } 1041da177e4SLinus Torvalds 1051da177e4SLinus Torvalds if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PKEY_TABLE) { 1061da177e4SLinus Torvalds event.device = ibdev; 1071da177e4SLinus Torvalds event.event = IB_EVENT_PKEY_CHANGE; 1081da177e4SLinus Torvalds event.element.port_num = port_num; 1091da177e4SLinus Torvalds ib_dispatch_event(&event); 1101da177e4SLinus Torvalds } 1111da177e4SLinus Torvalds } 1121da177e4SLinus Torvalds } 1131da177e4SLinus Torvalds 1141da177e4SLinus Torvalds static void forward_trap(struct mthca_dev *dev, 1151da177e4SLinus Torvalds u8 port_num, 1161da177e4SLinus Torvalds struct ib_mad *mad) 1171da177e4SLinus Torvalds { 1181da177e4SLinus Torvalds int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; 1191da177e4SLinus Torvalds struct mthca_trap_mad *tmad; 1201da177e4SLinus Torvalds struct ib_sge gather_list; 1211da177e4SLinus Torvalds struct ib_send_wr *bad_wr, wr = { 1221da177e4SLinus Torvalds .opcode = IB_WR_SEND, 1231da177e4SLinus Torvalds .sg_list = &gather_list, 1241da177e4SLinus Torvalds .num_sge = 1, 1251da177e4SLinus Torvalds .send_flags = IB_SEND_SIGNALED, 1261da177e4SLinus Torvalds .wr = { 1271da177e4SLinus Torvalds .ud = { 1281da177e4SLinus Torvalds .remote_qpn = qpn, 1291da177e4SLinus Torvalds .remote_qkey = qpn ? IB_QP1_QKEY : 0, 1301da177e4SLinus Torvalds .timeout_ms = 0 1311da177e4SLinus Torvalds } 1321da177e4SLinus Torvalds } 1331da177e4SLinus Torvalds }; 1341da177e4SLinus Torvalds struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn]; 1351da177e4SLinus Torvalds int ret; 1361da177e4SLinus Torvalds unsigned long flags; 1371da177e4SLinus Torvalds 1381da177e4SLinus Torvalds if (agent) { 1391da177e4SLinus Torvalds tmad = kmalloc(sizeof *tmad, GFP_KERNEL); 1401da177e4SLinus Torvalds if (!tmad) 1411da177e4SLinus Torvalds return; 1421da177e4SLinus Torvalds 1431da177e4SLinus Torvalds tmad->mad = kmalloc(sizeof *tmad->mad, GFP_KERNEL); 1441da177e4SLinus Torvalds if (!tmad->mad) { 1451da177e4SLinus Torvalds kfree(tmad); 1461da177e4SLinus Torvalds return; 1471da177e4SLinus Torvalds } 1481da177e4SLinus Torvalds 1491da177e4SLinus Torvalds memcpy(tmad->mad, mad, sizeof *mad); 1501da177e4SLinus Torvalds 1511da177e4SLinus Torvalds wr.wr.ud.mad_hdr = &tmad->mad->mad_hdr; 1521da177e4SLinus Torvalds wr.wr_id = (unsigned long) tmad; 1531da177e4SLinus Torvalds 1541da177e4SLinus Torvalds gather_list.addr = dma_map_single(agent->device->dma_device, 1551da177e4SLinus Torvalds tmad->mad, 1561da177e4SLinus Torvalds sizeof *tmad->mad, 1571da177e4SLinus Torvalds DMA_TO_DEVICE); 1581da177e4SLinus Torvalds gather_list.length = sizeof *tmad->mad; 1591da177e4SLinus Torvalds gather_list.lkey = to_mpd(agent->qp->pd)->ntmr.ibmr.lkey; 1601da177e4SLinus Torvalds pci_unmap_addr_set(tmad, mapping, gather_list.addr); 1611da177e4SLinus Torvalds 1621da177e4SLinus Torvalds /* 1631da177e4SLinus Torvalds * We rely here on the fact that MLX QPs don't use the 1641da177e4SLinus Torvalds * address handle after the send is posted (this is 1651da177e4SLinus Torvalds * wrong following the IB spec strictly, but we know 1661da177e4SLinus Torvalds * it's OK for our devices). 1671da177e4SLinus Torvalds */ 1681da177e4SLinus Torvalds spin_lock_irqsave(&dev->sm_lock, flags); 1691da177e4SLinus Torvalds wr.wr.ud.ah = dev->sm_ah[port_num - 1]; 1701da177e4SLinus Torvalds if (wr.wr.ud.ah) 1711da177e4SLinus Torvalds ret = ib_post_send_mad(agent, &wr, &bad_wr); 1721da177e4SLinus Torvalds else 1731da177e4SLinus Torvalds ret = -EINVAL; 1741da177e4SLinus Torvalds spin_unlock_irqrestore(&dev->sm_lock, flags); 1751da177e4SLinus Torvalds 1761da177e4SLinus Torvalds if (ret) { 1771da177e4SLinus Torvalds dma_unmap_single(agent->device->dma_device, 1781da177e4SLinus Torvalds pci_unmap_addr(tmad, mapping), 1791da177e4SLinus Torvalds sizeof *tmad->mad, 1801da177e4SLinus Torvalds DMA_TO_DEVICE); 1811da177e4SLinus Torvalds kfree(tmad->mad); 1821da177e4SLinus Torvalds kfree(tmad); 1831da177e4SLinus Torvalds } 1841da177e4SLinus Torvalds } 1851da177e4SLinus Torvalds } 1861da177e4SLinus Torvalds 1871da177e4SLinus Torvalds int mthca_process_mad(struct ib_device *ibdev, 1881da177e4SLinus Torvalds int mad_flags, 1891da177e4SLinus Torvalds u8 port_num, 1901da177e4SLinus Torvalds struct ib_wc *in_wc, 1911da177e4SLinus Torvalds struct ib_grh *in_grh, 1921da177e4SLinus Torvalds struct ib_mad *in_mad, 1931da177e4SLinus Torvalds struct ib_mad *out_mad) 1941da177e4SLinus Torvalds { 1951da177e4SLinus Torvalds int err; 1961da177e4SLinus Torvalds u8 status; 19797f52eb4SSean Hefty u16 slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); 1981da177e4SLinus Torvalds 1991da177e4SLinus Torvalds /* Forward locally generated traps to the SM */ 2001da177e4SLinus Torvalds if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && 2011da177e4SLinus Torvalds slid == 0) { 2021da177e4SLinus Torvalds forward_trap(to_mdev(ibdev), port_num, in_mad); 2031da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; 2041da177e4SLinus Torvalds } 2051da177e4SLinus Torvalds 2061da177e4SLinus Torvalds /* 2071da177e4SLinus Torvalds * Only handle SM gets, sets and trap represses for SM class 2081da177e4SLinus Torvalds * 2091da177e4SLinus Torvalds * Only handle PMA and Mellanox vendor-specific class gets and 2101da177e4SLinus Torvalds * sets for other classes. 2111da177e4SLinus Torvalds */ 2121da177e4SLinus Torvalds if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 2131da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 2141da177e4SLinus Torvalds if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && 2151da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && 2161da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) 2171da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2181da177e4SLinus Torvalds 2191da177e4SLinus Torvalds /* 2201da177e4SLinus Torvalds * Don't process SMInfo queries or vendor-specific 2211da177e4SLinus Torvalds * MADs -- the SMA can't handle them. 2221da177e4SLinus Torvalds */ 2231da177e4SLinus Torvalds if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO || 2241da177e4SLinus Torvalds ((in_mad->mad_hdr.attr_id & IB_SMP_ATTR_VENDOR_MASK) == 2251da177e4SLinus Torvalds IB_SMP_ATTR_VENDOR_MASK)) 2261da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2271da177e4SLinus Torvalds } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || 2281da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS1 || 2291da177e4SLinus Torvalds in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS2) { 2301da177e4SLinus Torvalds if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && 2311da177e4SLinus Torvalds in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) 2321da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2331da177e4SLinus Torvalds } else 2341da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2351da177e4SLinus Torvalds 2361da177e4SLinus Torvalds err = mthca_MAD_IFC(to_mdev(ibdev), 2371da177e4SLinus Torvalds mad_flags & IB_MAD_IGNORE_MKEY, 2381da177e4SLinus Torvalds mad_flags & IB_MAD_IGNORE_BKEY, 2391da177e4SLinus Torvalds port_num, in_wc, in_grh, in_mad, out_mad, 2401da177e4SLinus Torvalds &status); 2411da177e4SLinus Torvalds if (err) { 2421da177e4SLinus Torvalds mthca_err(to_mdev(ibdev), "MAD_IFC failed\n"); 2431da177e4SLinus Torvalds return IB_MAD_RESULT_FAILURE; 2441da177e4SLinus Torvalds } 2451da177e4SLinus Torvalds if (status == MTHCA_CMD_STAT_BAD_PKT) 2461da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS; 2471da177e4SLinus Torvalds if (status) { 2481da177e4SLinus Torvalds mthca_err(to_mdev(ibdev), "MAD_IFC returned status %02x\n", 2491da177e4SLinus Torvalds status); 2501da177e4SLinus Torvalds return IB_MAD_RESULT_FAILURE; 2511da177e4SLinus Torvalds } 2521da177e4SLinus Torvalds 2531da177e4SLinus Torvalds if (!out_mad->mad_hdr.status) 2541da177e4SLinus Torvalds smp_snoop(ibdev, port_num, in_mad); 2551da177e4SLinus Torvalds 2561da177e4SLinus Torvalds /* set return bit in status of directed route responses */ 2571da177e4SLinus Torvalds if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 2581da177e4SLinus Torvalds out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); 2591da177e4SLinus Torvalds 2601da177e4SLinus Torvalds if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) 2611da177e4SLinus Torvalds /* no response for trap repress */ 2621da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; 2631da177e4SLinus Torvalds 2641da177e4SLinus Torvalds return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; 2651da177e4SLinus Torvalds } 2661da177e4SLinus Torvalds 2671da177e4SLinus Torvalds static void send_handler(struct ib_mad_agent *agent, 2681da177e4SLinus Torvalds struct ib_mad_send_wc *mad_send_wc) 2691da177e4SLinus Torvalds { 2701da177e4SLinus Torvalds struct mthca_trap_mad *tmad = 2711da177e4SLinus Torvalds (void *) (unsigned long) mad_send_wc->wr_id; 2721da177e4SLinus Torvalds 2731da177e4SLinus Torvalds dma_unmap_single(agent->device->dma_device, 2741da177e4SLinus Torvalds pci_unmap_addr(tmad, mapping), 2751da177e4SLinus Torvalds sizeof *tmad->mad, 2761da177e4SLinus Torvalds DMA_TO_DEVICE); 2771da177e4SLinus Torvalds kfree(tmad->mad); 2781da177e4SLinus Torvalds kfree(tmad); 2791da177e4SLinus Torvalds } 2801da177e4SLinus Torvalds 2811da177e4SLinus Torvalds int mthca_create_agents(struct mthca_dev *dev) 2821da177e4SLinus Torvalds { 2831da177e4SLinus Torvalds struct ib_mad_agent *agent; 2841da177e4SLinus Torvalds int p, q; 2851da177e4SLinus Torvalds 2861da177e4SLinus Torvalds spin_lock_init(&dev->sm_lock); 2871da177e4SLinus Torvalds 2881da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) 2891da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) { 2901da177e4SLinus Torvalds agent = ib_register_mad_agent(&dev->ib_dev, p + 1, 2911da177e4SLinus Torvalds q ? IB_QPT_GSI : IB_QPT_SMI, 2921da177e4SLinus Torvalds NULL, 0, send_handler, 2931da177e4SLinus Torvalds NULL, NULL); 2941da177e4SLinus Torvalds if (IS_ERR(agent)) 2951da177e4SLinus Torvalds goto err; 2961da177e4SLinus Torvalds dev->send_agent[p][q] = agent; 2971da177e4SLinus Torvalds } 2981da177e4SLinus Torvalds 2991da177e4SLinus Torvalds return 0; 3001da177e4SLinus Torvalds 3011da177e4SLinus Torvalds err: 3021da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) 3031da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) 3041da177e4SLinus Torvalds if (dev->send_agent[p][q]) 3051da177e4SLinus Torvalds ib_unregister_mad_agent(dev->send_agent[p][q]); 3061da177e4SLinus Torvalds 3071da177e4SLinus Torvalds return PTR_ERR(agent); 3081da177e4SLinus Torvalds } 3091da177e4SLinus Torvalds 3101da177e4SLinus Torvalds void mthca_free_agents(struct mthca_dev *dev) 3111da177e4SLinus Torvalds { 3121da177e4SLinus Torvalds struct ib_mad_agent *agent; 3131da177e4SLinus Torvalds int p, q; 3141da177e4SLinus Torvalds 3151da177e4SLinus Torvalds for (p = 0; p < dev->limits.num_ports; ++p) { 3161da177e4SLinus Torvalds for (q = 0; q <= 1; ++q) { 3171da177e4SLinus Torvalds agent = dev->send_agent[p][q]; 3181da177e4SLinus Torvalds dev->send_agent[p][q] = NULL; 3191da177e4SLinus Torvalds ib_unregister_mad_agent(agent); 3201da177e4SLinus Torvalds } 3211da177e4SLinus Torvalds 3221da177e4SLinus Torvalds if (dev->sm_ah[p]) 3231da177e4SLinus Torvalds ib_destroy_ah(dev->sm_ah[p]); 3241da177e4SLinus Torvalds } 3251da177e4SLinus Torvalds } 326