1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 * $Id: mthca_av.c 1349 2004-12-16 21:09:43Z roland $ 33 */ 34 35 #include <linux/init.h> 36 37 #include <ib_verbs.h> 38 #include <ib_cache.h> 39 40 #include "mthca_dev.h" 41 42 struct mthca_av { 43 u32 port_pd; 44 u8 reserved1; 45 u8 g_slid; 46 u16 dlid; 47 u8 reserved2; 48 u8 gid_index; 49 u8 msg_sr; 50 u8 hop_limit; 51 u32 sl_tclass_flowlabel; 52 u32 dgid[4]; 53 }; 54 55 int mthca_create_ah(struct mthca_dev *dev, 56 struct mthca_pd *pd, 57 struct ib_ah_attr *ah_attr, 58 struct mthca_ah *ah) 59 { 60 u32 index = -1; 61 struct mthca_av *av = NULL; 62 63 ah->type = MTHCA_AH_PCI_POOL; 64 65 if (dev->hca_type == ARBEL_NATIVE) { 66 ah->av = kmalloc(sizeof *ah->av, GFP_KERNEL); 67 if (!ah->av) 68 return -ENOMEM; 69 70 ah->type = MTHCA_AH_KMALLOC; 71 av = ah->av; 72 } else if (!atomic_read(&pd->sqp_count) && 73 !(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { 74 index = mthca_alloc(&dev->av_table.alloc); 75 76 /* fall back to allocate in host memory */ 77 if (index == -1) 78 goto on_hca_fail; 79 80 av = kmalloc(sizeof *av, GFP_KERNEL); 81 if (!av) 82 goto on_hca_fail; 83 84 ah->type = MTHCA_AH_ON_HCA; 85 ah->avdma = dev->av_table.ddr_av_base + 86 index * MTHCA_AV_SIZE; 87 } 88 89 on_hca_fail: 90 if (ah->type == MTHCA_AH_PCI_POOL) { 91 ah->av = pci_pool_alloc(dev->av_table.pool, 92 SLAB_KERNEL, &ah->avdma); 93 if (!ah->av) 94 return -ENOMEM; 95 96 av = ah->av; 97 } 98 99 ah->key = pd->ntmr.ibmr.lkey; 100 101 memset(av, 0, MTHCA_AV_SIZE); 102 103 av->port_pd = cpu_to_be32(pd->pd_num | (ah_attr->port_num << 24)); 104 av->g_slid = ah_attr->src_path_bits; 105 av->dlid = cpu_to_be16(ah_attr->dlid); 106 av->msg_sr = (3 << 4) | /* 2K message */ 107 ah_attr->static_rate; 108 av->sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); 109 if (ah_attr->ah_flags & IB_AH_GRH) { 110 av->g_slid |= 0x80; 111 av->gid_index = (ah_attr->port_num - 1) * dev->limits.gid_table_len + 112 ah_attr->grh.sgid_index; 113 av->hop_limit = ah_attr->grh.hop_limit; 114 av->sl_tclass_flowlabel |= 115 cpu_to_be32((ah_attr->grh.traffic_class << 20) | 116 ah_attr->grh.flow_label); 117 memcpy(av->dgid, ah_attr->grh.dgid.raw, 16); 118 } else { 119 /* Arbel workaround -- low byte of GID must be 2 */ 120 av->dgid[3] = cpu_to_be32(2); 121 } 122 123 if (0) { 124 int j; 125 126 mthca_dbg(dev, "Created UDAV at %p/%08lx:\n", 127 av, (unsigned long) ah->avdma); 128 for (j = 0; j < 8; ++j) 129 printk(KERN_DEBUG " [%2x] %08x\n", 130 j * 4, be32_to_cpu(((u32 *) av)[j])); 131 } 132 133 if (ah->type == MTHCA_AH_ON_HCA) { 134 memcpy_toio(dev->av_table.av_map + index * MTHCA_AV_SIZE, 135 av, MTHCA_AV_SIZE); 136 kfree(av); 137 } 138 139 return 0; 140 } 141 142 int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah) 143 { 144 switch (ah->type) { 145 case MTHCA_AH_ON_HCA: 146 mthca_free(&dev->av_table.alloc, 147 (ah->avdma - dev->av_table.ddr_av_base) / 148 MTHCA_AV_SIZE); 149 break; 150 151 case MTHCA_AH_PCI_POOL: 152 pci_pool_free(dev->av_table.pool, ah->av, ah->avdma); 153 break; 154 155 case MTHCA_AH_KMALLOC: 156 kfree(ah->av); 157 break; 158 } 159 160 return 0; 161 } 162 163 int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah, 164 struct ib_ud_header *header) 165 { 166 if (ah->type == MTHCA_AH_ON_HCA) 167 return -EINVAL; 168 169 header->lrh.service_level = be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 28; 170 header->lrh.destination_lid = ah->av->dlid; 171 header->lrh.source_lid = ah->av->g_slid & 0x7f; 172 if (ah->av->g_slid & 0x80) { 173 header->grh_present = 1; 174 header->grh.traffic_class = 175 (be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 20) & 0xff; 176 header->grh.flow_label = 177 ah->av->sl_tclass_flowlabel & cpu_to_be32(0xfffff); 178 ib_get_cached_gid(&dev->ib_dev, 179 be32_to_cpu(ah->av->port_pd) >> 24, 180 ah->av->gid_index, 181 &header->grh.source_gid); 182 memcpy(header->grh.destination_gid.raw, 183 ah->av->dgid, 16); 184 } else { 185 header->grh_present = 0; 186 } 187 188 return 0; 189 } 190 191 int __devinit mthca_init_av_table(struct mthca_dev *dev) 192 { 193 int err; 194 195 if (dev->hca_type == ARBEL_NATIVE) 196 return 0; 197 198 err = mthca_alloc_init(&dev->av_table.alloc, 199 dev->av_table.num_ddr_avs, 200 dev->av_table.num_ddr_avs - 1, 201 0); 202 if (err) 203 return err; 204 205 dev->av_table.pool = pci_pool_create("mthca_av", dev->pdev, 206 MTHCA_AV_SIZE, 207 MTHCA_AV_SIZE, 0); 208 if (!dev->av_table.pool) 209 goto out_free_alloc; 210 211 if (!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { 212 dev->av_table.av_map = ioremap(pci_resource_start(dev->pdev, 4) + 213 dev->av_table.ddr_av_base - 214 dev->ddr_start, 215 dev->av_table.num_ddr_avs * 216 MTHCA_AV_SIZE); 217 if (!dev->av_table.av_map) 218 goto out_free_pool; 219 } else 220 dev->av_table.av_map = NULL; 221 222 return 0; 223 224 out_free_pool: 225 pci_pool_destroy(dev->av_table.pool); 226 227 out_free_alloc: 228 mthca_alloc_cleanup(&dev->av_table.alloc); 229 return -ENOMEM; 230 } 231 232 void __devexit mthca_cleanup_av_table(struct mthca_dev *dev) 233 { 234 if (dev->hca_type == ARBEL_NATIVE) 235 return; 236 237 if (dev->av_table.av_map) 238 iounmap(dev->av_table.av_map); 239 pci_pool_destroy(dev->av_table.pool); 240 mthca_alloc_cleanup(&dev->av_table.alloc); 241 } 242