1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $ 33 */ 34 35 #include <linux/slab.h> 36 #include <linux/init.h> 37 #include <linux/errno.h> 38 39 #include "mthca_dev.h" 40 #include "mthca_cmd.h" 41 42 /* 43 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. 44 */ 45 struct mthca_mpt_entry { 46 u32 flags; 47 u32 page_size; 48 u32 key; 49 u32 pd; 50 u64 start; 51 u64 length; 52 u32 lkey; 53 u32 window_count; 54 u32 window_count_limit; 55 u64 mtt_seg; 56 u32 mtt_sz; /* Arbel only */ 57 u32 reserved[2]; 58 } __attribute__((packed)); 59 60 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28) 61 #define MTHCA_MPT_FLAG_MIO (1 << 17) 62 #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15) 63 #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9) 64 #define MTHCA_MPT_FLAG_REGION (1 << 8) 65 66 #define MTHCA_MTT_FLAG_PRESENT 1 67 68 /* 69 * Buddy allocator for MTT segments (currently not very efficient 70 * since it doesn't keep a free list and just searches linearly 71 * through the bitmaps) 72 */ 73 74 static u32 mthca_alloc_mtt(struct mthca_dev *dev, int order) 75 { 76 int o; 77 int m; 78 u32 seg; 79 80 spin_lock(&dev->mr_table.mpt_alloc.lock); 81 82 for (o = order; o <= dev->mr_table.max_mtt_order; ++o) { 83 m = 1 << (dev->mr_table.max_mtt_order - o); 84 seg = find_first_bit(dev->mr_table.mtt_buddy[o], m); 85 if (seg < m) 86 goto found; 87 } 88 89 spin_unlock(&dev->mr_table.mpt_alloc.lock); 90 return -1; 91 92 found: 93 clear_bit(seg, dev->mr_table.mtt_buddy[o]); 94 95 while (o > order) { 96 --o; 97 seg <<= 1; 98 set_bit(seg ^ 1, dev->mr_table.mtt_buddy[o]); 99 } 100 101 spin_unlock(&dev->mr_table.mpt_alloc.lock); 102 103 seg <<= order; 104 105 return seg; 106 } 107 108 static void mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order) 109 { 110 seg >>= order; 111 112 spin_lock(&dev->mr_table.mpt_alloc.lock); 113 114 while (test_bit(seg ^ 1, dev->mr_table.mtt_buddy[order])) { 115 clear_bit(seg ^ 1, dev->mr_table.mtt_buddy[order]); 116 seg >>= 1; 117 ++order; 118 } 119 120 set_bit(seg, dev->mr_table.mtt_buddy[order]); 121 122 spin_unlock(&dev->mr_table.mpt_alloc.lock); 123 } 124 125 static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind) 126 { 127 if (dev->hca_type == ARBEL_NATIVE) 128 return (ind >> 24) | (ind << 8); 129 else 130 return ind; 131 } 132 133 static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key) 134 { 135 if (dev->hca_type == ARBEL_NATIVE) 136 return (key << 24) | (key >> 8); 137 else 138 return key; 139 } 140 141 int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, 142 u32 access, struct mthca_mr *mr) 143 { 144 void *mailbox; 145 struct mthca_mpt_entry *mpt_entry; 146 u32 key; 147 int err; 148 u8 status; 149 150 might_sleep(); 151 152 mr->order = -1; 153 key = mthca_alloc(&dev->mr_table.mpt_alloc); 154 if (key == -1) 155 return -ENOMEM; 156 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 157 158 mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA, 159 GFP_KERNEL); 160 if (!mailbox) { 161 mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); 162 return -ENOMEM; 163 } 164 mpt_entry = MAILBOX_ALIGN(mailbox); 165 166 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 167 MTHCA_MPT_FLAG_MIO | 168 MTHCA_MPT_FLAG_PHYSICAL | 169 MTHCA_MPT_FLAG_REGION | 170 access); 171 mpt_entry->page_size = 0; 172 mpt_entry->key = cpu_to_be32(key); 173 mpt_entry->pd = cpu_to_be32(pd); 174 mpt_entry->start = 0; 175 mpt_entry->length = ~0ULL; 176 177 memset(&mpt_entry->lkey, 0, 178 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); 179 180 err = mthca_SW2HW_MPT(dev, mpt_entry, 181 key & (dev->limits.num_mpts - 1), 182 &status); 183 if (err) 184 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 185 else if (status) { 186 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 187 status); 188 err = -EINVAL; 189 } 190 191 kfree(mailbox); 192 return err; 193 } 194 195 int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, 196 u64 *buffer_list, int buffer_size_shift, 197 int list_len, u64 iova, u64 total_size, 198 u32 access, struct mthca_mr *mr) 199 { 200 void *mailbox; 201 u64 *mtt_entry; 202 struct mthca_mpt_entry *mpt_entry; 203 u32 key; 204 int err = -ENOMEM; 205 u8 status; 206 int i; 207 208 might_sleep(); 209 WARN_ON(buffer_size_shift >= 32); 210 211 key = mthca_alloc(&dev->mr_table.mpt_alloc); 212 if (key == -1) 213 return -ENOMEM; 214 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 215 216 for (i = dev->limits.mtt_seg_size / 8, mr->order = 0; 217 i < list_len; 218 i <<= 1, ++mr->order) 219 ; /* nothing */ 220 221 mr->first_seg = mthca_alloc_mtt(dev, mr->order); 222 if (mr->first_seg == -1) 223 goto err_out_mpt_free; 224 225 /* 226 * If list_len is odd, we add one more dummy entry for 227 * firmware efficiency. 228 */ 229 mailbox = kmalloc(max(sizeof *mpt_entry, 230 (size_t) 8 * (list_len + (list_len & 1) + 2)) + 231 MTHCA_CMD_MAILBOX_EXTRA, 232 GFP_KERNEL); 233 if (!mailbox) 234 goto err_out_free_mtt; 235 236 mtt_entry = MAILBOX_ALIGN(mailbox); 237 238 mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base + 239 mr->first_seg * dev->limits.mtt_seg_size); 240 mtt_entry[1] = 0; 241 for (i = 0; i < list_len; ++i) 242 mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] | 243 MTHCA_MTT_FLAG_PRESENT); 244 if (list_len & 1) { 245 mtt_entry[i + 2] = 0; 246 ++list_len; 247 } 248 249 if (0) { 250 mthca_dbg(dev, "Dumping MPT entry\n"); 251 for (i = 0; i < list_len + 2; ++i) 252 printk(KERN_ERR "[%2d] %016llx\n", 253 i, (unsigned long long) be64_to_cpu(mtt_entry[i])); 254 } 255 256 err = mthca_WRITE_MTT(dev, mtt_entry, list_len, &status); 257 if (err) { 258 mthca_warn(dev, "WRITE_MTT failed (%d)\n", err); 259 goto err_out_mailbox_free; 260 } 261 if (status) { 262 mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n", 263 status); 264 err = -EINVAL; 265 goto err_out_mailbox_free; 266 } 267 268 mpt_entry = MAILBOX_ALIGN(mailbox); 269 270 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 271 MTHCA_MPT_FLAG_MIO | 272 MTHCA_MPT_FLAG_REGION | 273 access); 274 275 mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12); 276 mpt_entry->key = cpu_to_be32(key); 277 mpt_entry->pd = cpu_to_be32(pd); 278 mpt_entry->start = cpu_to_be64(iova); 279 mpt_entry->length = cpu_to_be64(total_size); 280 memset(&mpt_entry->lkey, 0, 281 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); 282 mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + 283 mr->first_seg * dev->limits.mtt_seg_size); 284 285 if (0) { 286 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); 287 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { 288 if (i % 4 == 0) 289 printk("[%02x] ", i * 4); 290 printk(" %08x", be32_to_cpu(((u32 *) mpt_entry)[i])); 291 if ((i + 1) % 4 == 0) 292 printk("\n"); 293 } 294 } 295 296 err = mthca_SW2HW_MPT(dev, mpt_entry, 297 key & (dev->limits.num_mpts - 1), 298 &status); 299 if (err) 300 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 301 else if (status) { 302 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 303 status); 304 err = -EINVAL; 305 } 306 307 kfree(mailbox); 308 return err; 309 310 err_out_mailbox_free: 311 kfree(mailbox); 312 313 err_out_free_mtt: 314 mthca_free_mtt(dev, mr->first_seg, mr->order); 315 316 err_out_mpt_free: 317 mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); 318 return err; 319 } 320 321 void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr) 322 { 323 int err; 324 u8 status; 325 326 might_sleep(); 327 328 err = mthca_HW2SW_MPT(dev, NULL, 329 key_to_hw_index(dev, mr->ibmr.lkey) & 330 (dev->limits.num_mpts - 1), 331 &status); 332 if (err) 333 mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err); 334 else if (status) 335 mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n", 336 status); 337 338 if (mr->order >= 0) 339 mthca_free_mtt(dev, mr->first_seg, mr->order); 340 341 mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, mr->ibmr.lkey)); 342 } 343 344 int __devinit mthca_init_mr_table(struct mthca_dev *dev) 345 { 346 int err; 347 int i, s; 348 349 err = mthca_alloc_init(&dev->mr_table.mpt_alloc, 350 dev->limits.num_mpts, 351 ~0, dev->limits.reserved_mrws); 352 if (err) 353 return err; 354 355 err = -ENOMEM; 356 357 for (i = 1, dev->mr_table.max_mtt_order = 0; 358 i < dev->limits.num_mtt_segs; 359 i <<= 1, ++dev->mr_table.max_mtt_order) 360 ; /* nothing */ 361 362 dev->mr_table.mtt_buddy = kmalloc((dev->mr_table.max_mtt_order + 1) * 363 sizeof (long *), 364 GFP_KERNEL); 365 if (!dev->mr_table.mtt_buddy) 366 goto err_out; 367 368 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i) 369 dev->mr_table.mtt_buddy[i] = NULL; 370 371 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i) { 372 s = BITS_TO_LONGS(1 << (dev->mr_table.max_mtt_order - i)); 373 dev->mr_table.mtt_buddy[i] = kmalloc(s * sizeof (long), 374 GFP_KERNEL); 375 if (!dev->mr_table.mtt_buddy[i]) 376 goto err_out_free; 377 bitmap_zero(dev->mr_table.mtt_buddy[i], 378 1 << (dev->mr_table.max_mtt_order - i)); 379 } 380 381 set_bit(0, dev->mr_table.mtt_buddy[dev->mr_table.max_mtt_order]); 382 383 for (i = 0; i < dev->mr_table.max_mtt_order; ++i) 384 if (1 << i >= dev->limits.reserved_mtts) 385 break; 386 387 if (i == dev->mr_table.max_mtt_order) { 388 mthca_err(dev, "MTT table of order %d is " 389 "too small.\n", i); 390 goto err_out_free; 391 } 392 393 (void) mthca_alloc_mtt(dev, i); 394 395 return 0; 396 397 err_out_free: 398 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i) 399 kfree(dev->mr_table.mtt_buddy[i]); 400 401 err_out: 402 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 403 404 return err; 405 } 406 407 void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev) 408 { 409 int i; 410 411 /* XXX check if any MRs are still allocated? */ 412 for (i = 0; i <= dev->mr_table.max_mtt_order; ++i) 413 kfree(dev->mr_table.mtt_buddy[i]); 414 kfree(dev->mr_table.mtt_buddy); 415 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 416 } 417