1 /* 2 * Copyright(c) 2015-2018 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 #include <asm/page.h> 48 #include <linux/string.h> 49 50 #include "mmu_rb.h" 51 #include "user_exp_rcv.h" 52 #include "trace.h" 53 54 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt, 55 struct exp_tid_set *set, 56 struct hfi1_filedata *fd); 57 static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages); 58 static int set_rcvarray_entry(struct hfi1_filedata *fd, 59 struct tid_user_buf *tbuf, 60 u32 rcventry, struct tid_group *grp, 61 u16 pageidx, unsigned int npages); 62 static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata, 63 struct tid_rb_node *tnode); 64 static bool tid_rb_invalidate(struct mmu_interval_notifier *mni, 65 const struct mmu_notifier_range *range, 66 unsigned long cur_seq); 67 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *, 68 struct tid_group *grp, 69 unsigned int start, u16 count, 70 u32 *tidlist, unsigned int *tididx, 71 unsigned int *pmapped); 72 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo, 73 struct tid_group **grp); 74 static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node); 75 76 static const struct mmu_interval_notifier_ops tid_mn_ops = { 77 .invalidate = tid_rb_invalidate, 78 }; 79 80 /* 81 * Initialize context and file private data needed for Expected 82 * receive caching. This needs to be done after the context has 83 * been configured with the eager/expected RcvEntry counts. 84 */ 85 int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd, 86 struct hfi1_ctxtdata *uctxt) 87 { 88 int ret = 0; 89 90 spin_lock_init(&fd->tid_lock); 91 spin_lock_init(&fd->invalid_lock); 92 93 fd->entry_to_rb = kcalloc(uctxt->expected_count, 94 sizeof(struct rb_node *), 95 GFP_KERNEL); 96 if (!fd->entry_to_rb) 97 return -ENOMEM; 98 99 if (!HFI1_CAP_UGET_MASK(uctxt->flags, TID_UNMAP)) { 100 fd->invalid_tid_idx = 0; 101 fd->invalid_tids = kcalloc(uctxt->expected_count, 102 sizeof(*fd->invalid_tids), 103 GFP_KERNEL); 104 if (!fd->invalid_tids) { 105 kfree(fd->entry_to_rb); 106 fd->entry_to_rb = NULL; 107 return -ENOMEM; 108 } 109 fd->use_mn = true; 110 } 111 112 /* 113 * PSM does not have a good way to separate, count, and 114 * effectively enforce a limit on RcvArray entries used by 115 * subctxts (when context sharing is used) when TID caching 116 * is enabled. To help with that, we calculate a per-process 117 * RcvArray entry share and enforce that. 118 * If TID caching is not in use, PSM deals with usage on its 119 * own. In that case, we allow any subctxt to take all of the 120 * entries. 121 * 122 * Make sure that we set the tid counts only after successful 123 * init. 124 */ 125 spin_lock(&fd->tid_lock); 126 if (uctxt->subctxt_cnt && fd->use_mn) { 127 u16 remainder; 128 129 fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt; 130 remainder = uctxt->expected_count % uctxt->subctxt_cnt; 131 if (remainder && fd->subctxt < remainder) 132 fd->tid_limit++; 133 } else { 134 fd->tid_limit = uctxt->expected_count; 135 } 136 spin_unlock(&fd->tid_lock); 137 138 return ret; 139 } 140 141 void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd) 142 { 143 struct hfi1_ctxtdata *uctxt = fd->uctxt; 144 145 if (!EXP_TID_SET_EMPTY(uctxt->tid_full_list)) 146 unlock_exp_tids(uctxt, &uctxt->tid_full_list, fd); 147 if (!EXP_TID_SET_EMPTY(uctxt->tid_used_list)) 148 unlock_exp_tids(uctxt, &uctxt->tid_used_list, fd); 149 150 kfree(fd->invalid_tids); 151 fd->invalid_tids = NULL; 152 153 kfree(fd->entry_to_rb); 154 fd->entry_to_rb = NULL; 155 } 156 157 /** 158 * Release pinned receive buffer pages. 159 * 160 * @mapped - true if the pages have been DMA mapped. false otherwise. 161 * @idx - Index of the first page to unpin. 162 * @npages - No of pages to unpin. 163 * 164 * If the pages have been DMA mapped (indicated by mapped parameter), their 165 * info will be passed via a struct tid_rb_node. If they haven't been mapped, 166 * their info will be passed via a struct tid_user_buf. 167 */ 168 static void unpin_rcv_pages(struct hfi1_filedata *fd, 169 struct tid_user_buf *tidbuf, 170 struct tid_rb_node *node, 171 unsigned int idx, 172 unsigned int npages, 173 bool mapped) 174 { 175 struct page **pages; 176 struct hfi1_devdata *dd = fd->uctxt->dd; 177 178 if (mapped) { 179 pci_unmap_single(dd->pcidev, node->dma_addr, 180 node->npages * PAGE_SIZE, PCI_DMA_FROMDEVICE); 181 pages = &node->pages[idx]; 182 } else { 183 pages = &tidbuf->pages[idx]; 184 } 185 hfi1_release_user_pages(fd->mm, pages, npages, mapped); 186 fd->tid_n_pinned -= npages; 187 } 188 189 /** 190 * Pin receive buffer pages. 191 */ 192 static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf) 193 { 194 int pinned; 195 unsigned int npages; 196 unsigned long vaddr = tidbuf->vaddr; 197 struct page **pages = NULL; 198 struct hfi1_devdata *dd = fd->uctxt->dd; 199 200 /* Get the number of pages the user buffer spans */ 201 npages = num_user_pages(vaddr, tidbuf->length); 202 if (!npages) 203 return -EINVAL; 204 205 if (npages > fd->uctxt->expected_count) { 206 dd_dev_err(dd, "Expected buffer too big\n"); 207 return -EINVAL; 208 } 209 210 /* Verify that access is OK for the user buffer */ 211 if (!access_ok((void __user *)vaddr, 212 npages * PAGE_SIZE)) { 213 dd_dev_err(dd, "Fail vaddr %p, %u pages, !access_ok\n", 214 (void *)vaddr, npages); 215 return -EFAULT; 216 } 217 /* Allocate the array of struct page pointers needed for pinning */ 218 pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL); 219 if (!pages) 220 return -ENOMEM; 221 222 /* 223 * Pin all the pages of the user buffer. If we can't pin all the 224 * pages, accept the amount pinned so far and program only that. 225 * User space knows how to deal with partially programmed buffers. 226 */ 227 if (!hfi1_can_pin_pages(dd, fd->mm, fd->tid_n_pinned, npages)) { 228 kfree(pages); 229 return -ENOMEM; 230 } 231 232 pinned = hfi1_acquire_user_pages(fd->mm, vaddr, npages, true, pages); 233 if (pinned <= 0) { 234 kfree(pages); 235 return pinned; 236 } 237 tidbuf->pages = pages; 238 tidbuf->npages = npages; 239 fd->tid_n_pinned += pinned; 240 return pinned; 241 } 242 243 /* 244 * RcvArray entry allocation for Expected Receives is done by the 245 * following algorithm: 246 * 247 * The context keeps 3 lists of groups of RcvArray entries: 248 * 1. List of empty groups - tid_group_list 249 * This list is created during user context creation and 250 * contains elements which describe sets (of 8) of empty 251 * RcvArray entries. 252 * 2. List of partially used groups - tid_used_list 253 * This list contains sets of RcvArray entries which are 254 * not completely used up. Another mapping request could 255 * use some of all of the remaining entries. 256 * 3. List of full groups - tid_full_list 257 * This is the list where sets that are completely used 258 * up go. 259 * 260 * An attempt to optimize the usage of RcvArray entries is 261 * made by finding all sets of physically contiguous pages in a 262 * user's buffer. 263 * These physically contiguous sets are further split into 264 * sizes supported by the receive engine of the HFI. The 265 * resulting sets of pages are stored in struct tid_pageset, 266 * which describes the sets as: 267 * * .count - number of pages in this set 268 * * .idx - starting index into struct page ** array 269 * of this set 270 * 271 * From this point on, the algorithm deals with the page sets 272 * described above. The number of pagesets is divided by the 273 * RcvArray group size to produce the number of full groups 274 * needed. 275 * 276 * Groups from the 3 lists are manipulated using the following 277 * rules: 278 * 1. For each set of 8 pagesets, a complete group from 279 * tid_group_list is taken, programmed, and moved to 280 * the tid_full_list list. 281 * 2. For all remaining pagesets: 282 * 2.1 If the tid_used_list is empty and the tid_group_list 283 * is empty, stop processing pageset and return only 284 * what has been programmed up to this point. 285 * 2.2 If the tid_used_list is empty and the tid_group_list 286 * is not empty, move a group from tid_group_list to 287 * tid_used_list. 288 * 2.3 For each group is tid_used_group, program as much as 289 * can fit into the group. If the group becomes fully 290 * used, move it to tid_full_list. 291 */ 292 int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, 293 struct hfi1_tid_info *tinfo) 294 { 295 int ret = 0, need_group = 0, pinned; 296 struct hfi1_ctxtdata *uctxt = fd->uctxt; 297 struct hfi1_devdata *dd = uctxt->dd; 298 unsigned int ngroups, pageidx = 0, pageset_count, 299 tididx = 0, mapped, mapped_pages = 0; 300 u32 *tidlist = NULL; 301 struct tid_user_buf *tidbuf; 302 303 if (!PAGE_ALIGNED(tinfo->vaddr)) 304 return -EINVAL; 305 306 tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL); 307 if (!tidbuf) 308 return -ENOMEM; 309 310 tidbuf->vaddr = tinfo->vaddr; 311 tidbuf->length = tinfo->length; 312 tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets), 313 GFP_KERNEL); 314 if (!tidbuf->psets) { 315 kfree(tidbuf); 316 return -ENOMEM; 317 } 318 319 pinned = pin_rcv_pages(fd, tidbuf); 320 if (pinned <= 0) { 321 kfree(tidbuf->psets); 322 kfree(tidbuf); 323 return pinned; 324 } 325 326 /* Find sets of physically contiguous pages */ 327 tidbuf->n_psets = find_phys_blocks(tidbuf, pinned); 328 329 /* 330 * We don't need to access this under a lock since tid_used is per 331 * process and the same process cannot be in hfi1_user_exp_rcv_clear() 332 * and hfi1_user_exp_rcv_setup() at the same time. 333 */ 334 spin_lock(&fd->tid_lock); 335 if (fd->tid_used + tidbuf->n_psets > fd->tid_limit) 336 pageset_count = fd->tid_limit - fd->tid_used; 337 else 338 pageset_count = tidbuf->n_psets; 339 spin_unlock(&fd->tid_lock); 340 341 if (!pageset_count) 342 goto bail; 343 344 ngroups = pageset_count / dd->rcv_entries.group_size; 345 tidlist = kcalloc(pageset_count, sizeof(*tidlist), GFP_KERNEL); 346 if (!tidlist) { 347 ret = -ENOMEM; 348 goto nomem; 349 } 350 351 tididx = 0; 352 353 /* 354 * From this point on, we are going to be using shared (between master 355 * and subcontexts) context resources. We need to take the lock. 356 */ 357 mutex_lock(&uctxt->exp_mutex); 358 /* 359 * The first step is to program the RcvArray entries which are complete 360 * groups. 361 */ 362 while (ngroups && uctxt->tid_group_list.count) { 363 struct tid_group *grp = 364 tid_group_pop(&uctxt->tid_group_list); 365 366 ret = program_rcvarray(fd, tidbuf, grp, 367 pageidx, dd->rcv_entries.group_size, 368 tidlist, &tididx, &mapped); 369 /* 370 * If there was a failure to program the RcvArray 371 * entries for the entire group, reset the grp fields 372 * and add the grp back to the free group list. 373 */ 374 if (ret <= 0) { 375 tid_group_add_tail(grp, &uctxt->tid_group_list); 376 hfi1_cdbg(TID, 377 "Failed to program RcvArray group %d", ret); 378 goto unlock; 379 } 380 381 tid_group_add_tail(grp, &uctxt->tid_full_list); 382 ngroups--; 383 pageidx += ret; 384 mapped_pages += mapped; 385 } 386 387 while (pageidx < pageset_count) { 388 struct tid_group *grp, *ptr; 389 /* 390 * If we don't have any partially used tid groups, check 391 * if we have empty groups. If so, take one from there and 392 * put in the partially used list. 393 */ 394 if (!uctxt->tid_used_list.count || need_group) { 395 if (!uctxt->tid_group_list.count) 396 goto unlock; 397 398 grp = tid_group_pop(&uctxt->tid_group_list); 399 tid_group_add_tail(grp, &uctxt->tid_used_list); 400 need_group = 0; 401 } 402 /* 403 * There is an optimization opportunity here - instead of 404 * fitting as many page sets as we can, check for a group 405 * later on in the list that could fit all of them. 406 */ 407 list_for_each_entry_safe(grp, ptr, &uctxt->tid_used_list.list, 408 list) { 409 unsigned use = min_t(unsigned, pageset_count - pageidx, 410 grp->size - grp->used); 411 412 ret = program_rcvarray(fd, tidbuf, grp, 413 pageidx, use, tidlist, 414 &tididx, &mapped); 415 if (ret < 0) { 416 hfi1_cdbg(TID, 417 "Failed to program RcvArray entries %d", 418 ret); 419 goto unlock; 420 } else if (ret > 0) { 421 if (grp->used == grp->size) 422 tid_group_move(grp, 423 &uctxt->tid_used_list, 424 &uctxt->tid_full_list); 425 pageidx += ret; 426 mapped_pages += mapped; 427 need_group = 0; 428 /* Check if we are done so we break out early */ 429 if (pageidx >= pageset_count) 430 break; 431 } else if (WARN_ON(ret == 0)) { 432 /* 433 * If ret is 0, we did not program any entries 434 * into this group, which can only happen if 435 * we've screwed up the accounting somewhere. 436 * Warn and try to continue. 437 */ 438 need_group = 1; 439 } 440 } 441 } 442 unlock: 443 mutex_unlock(&uctxt->exp_mutex); 444 nomem: 445 hfi1_cdbg(TID, "total mapped: tidpairs:%u pages:%u (%d)", tididx, 446 mapped_pages, ret); 447 if (tididx) { 448 spin_lock(&fd->tid_lock); 449 fd->tid_used += tididx; 450 spin_unlock(&fd->tid_lock); 451 tinfo->tidcnt = tididx; 452 tinfo->length = mapped_pages * PAGE_SIZE; 453 454 if (copy_to_user(u64_to_user_ptr(tinfo->tidlist), 455 tidlist, sizeof(tidlist[0]) * tididx)) { 456 /* 457 * On failure to copy to the user level, we need to undo 458 * everything done so far so we don't leak resources. 459 */ 460 tinfo->tidlist = (unsigned long)&tidlist; 461 hfi1_user_exp_rcv_clear(fd, tinfo); 462 tinfo->tidlist = 0; 463 ret = -EFAULT; 464 goto bail; 465 } 466 } 467 468 /* 469 * If not everything was mapped (due to insufficient RcvArray entries, 470 * for example), unpin all unmapped pages so we can pin them nex time. 471 */ 472 if (mapped_pages != pinned) 473 unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages, 474 (pinned - mapped_pages), false); 475 bail: 476 kfree(tidbuf->psets); 477 kfree(tidlist); 478 kfree(tidbuf->pages); 479 kfree(tidbuf); 480 return ret > 0 ? 0 : ret; 481 } 482 483 int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd, 484 struct hfi1_tid_info *tinfo) 485 { 486 int ret = 0; 487 struct hfi1_ctxtdata *uctxt = fd->uctxt; 488 u32 *tidinfo; 489 unsigned tididx; 490 491 if (unlikely(tinfo->tidcnt > fd->tid_used)) 492 return -EINVAL; 493 494 tidinfo = memdup_user(u64_to_user_ptr(tinfo->tidlist), 495 sizeof(tidinfo[0]) * tinfo->tidcnt); 496 if (IS_ERR(tidinfo)) 497 return PTR_ERR(tidinfo); 498 499 mutex_lock(&uctxt->exp_mutex); 500 for (tididx = 0; tididx < tinfo->tidcnt; tididx++) { 501 ret = unprogram_rcvarray(fd, tidinfo[tididx], NULL); 502 if (ret) { 503 hfi1_cdbg(TID, "Failed to unprogram rcv array %d", 504 ret); 505 break; 506 } 507 } 508 spin_lock(&fd->tid_lock); 509 fd->tid_used -= tididx; 510 spin_unlock(&fd->tid_lock); 511 tinfo->tidcnt = tididx; 512 mutex_unlock(&uctxt->exp_mutex); 513 514 kfree(tidinfo); 515 return ret; 516 } 517 518 int hfi1_user_exp_rcv_invalid(struct hfi1_filedata *fd, 519 struct hfi1_tid_info *tinfo) 520 { 521 struct hfi1_ctxtdata *uctxt = fd->uctxt; 522 unsigned long *ev = uctxt->dd->events + 523 (uctxt_offset(uctxt) + fd->subctxt); 524 u32 *array; 525 int ret = 0; 526 527 /* 528 * copy_to_user() can sleep, which will leave the invalid_lock 529 * locked and cause the MMU notifier to be blocked on the lock 530 * for a long time. 531 * Copy the data to a local buffer so we can release the lock. 532 */ 533 array = kcalloc(uctxt->expected_count, sizeof(*array), GFP_KERNEL); 534 if (!array) 535 return -EFAULT; 536 537 spin_lock(&fd->invalid_lock); 538 if (fd->invalid_tid_idx) { 539 memcpy(array, fd->invalid_tids, sizeof(*array) * 540 fd->invalid_tid_idx); 541 memset(fd->invalid_tids, 0, sizeof(*fd->invalid_tids) * 542 fd->invalid_tid_idx); 543 tinfo->tidcnt = fd->invalid_tid_idx; 544 fd->invalid_tid_idx = 0; 545 /* 546 * Reset the user flag while still holding the lock. 547 * Otherwise, PSM can miss events. 548 */ 549 clear_bit(_HFI1_EVENT_TID_MMU_NOTIFY_BIT, ev); 550 } else { 551 tinfo->tidcnt = 0; 552 } 553 spin_unlock(&fd->invalid_lock); 554 555 if (tinfo->tidcnt) { 556 if (copy_to_user((void __user *)tinfo->tidlist, 557 array, sizeof(*array) * tinfo->tidcnt)) 558 ret = -EFAULT; 559 } 560 kfree(array); 561 562 return ret; 563 } 564 565 static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages) 566 { 567 unsigned pagecount, pageidx, setcount = 0, i; 568 unsigned long pfn, this_pfn; 569 struct page **pages = tidbuf->pages; 570 struct tid_pageset *list = tidbuf->psets; 571 572 if (!npages) 573 return 0; 574 575 /* 576 * Look for sets of physically contiguous pages in the user buffer. 577 * This will allow us to optimize Expected RcvArray entry usage by 578 * using the bigger supported sizes. 579 */ 580 pfn = page_to_pfn(pages[0]); 581 for (pageidx = 0, pagecount = 1, i = 1; i <= npages; i++) { 582 this_pfn = i < npages ? page_to_pfn(pages[i]) : 0; 583 584 /* 585 * If the pfn's are not sequential, pages are not physically 586 * contiguous. 587 */ 588 if (this_pfn != ++pfn) { 589 /* 590 * At this point we have to loop over the set of 591 * physically contiguous pages and break them down it 592 * sizes supported by the HW. 593 * There are two main constraints: 594 * 1. The max buffer size is MAX_EXPECTED_BUFFER. 595 * If the total set size is bigger than that 596 * program only a MAX_EXPECTED_BUFFER chunk. 597 * 2. The buffer size has to be a power of two. If 598 * it is not, round down to the closes power of 599 * 2 and program that size. 600 */ 601 while (pagecount) { 602 int maxpages = pagecount; 603 u32 bufsize = pagecount * PAGE_SIZE; 604 605 if (bufsize > MAX_EXPECTED_BUFFER) 606 maxpages = 607 MAX_EXPECTED_BUFFER >> 608 PAGE_SHIFT; 609 else if (!is_power_of_2(bufsize)) 610 maxpages = 611 rounddown_pow_of_two(bufsize) >> 612 PAGE_SHIFT; 613 614 list[setcount].idx = pageidx; 615 list[setcount].count = maxpages; 616 pagecount -= maxpages; 617 pageidx += maxpages; 618 setcount++; 619 } 620 pageidx = i; 621 pagecount = 1; 622 pfn = this_pfn; 623 } else { 624 pagecount++; 625 } 626 } 627 return setcount; 628 } 629 630 /** 631 * program_rcvarray() - program an RcvArray group with receive buffers 632 * @fd: filedata pointer 633 * @tbuf: pointer to struct tid_user_buf that has the user buffer starting 634 * virtual address, buffer length, page pointers, pagesets (array of 635 * struct tid_pageset holding information on physically contiguous 636 * chunks from the user buffer), and other fields. 637 * @grp: RcvArray group 638 * @start: starting index into sets array 639 * @count: number of struct tid_pageset's to program 640 * @tidlist: the array of u32 elements when the information about the 641 * programmed RcvArray entries is to be encoded. 642 * @tididx: starting offset into tidlist 643 * @pmapped: (output parameter) number of pages programmed into the RcvArray 644 * entries. 645 * 646 * This function will program up to 'count' number of RcvArray entries from the 647 * group 'grp'. To make best use of write-combining writes, the function will 648 * perform writes to the unused RcvArray entries which will be ignored by the 649 * HW. Each RcvArray entry will be programmed with a physically contiguous 650 * buffer chunk from the user's virtual buffer. 651 * 652 * Return: 653 * -EINVAL if the requested count is larger than the size of the group, 654 * -ENOMEM or -EFAULT on error from set_rcvarray_entry(), or 655 * number of RcvArray entries programmed. 656 */ 657 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf, 658 struct tid_group *grp, 659 unsigned int start, u16 count, 660 u32 *tidlist, unsigned int *tididx, 661 unsigned int *pmapped) 662 { 663 struct hfi1_ctxtdata *uctxt = fd->uctxt; 664 struct hfi1_devdata *dd = uctxt->dd; 665 u16 idx; 666 u32 tidinfo = 0, rcventry, useidx = 0; 667 int mapped = 0; 668 669 /* Count should never be larger than the group size */ 670 if (count > grp->size) 671 return -EINVAL; 672 673 /* Find the first unused entry in the group */ 674 for (idx = 0; idx < grp->size; idx++) { 675 if (!(grp->map & (1 << idx))) { 676 useidx = idx; 677 break; 678 } 679 rcv_array_wc_fill(dd, grp->base + idx); 680 } 681 682 idx = 0; 683 while (idx < count) { 684 u16 npages, pageidx, setidx = start + idx; 685 int ret = 0; 686 687 /* 688 * If this entry in the group is used, move to the next one. 689 * If we go past the end of the group, exit the loop. 690 */ 691 if (useidx >= grp->size) { 692 break; 693 } else if (grp->map & (1 << useidx)) { 694 rcv_array_wc_fill(dd, grp->base + useidx); 695 useidx++; 696 continue; 697 } 698 699 rcventry = grp->base + useidx; 700 npages = tbuf->psets[setidx].count; 701 pageidx = tbuf->psets[setidx].idx; 702 703 ret = set_rcvarray_entry(fd, tbuf, 704 rcventry, grp, pageidx, 705 npages); 706 if (ret) 707 return ret; 708 mapped += npages; 709 710 tidinfo = rcventry2tidinfo(rcventry - uctxt->expected_base) | 711 EXP_TID_SET(LEN, npages); 712 tidlist[(*tididx)++] = tidinfo; 713 grp->used++; 714 grp->map |= 1 << useidx++; 715 idx++; 716 } 717 718 /* Fill the rest of the group with "blank" writes */ 719 for (; useidx < grp->size; useidx++) 720 rcv_array_wc_fill(dd, grp->base + useidx); 721 *pmapped = mapped; 722 return idx; 723 } 724 725 static int set_rcvarray_entry(struct hfi1_filedata *fd, 726 struct tid_user_buf *tbuf, 727 u32 rcventry, struct tid_group *grp, 728 u16 pageidx, unsigned int npages) 729 { 730 int ret; 731 struct hfi1_ctxtdata *uctxt = fd->uctxt; 732 struct tid_rb_node *node; 733 struct hfi1_devdata *dd = uctxt->dd; 734 dma_addr_t phys; 735 struct page **pages = tbuf->pages + pageidx; 736 737 /* 738 * Allocate the node first so we can handle a potential 739 * failure before we've programmed anything. 740 */ 741 node = kzalloc(sizeof(*node) + (sizeof(struct page *) * npages), 742 GFP_KERNEL); 743 if (!node) 744 return -ENOMEM; 745 746 phys = pci_map_single(dd->pcidev, 747 __va(page_to_phys(pages[0])), 748 npages * PAGE_SIZE, PCI_DMA_FROMDEVICE); 749 if (dma_mapping_error(&dd->pcidev->dev, phys)) { 750 dd_dev_err(dd, "Failed to DMA map Exp Rcv pages 0x%llx\n", 751 phys); 752 kfree(node); 753 return -EFAULT; 754 } 755 756 node->fdata = fd; 757 node->phys = page_to_phys(pages[0]); 758 node->npages = npages; 759 node->rcventry = rcventry; 760 node->dma_addr = phys; 761 node->grp = grp; 762 node->freed = false; 763 memcpy(node->pages, pages, sizeof(struct page *) * npages); 764 765 if (fd->use_mn) { 766 ret = mmu_interval_notifier_insert( 767 &node->notifier, fd->mm, 768 tbuf->vaddr + (pageidx * PAGE_SIZE), npages * PAGE_SIZE, 769 &tid_mn_ops); 770 if (ret) 771 goto out_unmap; 772 /* 773 * FIXME: This is in the wrong order, the notifier should be 774 * established before the pages are pinned by pin_rcv_pages. 775 */ 776 mmu_interval_read_begin(&node->notifier); 777 } 778 fd->entry_to_rb[node->rcventry - uctxt->expected_base] = node; 779 780 hfi1_put_tid(dd, rcventry, PT_EXPECTED, phys, ilog2(npages) + 1); 781 trace_hfi1_exp_tid_reg(uctxt->ctxt, fd->subctxt, rcventry, npages, 782 node->notifier.interval_tree.start, node->phys, 783 phys); 784 return 0; 785 786 out_unmap: 787 hfi1_cdbg(TID, "Failed to insert RB node %u 0x%lx, 0x%lx %d", 788 node->rcventry, node->notifier.interval_tree.start, 789 node->phys, ret); 790 pci_unmap_single(dd->pcidev, phys, npages * PAGE_SIZE, 791 PCI_DMA_FROMDEVICE); 792 kfree(node); 793 return -EFAULT; 794 } 795 796 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo, 797 struct tid_group **grp) 798 { 799 struct hfi1_ctxtdata *uctxt = fd->uctxt; 800 struct hfi1_devdata *dd = uctxt->dd; 801 struct tid_rb_node *node; 802 u8 tidctrl = EXP_TID_GET(tidinfo, CTRL); 803 u32 tididx = EXP_TID_GET(tidinfo, IDX) << 1, rcventry; 804 805 if (tididx >= uctxt->expected_count) { 806 dd_dev_err(dd, "Invalid RcvArray entry (%u) index for ctxt %u\n", 807 tididx, uctxt->ctxt); 808 return -EINVAL; 809 } 810 811 if (tidctrl == 0x3) 812 return -EINVAL; 813 814 rcventry = tididx + (tidctrl - 1); 815 816 node = fd->entry_to_rb[rcventry]; 817 if (!node || node->rcventry != (uctxt->expected_base + rcventry)) 818 return -EBADF; 819 820 if (grp) 821 *grp = node->grp; 822 823 if (fd->use_mn) 824 mmu_interval_notifier_remove(&node->notifier); 825 cacheless_tid_rb_remove(fd, node); 826 827 return 0; 828 } 829 830 static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node) 831 { 832 struct hfi1_ctxtdata *uctxt = fd->uctxt; 833 struct hfi1_devdata *dd = uctxt->dd; 834 835 trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry, 836 node->npages, 837 node->notifier.interval_tree.start, node->phys, 838 node->dma_addr); 839 840 /* 841 * Make sure device has seen the write before we unpin the 842 * pages. 843 */ 844 hfi1_put_tid(dd, node->rcventry, PT_INVALID_FLUSH, 0, 0); 845 846 unpin_rcv_pages(fd, NULL, node, 0, node->npages, true); 847 848 node->grp->used--; 849 node->grp->map &= ~(1 << (node->rcventry - node->grp->base)); 850 851 if (node->grp->used == node->grp->size - 1) 852 tid_group_move(node->grp, &uctxt->tid_full_list, 853 &uctxt->tid_used_list); 854 else if (!node->grp->used) 855 tid_group_move(node->grp, &uctxt->tid_used_list, 856 &uctxt->tid_group_list); 857 kfree(node); 858 } 859 860 /* 861 * As a simple helper for hfi1_user_exp_rcv_free, this function deals with 862 * clearing nodes in the non-cached case. 863 */ 864 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt, 865 struct exp_tid_set *set, 866 struct hfi1_filedata *fd) 867 { 868 struct tid_group *grp, *ptr; 869 int i; 870 871 list_for_each_entry_safe(grp, ptr, &set->list, list) { 872 list_del_init(&grp->list); 873 874 for (i = 0; i < grp->size; i++) { 875 if (grp->map & (1 << i)) { 876 u16 rcventry = grp->base + i; 877 struct tid_rb_node *node; 878 879 node = fd->entry_to_rb[rcventry - 880 uctxt->expected_base]; 881 if (!node || node->rcventry != rcventry) 882 continue; 883 884 if (fd->use_mn) 885 mmu_interval_notifier_remove( 886 &node->notifier); 887 cacheless_tid_rb_remove(fd, node); 888 } 889 } 890 } 891 } 892 893 static bool tid_rb_invalidate(struct mmu_interval_notifier *mni, 894 const struct mmu_notifier_range *range, 895 unsigned long cur_seq) 896 { 897 struct tid_rb_node *node = 898 container_of(mni, struct tid_rb_node, notifier); 899 struct hfi1_filedata *fdata = node->fdata; 900 struct hfi1_ctxtdata *uctxt = fdata->uctxt; 901 902 if (node->freed) 903 return true; 904 905 trace_hfi1_exp_tid_inval(uctxt->ctxt, fdata->subctxt, 906 node->notifier.interval_tree.start, 907 node->rcventry, node->npages, node->dma_addr); 908 node->freed = true; 909 910 spin_lock(&fdata->invalid_lock); 911 if (fdata->invalid_tid_idx < uctxt->expected_count) { 912 fdata->invalid_tids[fdata->invalid_tid_idx] = 913 rcventry2tidinfo(node->rcventry - uctxt->expected_base); 914 fdata->invalid_tids[fdata->invalid_tid_idx] |= 915 EXP_TID_SET(LEN, node->npages); 916 if (!fdata->invalid_tid_idx) { 917 unsigned long *ev; 918 919 /* 920 * hfi1_set_uevent_bits() sets a user event flag 921 * for all processes. Because calling into the 922 * driver to process TID cache invalidations is 923 * expensive and TID cache invalidations are 924 * handled on a per-process basis, we can 925 * optimize this to set the flag only for the 926 * process in question. 927 */ 928 ev = uctxt->dd->events + 929 (uctxt_offset(uctxt) + fdata->subctxt); 930 set_bit(_HFI1_EVENT_TID_MMU_NOTIFY_BIT, ev); 931 } 932 fdata->invalid_tid_idx++; 933 } 934 spin_unlock(&fdata->invalid_lock); 935 return true; 936 } 937 938 static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata, 939 struct tid_rb_node *tnode) 940 { 941 u32 base = fdata->uctxt->expected_base; 942 943 fdata->entry_to_rb[tnode->rcventry - base] = NULL; 944 clear_tid_node(fdata, tnode); 945 } 946