umr.c (29583dfcd2dd72c766422bd05c16f06c6b1fb356) umr.c (02648b4b09d506bd4df2e17bf109c229fc728640)
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */
3
4#include <rdma/ib_umem_odp.h>
5#include "mlx5_ib.h"
6#include "umr.h"
7#include "wr.h"
8

--- 404 unchanged lines hidden (view full) ---

413 if (err)
414 return err;
415
416 mr->access_flags = access_flags;
417 return 0;
418}
419
420#define MLX5_MAX_UMR_CHUNK \
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */
3
4#include <rdma/ib_umem_odp.h>
5#include "mlx5_ib.h"
6#include "umr.h"
7#include "wr.h"
8

--- 404 unchanged lines hidden (view full) ---

413 if (err)
414 return err;
415
416 mr->access_flags = access_flags;
417 return 0;
418}
419
420#define MLX5_MAX_UMR_CHUNK \
421 ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - MLX5_UMR_MTT_ALIGNMENT)
421 ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - MLX5_UMR_FLEX_ALIGNMENT)
422#define MLX5_SPARE_UMR_CHUNK 0x10000
423
424/*
425 * Allocate a temporary buffer to hold the per-page information to transfer to
426 * HW. For efficiency this should be as large as it can be, but buffer
427 * allocation failure is not allowed, so try smaller sizes.
428 */
429static void *mlx5r_umr_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask)
430{
422#define MLX5_SPARE_UMR_CHUNK 0x10000
423
424/*
425 * Allocate a temporary buffer to hold the per-page information to transfer to
426 * HW. For efficiency this should be as large as it can be, but buffer
427 * allocation failure is not allowed, so try smaller sizes.
428 */
429static void *mlx5r_umr_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask)
430{
431 const size_t xlt_chunk_align = MLX5_UMR_MTT_ALIGNMENT / ent_size;
431 const size_t xlt_chunk_align = MLX5_UMR_FLEX_ALIGNMENT / ent_size;
432 size_t size;
433 void *res = NULL;
434
432 size_t size;
433 void *res = NULL;
434
435 static_assert(PAGE_SIZE % MLX5_UMR_MTT_ALIGNMENT == 0);
435 static_assert(PAGE_SIZE % MLX5_UMR_FLEX_ALIGNMENT == 0);
436
437 /*
438 * MLX5_IB_UPD_XLT_ATOMIC doesn't signal an atomic context just that the
439 * allocation can't trigger any kind of reclaim.
440 */
441 might_sleep();
442
443 gfp_mask |= __GFP_ZERO | __GFP_NORETRY;

--- 217 unchanged lines hidden (view full) ---

661
662 if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP))
663 cur_mtt->ptag = 0;
664
665 cur_mtt++;
666 }
667
668 final_size = (void *)cur_mtt - (void *)mtt;
436
437 /*
438 * MLX5_IB_UPD_XLT_ATOMIC doesn't signal an atomic context just that the
439 * allocation can't trigger any kind of reclaim.
440 */
441 might_sleep();
442
443 gfp_mask |= __GFP_ZERO | __GFP_NORETRY;

--- 217 unchanged lines hidden (view full) ---

661
662 if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP))
663 cur_mtt->ptag = 0;
664
665 cur_mtt++;
666 }
667
668 final_size = (void *)cur_mtt - (void *)mtt;
669 sg.length = ALIGN(final_size, MLX5_UMR_MTT_ALIGNMENT);
669 sg.length = ALIGN(final_size, MLX5_UMR_FLEX_ALIGNMENT);
670 memset(cur_mtt, 0, sg.length - final_size);
671 mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags);
672
673 dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE);
674 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true);
675
676err:
677 sg.length = orig_sg_length;

--- 7 unchanged lines hidden (view full) ---

685}
686
687int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
688 int page_shift, int flags)
689{
690 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
691 ? sizeof(struct mlx5_klm)
692 : sizeof(struct mlx5_mtt);
670 memset(cur_mtt, 0, sg.length - final_size);
671 mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags);
672
673 dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE);
674 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true);
675
676err:
677 sg.length = orig_sg_length;

--- 7 unchanged lines hidden (view full) ---

685}
686
687int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
688 int page_shift, int flags)
689{
690 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
691 ? sizeof(struct mlx5_klm)
692 : sizeof(struct mlx5_mtt);
693 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;
693 const int page_align = MLX5_UMR_FLEX_ALIGNMENT / desc_size;
694 struct mlx5_ib_dev *dev = mr_to_mdev(mr);
695 struct device *ddev = &dev->mdev->pdev->dev;
696 const int page_mask = page_align - 1;
697 struct mlx5r_umr_wqe wqe = {};
698 size_t pages_mapped = 0;
699 size_t pages_to_map = 0;
700 size_t size_to_map = 0;
701 size_t orig_sg_length;

--- 4 unchanged lines hidden (view full) ---

706
707 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
708 !umr_can_use_indirect_mkey(dev))
709 return -EPERM;
710
711 if (WARN_ON(!mr->umem->is_odp))
712 return -EINVAL;
713
694 struct mlx5_ib_dev *dev = mr_to_mdev(mr);
695 struct device *ddev = &dev->mdev->pdev->dev;
696 const int page_mask = page_align - 1;
697 struct mlx5r_umr_wqe wqe = {};
698 size_t pages_mapped = 0;
699 size_t pages_to_map = 0;
700 size_t size_to_map = 0;
701 size_t orig_sg_length;

--- 4 unchanged lines hidden (view full) ---

706
707 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
708 !umr_can_use_indirect_mkey(dev))
709 return -EPERM;
710
711 if (WARN_ON(!mr->umem->is_odp))
712 return -EINVAL;
713
714 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
714 /* UMR copies MTTs in units of MLX5_UMR_FLEX_ALIGNMENT bytes,
715 * so we need to align the offset and length accordingly
716 */
717 if (idx & page_mask) {
718 npages += idx & page_mask;
719 idx &= ~page_mask;
720 }
721 pages_to_map = ALIGN(npages, page_align);
722

--- 20 unchanged lines hidden (view full) ---

743 pages_mapped += pages_iter, idx += pages_iter) {
744 npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
745 size_to_map = npages * desc_size;
746 dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
747 DMA_TO_DEVICE);
748 mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
749 dma_sync_single_for_device(ddev, sg.addr, sg.length,
750 DMA_TO_DEVICE);
715 * so we need to align the offset and length accordingly
716 */
717 if (idx & page_mask) {
718 npages += idx & page_mask;
719 idx &= ~page_mask;
720 }
721 pages_to_map = ALIGN(npages, page_align);
722

--- 20 unchanged lines hidden (view full) ---

743 pages_mapped += pages_iter, idx += pages_iter) {
744 npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
745 size_to_map = npages * desc_size;
746 dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
747 DMA_TO_DEVICE);
748 mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
749 dma_sync_single_for_device(ddev, sg.addr, sg.length,
750 DMA_TO_DEVICE);
751 sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT);
751 sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT);
752
753 if (pages_mapped + pages_iter >= pages_to_map)
754 mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags);
755 mlx5r_umr_update_offset(&wqe.ctrl_seg, idx * desc_size);
756 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true);
757 }
758 sg.length = orig_sg_length;
759 mlx5r_umr_unmap_free_xlt(dev, xlt, &sg);
760 return err;
761}
752
753 if (pages_mapped + pages_iter >= pages_to_map)
754 mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags);
755 mlx5r_umr_update_offset(&wqe.ctrl_seg, idx * desc_size);
756 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true);
757 }
758 sg.length = orig_sg_length;
759 mlx5r_umr_unmap_free_xlt(dev, xlt, &sg);
760 return err;
761}