1e088a685SYixian Liu /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ 2e088a685SYixian Liu /* 3e088a685SYixian Liu * Copyright (c) 2017 Hisilicon Limited. 4e088a685SYixian Liu * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 5e088a685SYixian Liu */ 6e088a685SYixian Liu 7e088a685SYixian Liu #include <linux/platform_device.h> 8e088a685SYixian Liu #include <rdma/ib_umem.h> 9e088a685SYixian Liu #include "hns_roce_device.h" 10e088a685SYixian Liu 11*69e0a42fSLang Cheng int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, 12e088a685SYixian Liu struct hns_roce_db *db) 13e088a685SYixian Liu { 1460c3becfSXi Wang unsigned long page_addr = virt & PAGE_MASK; 15e088a685SYixian Liu struct hns_roce_user_db_page *page; 1660c3becfSXi Wang unsigned int offset; 17e088a685SYixian Liu int ret = 0; 18e088a685SYixian Liu 19e088a685SYixian Liu mutex_lock(&context->page_mutex); 20e088a685SYixian Liu 21e088a685SYixian Liu list_for_each_entry(page, &context->page_list, list) 2260c3becfSXi Wang if (page->user_virt == page_addr) 23e088a685SYixian Liu goto found; 24e088a685SYixian Liu 25e088a685SYixian Liu page = kmalloc(sizeof(*page), GFP_KERNEL); 26e088a685SYixian Liu if (!page) { 27e088a685SYixian Liu ret = -ENOMEM; 28e088a685SYixian Liu goto out; 29e088a685SYixian Liu } 30e088a685SYixian Liu 31e088a685SYixian Liu refcount_set(&page->refcount, 1); 3260c3becfSXi Wang page->user_virt = page_addr; 33c320e527SMoni Shoua page->umem = ib_umem_get(context->ibucontext.device, page_addr, 34c320e527SMoni Shoua PAGE_SIZE, 0); 35e088a685SYixian Liu if (IS_ERR(page->umem)) { 36e088a685SYixian Liu ret = PTR_ERR(page->umem); 37e088a685SYixian Liu kfree(page); 38e088a685SYixian Liu goto out; 39e088a685SYixian Liu } 40e088a685SYixian Liu 41e088a685SYixian Liu list_add(&page->list, &context->page_list); 42e088a685SYixian Liu 43e088a685SYixian Liu found: 4460c3becfSXi Wang offset = virt - page_addr; 4560c3becfSXi Wang db->dma = sg_dma_address(page->umem->sg_head.sgl) + offset; 4660c3becfSXi Wang db->virt_addr = sg_virt(page->umem->sg_head.sgl) + offset; 47e088a685SYixian Liu db->u.user_page = page; 48e088a685SYixian Liu refcount_inc(&page->refcount); 49e088a685SYixian Liu 50e088a685SYixian Liu out: 51e088a685SYixian Liu mutex_unlock(&context->page_mutex); 52e088a685SYixian Liu 53e088a685SYixian Liu return ret; 54e088a685SYixian Liu } 55e088a685SYixian Liu 56e088a685SYixian Liu void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, 57e088a685SYixian Liu struct hns_roce_db *db) 58e088a685SYixian Liu { 59e088a685SYixian Liu mutex_lock(&context->page_mutex); 60e088a685SYixian Liu 61e088a685SYixian Liu refcount_dec(&db->u.user_page->refcount); 62e088a685SYixian Liu if (refcount_dec_if_one(&db->u.user_page->refcount)) { 63e088a685SYixian Liu list_del(&db->u.user_page->list); 64e088a685SYixian Liu ib_umem_release(db->u.user_page->umem); 65e088a685SYixian Liu kfree(db->u.user_page); 66e088a685SYixian Liu } 67e088a685SYixian Liu 68e088a685SYixian Liu mutex_unlock(&context->page_mutex); 69e088a685SYixian Liu } 70472bc0fbSYixian Liu 71472bc0fbSYixian Liu static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( 72472bc0fbSYixian Liu struct device *dma_device) 73472bc0fbSYixian Liu { 74472bc0fbSYixian Liu struct hns_roce_db_pgdir *pgdir; 75472bc0fbSYixian Liu 76472bc0fbSYixian Liu pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); 77472bc0fbSYixian Liu if (!pgdir) 78472bc0fbSYixian Liu return NULL; 79472bc0fbSYixian Liu 802a3d923fSLijun Ou bitmap_fill(pgdir->order1, 812a3d923fSLijun Ou HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); 82472bc0fbSYixian Liu pgdir->bits[0] = pgdir->order0; 83472bc0fbSYixian Liu pgdir->bits[1] = pgdir->order1; 84472bc0fbSYixian Liu pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, 85472bc0fbSYixian Liu &pgdir->db_dma, GFP_KERNEL); 86472bc0fbSYixian Liu if (!pgdir->page) { 87472bc0fbSYixian Liu kfree(pgdir); 88472bc0fbSYixian Liu return NULL; 89472bc0fbSYixian Liu } 90472bc0fbSYixian Liu 91472bc0fbSYixian Liu return pgdir; 92472bc0fbSYixian Liu } 93472bc0fbSYixian Liu 94472bc0fbSYixian Liu static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir, 95472bc0fbSYixian Liu struct hns_roce_db *db, int order) 96472bc0fbSYixian Liu { 97dcdc366aSWenpeng Liang unsigned long o; 98dcdc366aSWenpeng Liang unsigned long i; 99472bc0fbSYixian Liu 100472bc0fbSYixian Liu for (o = order; o <= 1; ++o) { 101472bc0fbSYixian Liu i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o); 102472bc0fbSYixian Liu if (i < HNS_ROCE_DB_PER_PAGE >> o) 103472bc0fbSYixian Liu goto found; 104472bc0fbSYixian Liu } 105472bc0fbSYixian Liu 106472bc0fbSYixian Liu return -ENOMEM; 107472bc0fbSYixian Liu 108472bc0fbSYixian Liu found: 109472bc0fbSYixian Liu clear_bit(i, pgdir->bits[o]); 110472bc0fbSYixian Liu 111472bc0fbSYixian Liu i <<= o; 112472bc0fbSYixian Liu 113472bc0fbSYixian Liu if (o > order) 114472bc0fbSYixian Liu set_bit(i ^ 1, pgdir->bits[order]); 115472bc0fbSYixian Liu 116472bc0fbSYixian Liu db->u.pgdir = pgdir; 117472bc0fbSYixian Liu db->index = i; 118472bc0fbSYixian Liu db->db_record = pgdir->page + db->index; 1192a3d923fSLijun Ou db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE; 120472bc0fbSYixian Liu db->order = order; 121472bc0fbSYixian Liu 122472bc0fbSYixian Liu return 0; 123472bc0fbSYixian Liu } 124472bc0fbSYixian Liu 125472bc0fbSYixian Liu int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, 126472bc0fbSYixian Liu int order) 127472bc0fbSYixian Liu { 128472bc0fbSYixian Liu struct hns_roce_db_pgdir *pgdir; 129472bc0fbSYixian Liu int ret = 0; 130472bc0fbSYixian Liu 131472bc0fbSYixian Liu mutex_lock(&hr_dev->pgdir_mutex); 132472bc0fbSYixian Liu 133472bc0fbSYixian Liu list_for_each_entry(pgdir, &hr_dev->pgdir_list, list) 134472bc0fbSYixian Liu if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order)) 135472bc0fbSYixian Liu goto out; 136472bc0fbSYixian Liu 137472bc0fbSYixian Liu pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev); 138472bc0fbSYixian Liu if (!pgdir) { 139472bc0fbSYixian Liu ret = -ENOMEM; 140472bc0fbSYixian Liu goto out; 141472bc0fbSYixian Liu } 142472bc0fbSYixian Liu 143472bc0fbSYixian Liu list_add(&pgdir->list, &hr_dev->pgdir_list); 144472bc0fbSYixian Liu 145472bc0fbSYixian Liu /* This should never fail -- we just allocated an empty page: */ 146472bc0fbSYixian Liu WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order)); 147472bc0fbSYixian Liu 148472bc0fbSYixian Liu out: 149472bc0fbSYixian Liu mutex_unlock(&hr_dev->pgdir_mutex); 150472bc0fbSYixian Liu 151472bc0fbSYixian Liu return ret; 152472bc0fbSYixian Liu } 153472bc0fbSYixian Liu 154472bc0fbSYixian Liu void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) 155472bc0fbSYixian Liu { 156dcdc366aSWenpeng Liang unsigned long o; 157dcdc366aSWenpeng Liang unsigned long i; 158472bc0fbSYixian Liu 159472bc0fbSYixian Liu mutex_lock(&hr_dev->pgdir_mutex); 160472bc0fbSYixian Liu 161472bc0fbSYixian Liu o = db->order; 162472bc0fbSYixian Liu i = db->index; 163472bc0fbSYixian Liu 164472bc0fbSYixian Liu if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { 165472bc0fbSYixian Liu clear_bit(i ^ 1, db->u.pgdir->order0); 166472bc0fbSYixian Liu ++o; 167472bc0fbSYixian Liu } 168472bc0fbSYixian Liu 169472bc0fbSYixian Liu i >>= o; 170472bc0fbSYixian Liu set_bit(i, db->u.pgdir->bits[o]); 171472bc0fbSYixian Liu 1722a3d923fSLijun Ou if (bitmap_full(db->u.pgdir->order1, 1732a3d923fSLijun Ou HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) { 174472bc0fbSYixian Liu dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, 175472bc0fbSYixian Liu db->u.pgdir->db_dma); 176472bc0fbSYixian Liu list_del(&db->u.pgdir->list); 177472bc0fbSYixian Liu kfree(db->u.pgdir); 178472bc0fbSYixian Liu } 179472bc0fbSYixian Liu 180472bc0fbSYixian Liu mutex_unlock(&hr_dev->pgdir_mutex); 181472bc0fbSYixian Liu } 182