1e088a685SYixian Liu /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
2e088a685SYixian Liu /*
3e088a685SYixian Liu * Copyright (c) 2017 Hisilicon Limited.
4e088a685SYixian Liu * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5e088a685SYixian Liu */
6e088a685SYixian Liu
7e088a685SYixian Liu #include <rdma/ib_umem.h>
8e088a685SYixian Liu #include "hns_roce_device.h"
9e088a685SYixian Liu
hns_roce_db_map_user(struct hns_roce_ucontext * context,unsigned long virt,struct hns_roce_db * db)1069e0a42fSLang Cheng int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
11e088a685SYixian Liu struct hns_roce_db *db)
12e088a685SYixian Liu {
1360c3becfSXi Wang unsigned long page_addr = virt & PAGE_MASK;
14e088a685SYixian Liu struct hns_roce_user_db_page *page;
1560c3becfSXi Wang unsigned int offset;
16e088a685SYixian Liu int ret = 0;
17e088a685SYixian Liu
18e088a685SYixian Liu mutex_lock(&context->page_mutex);
19e088a685SYixian Liu
20e088a685SYixian Liu list_for_each_entry(page, &context->page_list, list)
2160c3becfSXi Wang if (page->user_virt == page_addr)
22e088a685SYixian Liu goto found;
23e088a685SYixian Liu
24e088a685SYixian Liu page = kmalloc(sizeof(*page), GFP_KERNEL);
25e088a685SYixian Liu if (!page) {
26e088a685SYixian Liu ret = -ENOMEM;
27e088a685SYixian Liu goto out;
28e088a685SYixian Liu }
29e088a685SYixian Liu
30e088a685SYixian Liu refcount_set(&page->refcount, 1);
3160c3becfSXi Wang page->user_virt = page_addr;
32c320e527SMoni Shoua page->umem = ib_umem_get(context->ibucontext.device, page_addr,
33c320e527SMoni Shoua PAGE_SIZE, 0);
34e088a685SYixian Liu if (IS_ERR(page->umem)) {
35e088a685SYixian Liu ret = PTR_ERR(page->umem);
36e088a685SYixian Liu kfree(page);
37e088a685SYixian Liu goto out;
38e088a685SYixian Liu }
39e088a685SYixian Liu
40e088a685SYixian Liu list_add(&page->list, &context->page_list);
41e088a685SYixian Liu
42e088a685SYixian Liu found:
4360c3becfSXi Wang offset = virt - page_addr;
44*79fbd3e1SMaor Gottlieb db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset;
45*79fbd3e1SMaor Gottlieb db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset;
46e088a685SYixian Liu db->u.user_page = page;
47e088a685SYixian Liu refcount_inc(&page->refcount);
48e088a685SYixian Liu
49e088a685SYixian Liu out:
50e088a685SYixian Liu mutex_unlock(&context->page_mutex);
51e088a685SYixian Liu
52e088a685SYixian Liu return ret;
53e088a685SYixian Liu }
54e088a685SYixian Liu
hns_roce_db_unmap_user(struct hns_roce_ucontext * context,struct hns_roce_db * db)55e088a685SYixian Liu void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
56e088a685SYixian Liu struct hns_roce_db *db)
57e088a685SYixian Liu {
58e088a685SYixian Liu mutex_lock(&context->page_mutex);
59e088a685SYixian Liu
60e088a685SYixian Liu refcount_dec(&db->u.user_page->refcount);
61e088a685SYixian Liu if (refcount_dec_if_one(&db->u.user_page->refcount)) {
62e088a685SYixian Liu list_del(&db->u.user_page->list);
63e088a685SYixian Liu ib_umem_release(db->u.user_page->umem);
64e088a685SYixian Liu kfree(db->u.user_page);
65e088a685SYixian Liu }
66e088a685SYixian Liu
67e088a685SYixian Liu mutex_unlock(&context->page_mutex);
68e088a685SYixian Liu }
69472bc0fbSYixian Liu
hns_roce_alloc_db_pgdir(struct device * dma_device)70472bc0fbSYixian Liu static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(
71472bc0fbSYixian Liu struct device *dma_device)
72472bc0fbSYixian Liu {
73472bc0fbSYixian Liu struct hns_roce_db_pgdir *pgdir;
74472bc0fbSYixian Liu
75472bc0fbSYixian Liu pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);
76472bc0fbSYixian Liu if (!pgdir)
77472bc0fbSYixian Liu return NULL;
78472bc0fbSYixian Liu
792a3d923fSLijun Ou bitmap_fill(pgdir->order1,
802a3d923fSLijun Ou HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
81472bc0fbSYixian Liu pgdir->bits[0] = pgdir->order0;
82472bc0fbSYixian Liu pgdir->bits[1] = pgdir->order1;
83472bc0fbSYixian Liu pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE,
84472bc0fbSYixian Liu &pgdir->db_dma, GFP_KERNEL);
85472bc0fbSYixian Liu if (!pgdir->page) {
86472bc0fbSYixian Liu kfree(pgdir);
87472bc0fbSYixian Liu return NULL;
88472bc0fbSYixian Liu }
89472bc0fbSYixian Liu
90472bc0fbSYixian Liu return pgdir;
91472bc0fbSYixian Liu }
92472bc0fbSYixian Liu
hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir * pgdir,struct hns_roce_db * db,int order)93472bc0fbSYixian Liu static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,
94472bc0fbSYixian Liu struct hns_roce_db *db, int order)
95472bc0fbSYixian Liu {
96dcdc366aSWenpeng Liang unsigned long o;
97dcdc366aSWenpeng Liang unsigned long i;
98472bc0fbSYixian Liu
99472bc0fbSYixian Liu for (o = order; o <= 1; ++o) {
100472bc0fbSYixian Liu i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o);
101472bc0fbSYixian Liu if (i < HNS_ROCE_DB_PER_PAGE >> o)
102472bc0fbSYixian Liu goto found;
103472bc0fbSYixian Liu }
104472bc0fbSYixian Liu
105472bc0fbSYixian Liu return -ENOMEM;
106472bc0fbSYixian Liu
107472bc0fbSYixian Liu found:
108472bc0fbSYixian Liu clear_bit(i, pgdir->bits[o]);
109472bc0fbSYixian Liu
110472bc0fbSYixian Liu i <<= o;
111472bc0fbSYixian Liu
112472bc0fbSYixian Liu if (o > order)
113472bc0fbSYixian Liu set_bit(i ^ 1, pgdir->bits[order]);
114472bc0fbSYixian Liu
115472bc0fbSYixian Liu db->u.pgdir = pgdir;
116472bc0fbSYixian Liu db->index = i;
117472bc0fbSYixian Liu db->db_record = pgdir->page + db->index;
1182a3d923fSLijun Ou db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE;
119472bc0fbSYixian Liu db->order = order;
120472bc0fbSYixian Liu
121472bc0fbSYixian Liu return 0;
122472bc0fbSYixian Liu }
123472bc0fbSYixian Liu
hns_roce_alloc_db(struct hns_roce_dev * hr_dev,struct hns_roce_db * db,int order)124472bc0fbSYixian Liu int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
125472bc0fbSYixian Liu int order)
126472bc0fbSYixian Liu {
127472bc0fbSYixian Liu struct hns_roce_db_pgdir *pgdir;
128472bc0fbSYixian Liu int ret = 0;
129472bc0fbSYixian Liu
130472bc0fbSYixian Liu mutex_lock(&hr_dev->pgdir_mutex);
131472bc0fbSYixian Liu
132472bc0fbSYixian Liu list_for_each_entry(pgdir, &hr_dev->pgdir_list, list)
133472bc0fbSYixian Liu if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order))
134472bc0fbSYixian Liu goto out;
135472bc0fbSYixian Liu
136472bc0fbSYixian Liu pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev);
137472bc0fbSYixian Liu if (!pgdir) {
138472bc0fbSYixian Liu ret = -ENOMEM;
139472bc0fbSYixian Liu goto out;
140472bc0fbSYixian Liu }
141472bc0fbSYixian Liu
142472bc0fbSYixian Liu list_add(&pgdir->list, &hr_dev->pgdir_list);
143472bc0fbSYixian Liu
144472bc0fbSYixian Liu /* This should never fail -- we just allocated an empty page: */
145472bc0fbSYixian Liu WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order));
146472bc0fbSYixian Liu
147472bc0fbSYixian Liu out:
148472bc0fbSYixian Liu mutex_unlock(&hr_dev->pgdir_mutex);
149472bc0fbSYixian Liu
150472bc0fbSYixian Liu return ret;
151472bc0fbSYixian Liu }
152472bc0fbSYixian Liu
hns_roce_free_db(struct hns_roce_dev * hr_dev,struct hns_roce_db * db)153472bc0fbSYixian Liu void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
154472bc0fbSYixian Liu {
155dcdc366aSWenpeng Liang unsigned long o;
156dcdc366aSWenpeng Liang unsigned long i;
157472bc0fbSYixian Liu
158472bc0fbSYixian Liu mutex_lock(&hr_dev->pgdir_mutex);
159472bc0fbSYixian Liu
160472bc0fbSYixian Liu o = db->order;
161472bc0fbSYixian Liu i = db->index;
162472bc0fbSYixian Liu
163472bc0fbSYixian Liu if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
164472bc0fbSYixian Liu clear_bit(i ^ 1, db->u.pgdir->order0);
165472bc0fbSYixian Liu ++o;
166472bc0fbSYixian Liu }
167472bc0fbSYixian Liu
168472bc0fbSYixian Liu i >>= o;
169472bc0fbSYixian Liu set_bit(i, db->u.pgdir->bits[o]);
170472bc0fbSYixian Liu
1712a3d923fSLijun Ou if (bitmap_full(db->u.pgdir->order1,
1722a3d923fSLijun Ou HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {
173472bc0fbSYixian Liu dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page,
174472bc0fbSYixian Liu db->u.pgdir->db_dma);
175472bc0fbSYixian Liu list_del(&db->u.pgdir->list);
176472bc0fbSYixian Liu kfree(db->u.pgdir);
177472bc0fbSYixian Liu }
178472bc0fbSYixian Liu
179472bc0fbSYixian Liu mutex_unlock(&hr_dev->pgdir_mutex);
180472bc0fbSYixian Liu }
181