1e65e175bSOded Gabbay // SPDX-License-Identifier: GPL-2.0
2e65e175bSOded Gabbay
3e65e175bSOded Gabbay /*
4e65e175bSOded Gabbay * Copyright 2016-2019 HabanaLabs, Ltd.
5e65e175bSOded Gabbay * All Rights Reserved.
6e65e175bSOded Gabbay */
7e65e175bSOded Gabbay
8e65e175bSOded Gabbay #include <uapi/drm/habanalabs_accel.h>
9e65e175bSOded Gabbay #include "habanalabs.h"
10e65e175bSOded Gabbay
11e65e175bSOded Gabbay #include <linux/mm.h>
12e65e175bSOded Gabbay #include <linux/slab.h>
13e65e175bSOded Gabbay #include <linux/uaccess.h>
14e65e175bSOded Gabbay
15e65e175bSOded Gabbay #define CB_VA_POOL_SIZE (4UL * SZ_1G)
16e65e175bSOded Gabbay
cb_map_mem(struct hl_ctx * ctx,struct hl_cb * cb)17e65e175bSOded Gabbay static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)
18e65e175bSOded Gabbay {
19e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev;
20e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop;
21e65e175bSOded Gabbay u32 page_size = prop->pmmu.page_size;
22e65e175bSOded Gabbay int rc;
23e65e175bSOded Gabbay
24e65e175bSOded Gabbay if (!hdev->supports_cb_mapping) {
25e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev,
26e65e175bSOded Gabbay "Mapping a CB to the device's MMU is not supported\n");
27e65e175bSOded Gabbay return -EINVAL;
28e65e175bSOded Gabbay }
29e65e175bSOded Gabbay
30e65e175bSOded Gabbay if (cb->is_mmu_mapped)
31e65e175bSOded Gabbay return 0;
32e65e175bSOded Gabbay
33e65e175bSOded Gabbay cb->roundup_size = roundup(cb->size, page_size);
34e65e175bSOded Gabbay
35e65e175bSOded Gabbay cb->virtual_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, cb->roundup_size);
36e65e175bSOded Gabbay if (!cb->virtual_addr) {
37e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to allocate device virtual address for CB\n");
38e65e175bSOded Gabbay return -ENOMEM;
39e65e175bSOded Gabbay }
40e65e175bSOded Gabbay
41e65e175bSOded Gabbay mutex_lock(&hdev->mmu_lock);
42*6c31c137SKoby Elbaz
43e65e175bSOded Gabbay rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size);
44e65e175bSOded Gabbay if (rc) {
45e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr);
46*6c31c137SKoby Elbaz goto err_va_pool_free;
47e65e175bSOded Gabbay }
48*6c31c137SKoby Elbaz
49e65e175bSOded Gabbay rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV);
50*6c31c137SKoby Elbaz if (rc)
51*6c31c137SKoby Elbaz goto err_mmu_unmap;
52*6c31c137SKoby Elbaz
53e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock);
54e65e175bSOded Gabbay
55e65e175bSOded Gabbay cb->is_mmu_mapped = true;
56e65e175bSOded Gabbay
57*6c31c137SKoby Elbaz return 0;
58*6c31c137SKoby Elbaz
59*6c31c137SKoby Elbaz err_mmu_unmap:
60*6c31c137SKoby Elbaz hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size);
61*6c31c137SKoby Elbaz err_va_pool_free:
62e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock);
63e65e175bSOded Gabbay gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size);
64*6c31c137SKoby Elbaz
65e65e175bSOded Gabbay return rc;
66e65e175bSOded Gabbay }
67e65e175bSOded Gabbay
cb_unmap_mem(struct hl_ctx * ctx,struct hl_cb * cb)68e65e175bSOded Gabbay static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb)
69e65e175bSOded Gabbay {
70e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev;
71e65e175bSOded Gabbay
72e65e175bSOded Gabbay mutex_lock(&hdev->mmu_lock);
73e65e175bSOded Gabbay hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size);
74e65e175bSOded Gabbay hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
75e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock);
76e65e175bSOded Gabbay
77e65e175bSOded Gabbay gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size);
78e65e175bSOded Gabbay }
79e65e175bSOded Gabbay
cb_fini(struct hl_device * hdev,struct hl_cb * cb)80e65e175bSOded Gabbay static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
81e65e175bSOded Gabbay {
82e65e175bSOded Gabbay if (cb->is_internal)
83e65e175bSOded Gabbay gen_pool_free(hdev->internal_cb_pool,
84e65e175bSOded Gabbay (uintptr_t)cb->kernel_address, cb->size);
85e65e175bSOded Gabbay else
86e65e175bSOded Gabbay hl_asic_dma_free_coherent(hdev, cb->size, cb->kernel_address, cb->bus_address);
87e65e175bSOded Gabbay
88e65e175bSOded Gabbay kfree(cb);
89e65e175bSOded Gabbay }
90e65e175bSOded Gabbay
cb_do_release(struct hl_device * hdev,struct hl_cb * cb)91e65e175bSOded Gabbay static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
92e65e175bSOded Gabbay {
93e65e175bSOded Gabbay if (cb->is_pool) {
94e2a079a2STomer Tayar atomic_set(&cb->is_handle_destroyed, 0);
95e65e175bSOded Gabbay spin_lock(&hdev->cb_pool_lock);
96e65e175bSOded Gabbay list_add(&cb->pool_list, &hdev->cb_pool);
97e65e175bSOded Gabbay spin_unlock(&hdev->cb_pool_lock);
98e65e175bSOded Gabbay } else {
99e65e175bSOded Gabbay cb_fini(hdev, cb);
100e65e175bSOded Gabbay }
101e65e175bSOded Gabbay }
102e65e175bSOded Gabbay
hl_cb_alloc(struct hl_device * hdev,u32 cb_size,int ctx_id,bool internal_cb)103e65e175bSOded Gabbay static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
104e65e175bSOded Gabbay int ctx_id, bool internal_cb)
105e65e175bSOded Gabbay {
106e65e175bSOded Gabbay struct hl_cb *cb = NULL;
107e65e175bSOded Gabbay u32 cb_offset;
108e65e175bSOded Gabbay void *p;
109e65e175bSOded Gabbay
110e65e175bSOded Gabbay /*
111e65e175bSOded Gabbay * We use of GFP_ATOMIC here because this function can be called from
112e65e175bSOded Gabbay * the latency-sensitive code path for command submission. Due to H/W
113e65e175bSOded Gabbay * limitations in some of the ASICs, the kernel must copy the user CB
114e65e175bSOded Gabbay * that is designated for an external queue and actually enqueue
115e65e175bSOded Gabbay * the kernel's copy. Hence, we must never sleep in this code section
116e65e175bSOded Gabbay * and must use GFP_ATOMIC for all memory allocations.
117e65e175bSOded Gabbay */
118e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID && !hdev->disabled)
119e65e175bSOded Gabbay cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
120e65e175bSOded Gabbay
121e65e175bSOded Gabbay if (!cb)
122e65e175bSOded Gabbay cb = kzalloc(sizeof(*cb), GFP_KERNEL);
123e65e175bSOded Gabbay
124e65e175bSOded Gabbay if (!cb)
125e65e175bSOded Gabbay return NULL;
126e65e175bSOded Gabbay
127e65e175bSOded Gabbay if (internal_cb) {
128e65e175bSOded Gabbay p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size);
129e65e175bSOded Gabbay if (!p) {
130e65e175bSOded Gabbay kfree(cb);
131e65e175bSOded Gabbay return NULL;
132e65e175bSOded Gabbay }
133e65e175bSOded Gabbay
134e65e175bSOded Gabbay cb_offset = p - hdev->internal_cb_pool_virt_addr;
135e65e175bSOded Gabbay cb->is_internal = true;
136e65e175bSOded Gabbay cb->bus_address = hdev->internal_cb_va_base + cb_offset;
137e65e175bSOded Gabbay } else if (ctx_id == HL_KERNEL_ASID_ID) {
138e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_ATOMIC);
139e65e175bSOded Gabbay if (!p)
140e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_KERNEL);
141e65e175bSOded Gabbay } else {
142e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address,
143e65e175bSOded Gabbay GFP_USER | __GFP_ZERO);
144e65e175bSOded Gabbay }
145e65e175bSOded Gabbay
146e65e175bSOded Gabbay if (!p) {
147e65e175bSOded Gabbay dev_err(hdev->dev,
148e65e175bSOded Gabbay "failed to allocate %d of dma memory for CB\n",
149e65e175bSOded Gabbay cb_size);
150e65e175bSOded Gabbay kfree(cb);
151e65e175bSOded Gabbay return NULL;
152e65e175bSOded Gabbay }
153e65e175bSOded Gabbay
154e65e175bSOded Gabbay cb->kernel_address = p;
155e65e175bSOded Gabbay cb->size = cb_size;
156e65e175bSOded Gabbay
157e65e175bSOded Gabbay return cb;
158e65e175bSOded Gabbay }
159e65e175bSOded Gabbay
160e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args {
161e65e175bSOded Gabbay struct hl_device *hdev;
162e65e175bSOded Gabbay struct hl_ctx *ctx;
163e65e175bSOded Gabbay u32 cb_size;
164e65e175bSOded Gabbay bool internal_cb;
165e65e175bSOded Gabbay bool map_cb;
166e65e175bSOded Gabbay };
167e65e175bSOded Gabbay
hl_cb_mmap_mem_release(struct hl_mmap_mem_buf * buf)168e65e175bSOded Gabbay static void hl_cb_mmap_mem_release(struct hl_mmap_mem_buf *buf)
169e65e175bSOded Gabbay {
170e65e175bSOded Gabbay struct hl_cb *cb = buf->private;
171e65e175bSOded Gabbay
172e65e175bSOded Gabbay hl_debugfs_remove_cb(cb);
173e65e175bSOded Gabbay
174e65e175bSOded Gabbay if (cb->is_mmu_mapped)
175e65e175bSOded Gabbay cb_unmap_mem(cb->ctx, cb);
176e65e175bSOded Gabbay
177e65e175bSOded Gabbay hl_ctx_put(cb->ctx);
178e65e175bSOded Gabbay
179e65e175bSOded Gabbay cb_do_release(cb->hdev, cb);
180e65e175bSOded Gabbay }
181e65e175bSOded Gabbay
hl_cb_mmap_mem_alloc(struct hl_mmap_mem_buf * buf,gfp_t gfp,void * args)182e65e175bSOded Gabbay static int hl_cb_mmap_mem_alloc(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args)
183e65e175bSOded Gabbay {
184e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args *cb_args = args;
185e65e175bSOded Gabbay struct hl_cb *cb;
186e65e175bSOded Gabbay int rc, ctx_id = cb_args->ctx->asid;
187e65e175bSOded Gabbay bool alloc_new_cb = true;
188e65e175bSOded Gabbay
189e65e175bSOded Gabbay if (!cb_args->internal_cb) {
190e65e175bSOded Gabbay /* Minimum allocation must be PAGE SIZE */
191e65e175bSOded Gabbay if (cb_args->cb_size < PAGE_SIZE)
192e65e175bSOded Gabbay cb_args->cb_size = PAGE_SIZE;
193e65e175bSOded Gabbay
194e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID &&
195e65e175bSOded Gabbay cb_args->cb_size <= cb_args->hdev->asic_prop.cb_pool_cb_size) {
196e65e175bSOded Gabbay
197e65e175bSOded Gabbay spin_lock(&cb_args->hdev->cb_pool_lock);
198e65e175bSOded Gabbay if (!list_empty(&cb_args->hdev->cb_pool)) {
199e65e175bSOded Gabbay cb = list_first_entry(&cb_args->hdev->cb_pool,
200e65e175bSOded Gabbay typeof(*cb), pool_list);
201e65e175bSOded Gabbay list_del(&cb->pool_list);
202e65e175bSOded Gabbay spin_unlock(&cb_args->hdev->cb_pool_lock);
203e65e175bSOded Gabbay alloc_new_cb = false;
204e65e175bSOded Gabbay } else {
205e65e175bSOded Gabbay spin_unlock(&cb_args->hdev->cb_pool_lock);
206e65e175bSOded Gabbay dev_dbg(cb_args->hdev->dev, "CB pool is empty\n");
207e65e175bSOded Gabbay }
208e65e175bSOded Gabbay }
209e65e175bSOded Gabbay }
210e65e175bSOded Gabbay
211e65e175bSOded Gabbay if (alloc_new_cb) {
212e65e175bSOded Gabbay cb = hl_cb_alloc(cb_args->hdev, cb_args->cb_size, ctx_id, cb_args->internal_cb);
213e65e175bSOded Gabbay if (!cb)
214e65e175bSOded Gabbay return -ENOMEM;
215e65e175bSOded Gabbay }
216e65e175bSOded Gabbay
217e65e175bSOded Gabbay cb->hdev = cb_args->hdev;
218e65e175bSOded Gabbay cb->ctx = cb_args->ctx;
219e65e175bSOded Gabbay cb->buf = buf;
220e65e175bSOded Gabbay cb->buf->mappable_size = cb->size;
221e65e175bSOded Gabbay cb->buf->private = cb;
222e65e175bSOded Gabbay
223e65e175bSOded Gabbay hl_ctx_get(cb->ctx);
224e65e175bSOded Gabbay
225e65e175bSOded Gabbay if (cb_args->map_cb) {
226e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID) {
227e65e175bSOded Gabbay dev_err(cb_args->hdev->dev,
228e65e175bSOded Gabbay "CB mapping is not supported for kernel context\n");
229e65e175bSOded Gabbay rc = -EINVAL;
230e65e175bSOded Gabbay goto release_cb;
231e65e175bSOded Gabbay }
232e65e175bSOded Gabbay
233e65e175bSOded Gabbay rc = cb_map_mem(cb_args->ctx, cb);
234e65e175bSOded Gabbay if (rc)
235e65e175bSOded Gabbay goto release_cb;
236e65e175bSOded Gabbay }
237e65e175bSOded Gabbay
238e65e175bSOded Gabbay hl_debugfs_add_cb(cb);
239e65e175bSOded Gabbay
240e65e175bSOded Gabbay return 0;
241e65e175bSOded Gabbay
242e65e175bSOded Gabbay release_cb:
243e65e175bSOded Gabbay hl_ctx_put(cb->ctx);
244e65e175bSOded Gabbay cb_do_release(cb_args->hdev, cb);
245e65e175bSOded Gabbay
246e65e175bSOded Gabbay return rc;
247e65e175bSOded Gabbay }
248e65e175bSOded Gabbay
hl_cb_mmap(struct hl_mmap_mem_buf * buf,struct vm_area_struct * vma,void * args)249e65e175bSOded Gabbay static int hl_cb_mmap(struct hl_mmap_mem_buf *buf,
250e65e175bSOded Gabbay struct vm_area_struct *vma, void *args)
251e65e175bSOded Gabbay {
252e65e175bSOded Gabbay struct hl_cb *cb = buf->private;
253e65e175bSOded Gabbay
254e65e175bSOded Gabbay return cb->hdev->asic_funcs->mmap(cb->hdev, vma, cb->kernel_address,
255e65e175bSOded Gabbay cb->bus_address, cb->size);
256e65e175bSOded Gabbay }
257e65e175bSOded Gabbay
258e65e175bSOded Gabbay static struct hl_mmap_mem_buf_behavior cb_behavior = {
259e65e175bSOded Gabbay .topic = "CB",
260e65e175bSOded Gabbay .mem_id = HL_MMAP_TYPE_CB,
261e65e175bSOded Gabbay .alloc = hl_cb_mmap_mem_alloc,
262e65e175bSOded Gabbay .release = hl_cb_mmap_mem_release,
263e65e175bSOded Gabbay .mmap = hl_cb_mmap,
264e65e175bSOded Gabbay };
265e65e175bSOded Gabbay
hl_cb_create(struct hl_device * hdev,struct hl_mem_mgr * mmg,struct hl_ctx * ctx,u32 cb_size,bool internal_cb,bool map_cb,u64 * handle)266e65e175bSOded Gabbay int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg,
267e65e175bSOded Gabbay struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
268e65e175bSOded Gabbay bool map_cb, u64 *handle)
269e65e175bSOded Gabbay {
270e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args args = {
271e65e175bSOded Gabbay .hdev = hdev,
272e65e175bSOded Gabbay .ctx = ctx,
273e65e175bSOded Gabbay .cb_size = cb_size,
274e65e175bSOded Gabbay .internal_cb = internal_cb,
275e65e175bSOded Gabbay .map_cb = map_cb,
276e65e175bSOded Gabbay };
277e65e175bSOded Gabbay struct hl_mmap_mem_buf *buf;
278e65e175bSOded Gabbay int ctx_id = ctx->asid;
279e65e175bSOded Gabbay
280e65e175bSOded Gabbay if ((hdev->disabled) || (hdev->reset_info.in_reset && (ctx_id != HL_KERNEL_ASID_ID))) {
281e65e175bSOded Gabbay dev_warn_ratelimited(hdev->dev,
282e65e175bSOded Gabbay "Device is disabled or in reset. Can't create new CBs\n");
283e65e175bSOded Gabbay return -EBUSY;
284e65e175bSOded Gabbay }
285e65e175bSOded Gabbay
286e65e175bSOded Gabbay if (cb_size > SZ_2M) {
287e65e175bSOded Gabbay dev_err(hdev->dev, "CB size %d must be less than %d\n",
288e65e175bSOded Gabbay cb_size, SZ_2M);
289e65e175bSOded Gabbay return -EINVAL;
290e65e175bSOded Gabbay }
291e65e175bSOded Gabbay
292e65e175bSOded Gabbay buf = hl_mmap_mem_buf_alloc(
293e65e175bSOded Gabbay mmg, &cb_behavior,
294e65e175bSOded Gabbay ctx_id == HL_KERNEL_ASID_ID ? GFP_ATOMIC : GFP_KERNEL, &args);
295e65e175bSOded Gabbay if (!buf)
296e65e175bSOded Gabbay return -ENOMEM;
297e65e175bSOded Gabbay
298e65e175bSOded Gabbay *handle = buf->handle;
299e65e175bSOded Gabbay
300e65e175bSOded Gabbay return 0;
301e65e175bSOded Gabbay }
302e65e175bSOded Gabbay
hl_cb_destroy(struct hl_mem_mgr * mmg,u64 cb_handle)303e65e175bSOded Gabbay int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle)
304e65e175bSOded Gabbay {
305e65e175bSOded Gabbay struct hl_cb *cb;
306e65e175bSOded Gabbay int rc;
307e65e175bSOded Gabbay
308e65e175bSOded Gabbay cb = hl_cb_get(mmg, cb_handle);
309e65e175bSOded Gabbay if (!cb) {
310e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB destroy failed, no CB was found for handle %#llx\n",
311e65e175bSOded Gabbay cb_handle);
312e2a079a2STomer Tayar return -EINVAL;
313e65e175bSOded Gabbay }
314e65e175bSOded Gabbay
315e2a079a2STomer Tayar /* Make sure that CB handle isn't destroyed more than once */
316e65e175bSOded Gabbay rc = atomic_cmpxchg(&cb->is_handle_destroyed, 0, 1);
317e65e175bSOded Gabbay hl_cb_put(cb);
318e65e175bSOded Gabbay if (rc) {
319e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB destroy failed, handle %#llx was already destroyed\n",
320e65e175bSOded Gabbay cb_handle);
321e2a079a2STomer Tayar return -EINVAL;
322e65e175bSOded Gabbay }
323e65e175bSOded Gabbay
324e65e175bSOded Gabbay rc = hl_mmap_mem_buf_put_handle(mmg, cb_handle);
325e65e175bSOded Gabbay if (rc < 0)
326e65e175bSOded Gabbay return rc; /* Invalid handle */
327e65e175bSOded Gabbay
328e65e175bSOded Gabbay if (rc == 0)
329e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB 0x%llx is destroyed while still in use\n", cb_handle);
330e65e175bSOded Gabbay
331e65e175bSOded Gabbay return 0;
332e65e175bSOded Gabbay }
333e65e175bSOded Gabbay
hl_cb_info(struct hl_mem_mgr * mmg,u64 handle,u32 flags,u32 * usage_cnt,u64 * device_va)334e65e175bSOded Gabbay static int hl_cb_info(struct hl_mem_mgr *mmg,
335e65e175bSOded Gabbay u64 handle, u32 flags, u32 *usage_cnt, u64 *device_va)
336e65e175bSOded Gabbay {
337e65e175bSOded Gabbay struct hl_cb *cb;
338e65e175bSOded Gabbay int rc = 0;
339e65e175bSOded Gabbay
340e65e175bSOded Gabbay cb = hl_cb_get(mmg, handle);
341e65e175bSOded Gabbay if (!cb) {
342e65e175bSOded Gabbay dev_err(mmg->dev,
343e65e175bSOded Gabbay "CB info failed, no match to handle 0x%llx\n", handle);
344e65e175bSOded Gabbay return -EINVAL;
345e65e175bSOded Gabbay }
346e65e175bSOded Gabbay
347e65e175bSOded Gabbay if (flags & HL_CB_FLAGS_GET_DEVICE_VA) {
348e65e175bSOded Gabbay if (cb->is_mmu_mapped) {
349e65e175bSOded Gabbay *device_va = cb->virtual_addr;
350e65e175bSOded Gabbay } else {
351e65e175bSOded Gabbay dev_err(mmg->dev, "CB is not mapped to the device's MMU\n");
352e65e175bSOded Gabbay rc = -EINVAL;
353e65e175bSOded Gabbay goto out;
354e65e175bSOded Gabbay }
355e65e175bSOded Gabbay } else {
356e65e175bSOded Gabbay *usage_cnt = atomic_read(&cb->cs_cnt);
357e65e175bSOded Gabbay }
358e65e175bSOded Gabbay
359e65e175bSOded Gabbay out:
360e65e175bSOded Gabbay hl_cb_put(cb);
361e65e175bSOded Gabbay return rc;
362e65e175bSOded Gabbay }
363e65e175bSOded Gabbay
hl_cb_ioctl(struct hl_fpriv * hpriv,void * data)364e65e175bSOded Gabbay int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
365e65e175bSOded Gabbay {
366e65e175bSOded Gabbay union hl_cb_args *args = data;
367e65e175bSOded Gabbay struct hl_device *hdev = hpriv->hdev;
368e65e175bSOded Gabbay u64 handle = 0, device_va = 0;
369e65e175bSOded Gabbay enum hl_device_status status;
370e65e175bSOded Gabbay u32 usage_cnt = 0;
371e65e175bSOded Gabbay int rc;
372e65e175bSOded Gabbay
373e65e175bSOded Gabbay if (!hl_device_operational(hdev, &status)) {
374e1e8e747SOded Gabbay dev_dbg_ratelimited(hdev->dev,
375e65e175bSOded Gabbay "Device is %s. Can't execute CB IOCTL\n",
376e65e175bSOded Gabbay hdev->status[status]);
377e65e175bSOded Gabbay return -EBUSY;
378e65e175bSOded Gabbay }
379e65e175bSOded Gabbay
380e65e175bSOded Gabbay switch (args->in.op) {
381e65e175bSOded Gabbay case HL_CB_OP_CREATE:
382e65e175bSOded Gabbay if (args->in.cb_size > HL_MAX_CB_SIZE) {
383e65e175bSOded Gabbay dev_err(hdev->dev,
384e65e175bSOded Gabbay "User requested CB size %d must be less than %d\n",
385e65e175bSOded Gabbay args->in.cb_size, HL_MAX_CB_SIZE);
386e65e175bSOded Gabbay rc = -EINVAL;
387e65e175bSOded Gabbay } else {
388e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hpriv->mem_mgr, hpriv->ctx,
389e65e175bSOded Gabbay args->in.cb_size, false,
390e65e175bSOded Gabbay !!(args->in.flags & HL_CB_FLAGS_MAP),
391e65e175bSOded Gabbay &handle);
392e65e175bSOded Gabbay }
393e65e175bSOded Gabbay
394e65e175bSOded Gabbay memset(args, 0, sizeof(*args));
395e65e175bSOded Gabbay args->out.cb_handle = handle;
396e65e175bSOded Gabbay break;
397e65e175bSOded Gabbay
398e65e175bSOded Gabbay case HL_CB_OP_DESTROY:
399e65e175bSOded Gabbay rc = hl_cb_destroy(&hpriv->mem_mgr,
400e65e175bSOded Gabbay args->in.cb_handle);
401e65e175bSOded Gabbay break;
402e65e175bSOded Gabbay
403e65e175bSOded Gabbay case HL_CB_OP_INFO:
404e65e175bSOded Gabbay rc = hl_cb_info(&hpriv->mem_mgr, args->in.cb_handle,
405e65e175bSOded Gabbay args->in.flags,
406e65e175bSOded Gabbay &usage_cnt,
407e65e175bSOded Gabbay &device_va);
408e65e175bSOded Gabbay if (rc)
409e65e175bSOded Gabbay break;
410e65e175bSOded Gabbay
411e65e175bSOded Gabbay memset(&args->out, 0, sizeof(args->out));
412e65e175bSOded Gabbay
413e65e175bSOded Gabbay if (args->in.flags & HL_CB_FLAGS_GET_DEVICE_VA)
414e65e175bSOded Gabbay args->out.device_va = device_va;
415e65e175bSOded Gabbay else
416e65e175bSOded Gabbay args->out.usage_cnt = usage_cnt;
417e65e175bSOded Gabbay break;
418e65e175bSOded Gabbay
419e65e175bSOded Gabbay default:
420e65e175bSOded Gabbay rc = -EINVAL;
421e65e175bSOded Gabbay break;
422e65e175bSOded Gabbay }
423e65e175bSOded Gabbay
424e65e175bSOded Gabbay return rc;
425e65e175bSOded Gabbay }
426e65e175bSOded Gabbay
hl_cb_get(struct hl_mem_mgr * mmg,u64 handle)427e65e175bSOded Gabbay struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle)
428e65e175bSOded Gabbay {
429e65e175bSOded Gabbay struct hl_mmap_mem_buf *buf;
430e65e175bSOded Gabbay
431e65e175bSOded Gabbay buf = hl_mmap_mem_buf_get(mmg, handle);
432e65e175bSOded Gabbay if (!buf)
433e65e175bSOded Gabbay return NULL;
434e65e175bSOded Gabbay return buf->private;
435e65e175bSOded Gabbay
436e65e175bSOded Gabbay }
437e65e175bSOded Gabbay
hl_cb_put(struct hl_cb * cb)438e65e175bSOded Gabbay void hl_cb_put(struct hl_cb *cb)
439e65e175bSOded Gabbay {
440e65e175bSOded Gabbay hl_mmap_mem_buf_put(cb->buf);
441e65e175bSOded Gabbay }
442e65e175bSOded Gabbay
hl_cb_kernel_create(struct hl_device * hdev,u32 cb_size,bool internal_cb)443e65e175bSOded Gabbay struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
444e65e175bSOded Gabbay bool internal_cb)
445e65e175bSOded Gabbay {
446e65e175bSOded Gabbay u64 cb_handle;
447e65e175bSOded Gabbay struct hl_cb *cb;
448e65e175bSOded Gabbay int rc;
449e65e175bSOded Gabbay
450e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, cb_size,
451e65e175bSOded Gabbay internal_cb, false, &cb_handle);
452e65e175bSOded Gabbay if (rc) {
453e65e175bSOded Gabbay dev_err(hdev->dev,
454e65e175bSOded Gabbay "Failed to allocate CB for the kernel driver %d\n", rc);
455e65e175bSOded Gabbay return NULL;
456e65e175bSOded Gabbay }
457e65e175bSOded Gabbay
458e65e175bSOded Gabbay cb = hl_cb_get(&hdev->kernel_mem_mgr, cb_handle);
459e65e175bSOded Gabbay /* hl_cb_get should never fail here */
460e65e175bSOded Gabbay if (!cb) {
461e65e175bSOded Gabbay dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n",
462e65e175bSOded Gabbay (u32) cb_handle);
463e65e175bSOded Gabbay goto destroy_cb;
464e65e175bSOded Gabbay }
465e65e175bSOded Gabbay
466e65e175bSOded Gabbay return cb;
467e65e175bSOded Gabbay
468e65e175bSOded Gabbay destroy_cb:
469e65e175bSOded Gabbay hl_cb_destroy(&hdev->kernel_mem_mgr, cb_handle);
470e65e175bSOded Gabbay
471e65e175bSOded Gabbay return NULL;
472e65e175bSOded Gabbay }
473e65e175bSOded Gabbay
hl_cb_pool_init(struct hl_device * hdev)474e65e175bSOded Gabbay int hl_cb_pool_init(struct hl_device *hdev)
475e65e175bSOded Gabbay {
476e65e175bSOded Gabbay struct hl_cb *cb;
477e65e175bSOded Gabbay int i;
478e65e175bSOded Gabbay
479e65e175bSOded Gabbay INIT_LIST_HEAD(&hdev->cb_pool);
480e65e175bSOded Gabbay spin_lock_init(&hdev->cb_pool_lock);
481e65e175bSOded Gabbay
482e65e175bSOded Gabbay for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
483e65e175bSOded Gabbay cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
484e65e175bSOded Gabbay HL_KERNEL_ASID_ID, false);
485e65e175bSOded Gabbay if (cb) {
486e65e175bSOded Gabbay cb->is_pool = true;
487e65e175bSOded Gabbay list_add(&cb->pool_list, &hdev->cb_pool);
488e65e175bSOded Gabbay } else {
489e65e175bSOded Gabbay hl_cb_pool_fini(hdev);
490e65e175bSOded Gabbay return -ENOMEM;
491e65e175bSOded Gabbay }
492e65e175bSOded Gabbay }
493e65e175bSOded Gabbay
494e65e175bSOded Gabbay return 0;
495e65e175bSOded Gabbay }
496e65e175bSOded Gabbay
hl_cb_pool_fini(struct hl_device * hdev)497e65e175bSOded Gabbay int hl_cb_pool_fini(struct hl_device *hdev)
498e65e175bSOded Gabbay {
499e65e175bSOded Gabbay struct hl_cb *cb, *tmp;
500e65e175bSOded Gabbay
501e65e175bSOded Gabbay list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
502e65e175bSOded Gabbay list_del(&cb->pool_list);
503e65e175bSOded Gabbay cb_fini(hdev, cb);
504e65e175bSOded Gabbay }
505e65e175bSOded Gabbay
506e65e175bSOded Gabbay return 0;
507e65e175bSOded Gabbay }
508e65e175bSOded Gabbay
hl_cb_va_pool_init(struct hl_ctx * ctx)509e65e175bSOded Gabbay int hl_cb_va_pool_init(struct hl_ctx *ctx)
510e65e175bSOded Gabbay {
511e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev;
512e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop;
513e65e175bSOded Gabbay int rc;
514e65e175bSOded Gabbay
515e65e175bSOded Gabbay if (!hdev->supports_cb_mapping)
516e65e175bSOded Gabbay return 0;
517e65e175bSOded Gabbay
518e65e175bSOded Gabbay ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1);
519e65e175bSOded Gabbay if (!ctx->cb_va_pool) {
520e65e175bSOded Gabbay dev_err(hdev->dev,
521e65e175bSOded Gabbay "Failed to create VA gen pool for CB mapping\n");
522e65e175bSOded Gabbay return -ENOMEM;
523e65e175bSOded Gabbay }
524e65e175bSOded Gabbay
525e65e175bSOded Gabbay ctx->cb_va_pool_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST,
526e65e175bSOded Gabbay CB_VA_POOL_SIZE, HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
527e65e175bSOded Gabbay if (!ctx->cb_va_pool_base) {
528e65e175bSOded Gabbay rc = -ENOMEM;
529e65e175bSOded Gabbay goto err_pool_destroy;
530e65e175bSOded Gabbay }
531e65e175bSOded Gabbay rc = gen_pool_add(ctx->cb_va_pool, ctx->cb_va_pool_base, CB_VA_POOL_SIZE, -1);
532e65e175bSOded Gabbay if (rc) {
533e65e175bSOded Gabbay dev_err(hdev->dev,
534e65e175bSOded Gabbay "Failed to add memory to VA gen pool for CB mapping\n");
535e65e175bSOded Gabbay goto err_unreserve_va_block;
536e65e175bSOded Gabbay }
537e65e175bSOded Gabbay
538e65e175bSOded Gabbay return 0;
539e65e175bSOded Gabbay
540e65e175bSOded Gabbay err_unreserve_va_block:
541e65e175bSOded Gabbay hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE);
542e65e175bSOded Gabbay err_pool_destroy:
543e65e175bSOded Gabbay gen_pool_destroy(ctx->cb_va_pool);
544e65e175bSOded Gabbay
545e65e175bSOded Gabbay return rc;
546e65e175bSOded Gabbay }
547e65e175bSOded Gabbay
hl_cb_va_pool_fini(struct hl_ctx * ctx)548e65e175bSOded Gabbay void hl_cb_va_pool_fini(struct hl_ctx *ctx)
549e65e175bSOded Gabbay {
550e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev;
551e65e175bSOded Gabbay
552e65e175bSOded Gabbay if (!hdev->supports_cb_mapping)
553e65e175bSOded Gabbay return;
554e65e175bSOded Gabbay
555e65e175bSOded Gabbay gen_pool_destroy(ctx->cb_va_pool);
556e65e175bSOded Gabbay hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE);
557e65e175bSOded Gabbay }
558