1e65e175bSOded Gabbay // SPDX-License-Identifier: GPL-2.0 2e65e175bSOded Gabbay 3e65e175bSOded Gabbay /* 4e65e175bSOded Gabbay * Copyright 2016-2019 HabanaLabs, Ltd. 5e65e175bSOded Gabbay * All Rights Reserved. 6e65e175bSOded Gabbay */ 7e65e175bSOded Gabbay 8e65e175bSOded Gabbay #include <uapi/drm/habanalabs_accel.h> 9e65e175bSOded Gabbay #include "habanalabs.h" 10e65e175bSOded Gabbay 11e65e175bSOded Gabbay #include <linux/mm.h> 12e65e175bSOded Gabbay #include <linux/slab.h> 13e65e175bSOded Gabbay #include <linux/uaccess.h> 14e65e175bSOded Gabbay 15e65e175bSOded Gabbay #define CB_VA_POOL_SIZE (4UL * SZ_1G) 16e65e175bSOded Gabbay 17e65e175bSOded Gabbay static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb) 18e65e175bSOded Gabbay { 19e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 20e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 21e65e175bSOded Gabbay u32 page_size = prop->pmmu.page_size; 22e65e175bSOded Gabbay int rc; 23e65e175bSOded Gabbay 24e65e175bSOded Gabbay if (!hdev->supports_cb_mapping) { 25e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, 26e65e175bSOded Gabbay "Mapping a CB to the device's MMU is not supported\n"); 27e65e175bSOded Gabbay return -EINVAL; 28e65e175bSOded Gabbay } 29e65e175bSOded Gabbay 30e65e175bSOded Gabbay if (!hdev->mmu_enable) { 31e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, 32e65e175bSOded Gabbay "Cannot map CB because MMU is disabled\n"); 33e65e175bSOded Gabbay return -EINVAL; 34e65e175bSOded Gabbay } 35e65e175bSOded Gabbay 36e65e175bSOded Gabbay if (cb->is_mmu_mapped) 37e65e175bSOded Gabbay return 0; 38e65e175bSOded Gabbay 39e65e175bSOded Gabbay cb->roundup_size = roundup(cb->size, page_size); 40e65e175bSOded Gabbay 41e65e175bSOded Gabbay cb->virtual_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, cb->roundup_size); 42e65e175bSOded Gabbay if (!cb->virtual_addr) { 43e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to allocate device virtual address for CB\n"); 44e65e175bSOded Gabbay return -ENOMEM; 45e65e175bSOded Gabbay } 46e65e175bSOded Gabbay 47e65e175bSOded Gabbay mutex_lock(&hdev->mmu_lock); 48*6c31c137SKoby Elbaz 49e65e175bSOded Gabbay rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size); 50e65e175bSOded Gabbay if (rc) { 51e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr); 52*6c31c137SKoby Elbaz goto err_va_pool_free; 53e65e175bSOded Gabbay } 54*6c31c137SKoby Elbaz 55e65e175bSOded Gabbay rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV); 56*6c31c137SKoby Elbaz if (rc) 57*6c31c137SKoby Elbaz goto err_mmu_unmap; 58*6c31c137SKoby Elbaz 59e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock); 60e65e175bSOded Gabbay 61e65e175bSOded Gabbay cb->is_mmu_mapped = true; 62e65e175bSOded Gabbay 63*6c31c137SKoby Elbaz return 0; 64*6c31c137SKoby Elbaz 65*6c31c137SKoby Elbaz err_mmu_unmap: 66*6c31c137SKoby Elbaz hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); 67*6c31c137SKoby Elbaz err_va_pool_free: 68e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock); 69e65e175bSOded Gabbay gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 70*6c31c137SKoby Elbaz 71e65e175bSOded Gabbay return rc; 72e65e175bSOded Gabbay } 73e65e175bSOded Gabbay 74e65e175bSOded Gabbay static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb) 75e65e175bSOded Gabbay { 76e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 77e65e175bSOded Gabbay 78e65e175bSOded Gabbay mutex_lock(&hdev->mmu_lock); 79e65e175bSOded Gabbay hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); 80e65e175bSOded Gabbay hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 81e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock); 82e65e175bSOded Gabbay 83e65e175bSOded Gabbay gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 84e65e175bSOded Gabbay } 85e65e175bSOded Gabbay 86e65e175bSOded Gabbay static void cb_fini(struct hl_device *hdev, struct hl_cb *cb) 87e65e175bSOded Gabbay { 88e65e175bSOded Gabbay if (cb->is_internal) 89e65e175bSOded Gabbay gen_pool_free(hdev->internal_cb_pool, 90e65e175bSOded Gabbay (uintptr_t)cb->kernel_address, cb->size); 91e65e175bSOded Gabbay else 92e65e175bSOded Gabbay hl_asic_dma_free_coherent(hdev, cb->size, cb->kernel_address, cb->bus_address); 93e65e175bSOded Gabbay 94e65e175bSOded Gabbay kfree(cb); 95e65e175bSOded Gabbay } 96e65e175bSOded Gabbay 97e65e175bSOded Gabbay static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb) 98e65e175bSOded Gabbay { 99e65e175bSOded Gabbay if (cb->is_pool) { 100e2a079a2STomer Tayar atomic_set(&cb->is_handle_destroyed, 0); 101e65e175bSOded Gabbay spin_lock(&hdev->cb_pool_lock); 102e65e175bSOded Gabbay list_add(&cb->pool_list, &hdev->cb_pool); 103e65e175bSOded Gabbay spin_unlock(&hdev->cb_pool_lock); 104e65e175bSOded Gabbay } else { 105e65e175bSOded Gabbay cb_fini(hdev, cb); 106e65e175bSOded Gabbay } 107e65e175bSOded Gabbay } 108e65e175bSOded Gabbay 109e65e175bSOded Gabbay static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size, 110e65e175bSOded Gabbay int ctx_id, bool internal_cb) 111e65e175bSOded Gabbay { 112e65e175bSOded Gabbay struct hl_cb *cb = NULL; 113e65e175bSOded Gabbay u32 cb_offset; 114e65e175bSOded Gabbay void *p; 115e65e175bSOded Gabbay 116e65e175bSOded Gabbay /* 117e65e175bSOded Gabbay * We use of GFP_ATOMIC here because this function can be called from 118e65e175bSOded Gabbay * the latency-sensitive code path for command submission. Due to H/W 119e65e175bSOded Gabbay * limitations in some of the ASICs, the kernel must copy the user CB 120e65e175bSOded Gabbay * that is designated for an external queue and actually enqueue 121e65e175bSOded Gabbay * the kernel's copy. Hence, we must never sleep in this code section 122e65e175bSOded Gabbay * and must use GFP_ATOMIC for all memory allocations. 123e65e175bSOded Gabbay */ 124e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID && !hdev->disabled) 125e65e175bSOded Gabbay cb = kzalloc(sizeof(*cb), GFP_ATOMIC); 126e65e175bSOded Gabbay 127e65e175bSOded Gabbay if (!cb) 128e65e175bSOded Gabbay cb = kzalloc(sizeof(*cb), GFP_KERNEL); 129e65e175bSOded Gabbay 130e65e175bSOded Gabbay if (!cb) 131e65e175bSOded Gabbay return NULL; 132e65e175bSOded Gabbay 133e65e175bSOded Gabbay if (internal_cb) { 134e65e175bSOded Gabbay p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size); 135e65e175bSOded Gabbay if (!p) { 136e65e175bSOded Gabbay kfree(cb); 137e65e175bSOded Gabbay return NULL; 138e65e175bSOded Gabbay } 139e65e175bSOded Gabbay 140e65e175bSOded Gabbay cb_offset = p - hdev->internal_cb_pool_virt_addr; 141e65e175bSOded Gabbay cb->is_internal = true; 142e65e175bSOded Gabbay cb->bus_address = hdev->internal_cb_va_base + cb_offset; 143e65e175bSOded Gabbay } else if (ctx_id == HL_KERNEL_ASID_ID) { 144e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_ATOMIC); 145e65e175bSOded Gabbay if (!p) 146e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_KERNEL); 147e65e175bSOded Gabbay } else { 148e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, 149e65e175bSOded Gabbay GFP_USER | __GFP_ZERO); 150e65e175bSOded Gabbay } 151e65e175bSOded Gabbay 152e65e175bSOded Gabbay if (!p) { 153e65e175bSOded Gabbay dev_err(hdev->dev, 154e65e175bSOded Gabbay "failed to allocate %d of dma memory for CB\n", 155e65e175bSOded Gabbay cb_size); 156e65e175bSOded Gabbay kfree(cb); 157e65e175bSOded Gabbay return NULL; 158e65e175bSOded Gabbay } 159e65e175bSOded Gabbay 160e65e175bSOded Gabbay cb->kernel_address = p; 161e65e175bSOded Gabbay cb->size = cb_size; 162e65e175bSOded Gabbay 163e65e175bSOded Gabbay return cb; 164e65e175bSOded Gabbay } 165e65e175bSOded Gabbay 166e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args { 167e65e175bSOded Gabbay struct hl_device *hdev; 168e65e175bSOded Gabbay struct hl_ctx *ctx; 169e65e175bSOded Gabbay u32 cb_size; 170e65e175bSOded Gabbay bool internal_cb; 171e65e175bSOded Gabbay bool map_cb; 172e65e175bSOded Gabbay }; 173e65e175bSOded Gabbay 174e65e175bSOded Gabbay static void hl_cb_mmap_mem_release(struct hl_mmap_mem_buf *buf) 175e65e175bSOded Gabbay { 176e65e175bSOded Gabbay struct hl_cb *cb = buf->private; 177e65e175bSOded Gabbay 178e65e175bSOded Gabbay hl_debugfs_remove_cb(cb); 179e65e175bSOded Gabbay 180e65e175bSOded Gabbay if (cb->is_mmu_mapped) 181e65e175bSOded Gabbay cb_unmap_mem(cb->ctx, cb); 182e65e175bSOded Gabbay 183e65e175bSOded Gabbay hl_ctx_put(cb->ctx); 184e65e175bSOded Gabbay 185e65e175bSOded Gabbay cb_do_release(cb->hdev, cb); 186e65e175bSOded Gabbay } 187e65e175bSOded Gabbay 188e65e175bSOded Gabbay static int hl_cb_mmap_mem_alloc(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args) 189e65e175bSOded Gabbay { 190e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args *cb_args = args; 191e65e175bSOded Gabbay struct hl_cb *cb; 192e65e175bSOded Gabbay int rc, ctx_id = cb_args->ctx->asid; 193e65e175bSOded Gabbay bool alloc_new_cb = true; 194e65e175bSOded Gabbay 195e65e175bSOded Gabbay if (!cb_args->internal_cb) { 196e65e175bSOded Gabbay /* Minimum allocation must be PAGE SIZE */ 197e65e175bSOded Gabbay if (cb_args->cb_size < PAGE_SIZE) 198e65e175bSOded Gabbay cb_args->cb_size = PAGE_SIZE; 199e65e175bSOded Gabbay 200e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID && 201e65e175bSOded Gabbay cb_args->cb_size <= cb_args->hdev->asic_prop.cb_pool_cb_size) { 202e65e175bSOded Gabbay 203e65e175bSOded Gabbay spin_lock(&cb_args->hdev->cb_pool_lock); 204e65e175bSOded Gabbay if (!list_empty(&cb_args->hdev->cb_pool)) { 205e65e175bSOded Gabbay cb = list_first_entry(&cb_args->hdev->cb_pool, 206e65e175bSOded Gabbay typeof(*cb), pool_list); 207e65e175bSOded Gabbay list_del(&cb->pool_list); 208e65e175bSOded Gabbay spin_unlock(&cb_args->hdev->cb_pool_lock); 209e65e175bSOded Gabbay alloc_new_cb = false; 210e65e175bSOded Gabbay } else { 211e65e175bSOded Gabbay spin_unlock(&cb_args->hdev->cb_pool_lock); 212e65e175bSOded Gabbay dev_dbg(cb_args->hdev->dev, "CB pool is empty\n"); 213e65e175bSOded Gabbay } 214e65e175bSOded Gabbay } 215e65e175bSOded Gabbay } 216e65e175bSOded Gabbay 217e65e175bSOded Gabbay if (alloc_new_cb) { 218e65e175bSOded Gabbay cb = hl_cb_alloc(cb_args->hdev, cb_args->cb_size, ctx_id, cb_args->internal_cb); 219e65e175bSOded Gabbay if (!cb) 220e65e175bSOded Gabbay return -ENOMEM; 221e65e175bSOded Gabbay } 222e65e175bSOded Gabbay 223e65e175bSOded Gabbay cb->hdev = cb_args->hdev; 224e65e175bSOded Gabbay cb->ctx = cb_args->ctx; 225e65e175bSOded Gabbay cb->buf = buf; 226e65e175bSOded Gabbay cb->buf->mappable_size = cb->size; 227e65e175bSOded Gabbay cb->buf->private = cb; 228e65e175bSOded Gabbay 229e65e175bSOded Gabbay hl_ctx_get(cb->ctx); 230e65e175bSOded Gabbay 231e65e175bSOded Gabbay if (cb_args->map_cb) { 232e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID) { 233e65e175bSOded Gabbay dev_err(cb_args->hdev->dev, 234e65e175bSOded Gabbay "CB mapping is not supported for kernel context\n"); 235e65e175bSOded Gabbay rc = -EINVAL; 236e65e175bSOded Gabbay goto release_cb; 237e65e175bSOded Gabbay } 238e65e175bSOded Gabbay 239e65e175bSOded Gabbay rc = cb_map_mem(cb_args->ctx, cb); 240e65e175bSOded Gabbay if (rc) 241e65e175bSOded Gabbay goto release_cb; 242e65e175bSOded Gabbay } 243e65e175bSOded Gabbay 244e65e175bSOded Gabbay hl_debugfs_add_cb(cb); 245e65e175bSOded Gabbay 246e65e175bSOded Gabbay return 0; 247e65e175bSOded Gabbay 248e65e175bSOded Gabbay release_cb: 249e65e175bSOded Gabbay hl_ctx_put(cb->ctx); 250e65e175bSOded Gabbay cb_do_release(cb_args->hdev, cb); 251e65e175bSOded Gabbay 252e65e175bSOded Gabbay return rc; 253e65e175bSOded Gabbay } 254e65e175bSOded Gabbay 255e65e175bSOded Gabbay static int hl_cb_mmap(struct hl_mmap_mem_buf *buf, 256e65e175bSOded Gabbay struct vm_area_struct *vma, void *args) 257e65e175bSOded Gabbay { 258e65e175bSOded Gabbay struct hl_cb *cb = buf->private; 259e65e175bSOded Gabbay 260e65e175bSOded Gabbay return cb->hdev->asic_funcs->mmap(cb->hdev, vma, cb->kernel_address, 261e65e175bSOded Gabbay cb->bus_address, cb->size); 262e65e175bSOded Gabbay } 263e65e175bSOded Gabbay 264e65e175bSOded Gabbay static struct hl_mmap_mem_buf_behavior cb_behavior = { 265e65e175bSOded Gabbay .topic = "CB", 266e65e175bSOded Gabbay .mem_id = HL_MMAP_TYPE_CB, 267e65e175bSOded Gabbay .alloc = hl_cb_mmap_mem_alloc, 268e65e175bSOded Gabbay .release = hl_cb_mmap_mem_release, 269e65e175bSOded Gabbay .mmap = hl_cb_mmap, 270e65e175bSOded Gabbay }; 271e65e175bSOded Gabbay 272e65e175bSOded Gabbay int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, 273e65e175bSOded Gabbay struct hl_ctx *ctx, u32 cb_size, bool internal_cb, 274e65e175bSOded Gabbay bool map_cb, u64 *handle) 275e65e175bSOded Gabbay { 276e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args args = { 277e65e175bSOded Gabbay .hdev = hdev, 278e65e175bSOded Gabbay .ctx = ctx, 279e65e175bSOded Gabbay .cb_size = cb_size, 280e65e175bSOded Gabbay .internal_cb = internal_cb, 281e65e175bSOded Gabbay .map_cb = map_cb, 282e65e175bSOded Gabbay }; 283e65e175bSOded Gabbay struct hl_mmap_mem_buf *buf; 284e65e175bSOded Gabbay int ctx_id = ctx->asid; 285e65e175bSOded Gabbay 286e65e175bSOded Gabbay if ((hdev->disabled) || (hdev->reset_info.in_reset && (ctx_id != HL_KERNEL_ASID_ID))) { 287e65e175bSOded Gabbay dev_warn_ratelimited(hdev->dev, 288e65e175bSOded Gabbay "Device is disabled or in reset. Can't create new CBs\n"); 289e65e175bSOded Gabbay return -EBUSY; 290e65e175bSOded Gabbay } 291e65e175bSOded Gabbay 292e65e175bSOded Gabbay if (cb_size > SZ_2M) { 293e65e175bSOded Gabbay dev_err(hdev->dev, "CB size %d must be less than %d\n", 294e65e175bSOded Gabbay cb_size, SZ_2M); 295e65e175bSOded Gabbay return -EINVAL; 296e65e175bSOded Gabbay } 297e65e175bSOded Gabbay 298e65e175bSOded Gabbay buf = hl_mmap_mem_buf_alloc( 299e65e175bSOded Gabbay mmg, &cb_behavior, 300e65e175bSOded Gabbay ctx_id == HL_KERNEL_ASID_ID ? GFP_ATOMIC : GFP_KERNEL, &args); 301e65e175bSOded Gabbay if (!buf) 302e65e175bSOded Gabbay return -ENOMEM; 303e65e175bSOded Gabbay 304e65e175bSOded Gabbay *handle = buf->handle; 305e65e175bSOded Gabbay 306e65e175bSOded Gabbay return 0; 307e65e175bSOded Gabbay } 308e65e175bSOded Gabbay 309e65e175bSOded Gabbay int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle) 310e65e175bSOded Gabbay { 311e65e175bSOded Gabbay struct hl_cb *cb; 312e65e175bSOded Gabbay int rc; 313e65e175bSOded Gabbay 314e65e175bSOded Gabbay cb = hl_cb_get(mmg, cb_handle); 315e65e175bSOded Gabbay if (!cb) { 316e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB destroy failed, no CB was found for handle %#llx\n", 317e65e175bSOded Gabbay cb_handle); 318e2a079a2STomer Tayar return -EINVAL; 319e65e175bSOded Gabbay } 320e65e175bSOded Gabbay 321e2a079a2STomer Tayar /* Make sure that CB handle isn't destroyed more than once */ 322e65e175bSOded Gabbay rc = atomic_cmpxchg(&cb->is_handle_destroyed, 0, 1); 323e65e175bSOded Gabbay hl_cb_put(cb); 324e65e175bSOded Gabbay if (rc) { 325e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB destroy failed, handle %#llx was already destroyed\n", 326e65e175bSOded Gabbay cb_handle); 327e2a079a2STomer Tayar return -EINVAL; 328e65e175bSOded Gabbay } 329e65e175bSOded Gabbay 330e65e175bSOded Gabbay rc = hl_mmap_mem_buf_put_handle(mmg, cb_handle); 331e65e175bSOded Gabbay if (rc < 0) 332e65e175bSOded Gabbay return rc; /* Invalid handle */ 333e65e175bSOded Gabbay 334e65e175bSOded Gabbay if (rc == 0) 335e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB 0x%llx is destroyed while still in use\n", cb_handle); 336e65e175bSOded Gabbay 337e65e175bSOded Gabbay return 0; 338e65e175bSOded Gabbay } 339e65e175bSOded Gabbay 340e65e175bSOded Gabbay static int hl_cb_info(struct hl_mem_mgr *mmg, 341e65e175bSOded Gabbay u64 handle, u32 flags, u32 *usage_cnt, u64 *device_va) 342e65e175bSOded Gabbay { 343e65e175bSOded Gabbay struct hl_cb *cb; 344e65e175bSOded Gabbay int rc = 0; 345e65e175bSOded Gabbay 346e65e175bSOded Gabbay cb = hl_cb_get(mmg, handle); 347e65e175bSOded Gabbay if (!cb) { 348e65e175bSOded Gabbay dev_err(mmg->dev, 349e65e175bSOded Gabbay "CB info failed, no match to handle 0x%llx\n", handle); 350e65e175bSOded Gabbay return -EINVAL; 351e65e175bSOded Gabbay } 352e65e175bSOded Gabbay 353e65e175bSOded Gabbay if (flags & HL_CB_FLAGS_GET_DEVICE_VA) { 354e65e175bSOded Gabbay if (cb->is_mmu_mapped) { 355e65e175bSOded Gabbay *device_va = cb->virtual_addr; 356e65e175bSOded Gabbay } else { 357e65e175bSOded Gabbay dev_err(mmg->dev, "CB is not mapped to the device's MMU\n"); 358e65e175bSOded Gabbay rc = -EINVAL; 359e65e175bSOded Gabbay goto out; 360e65e175bSOded Gabbay } 361e65e175bSOded Gabbay } else { 362e65e175bSOded Gabbay *usage_cnt = atomic_read(&cb->cs_cnt); 363e65e175bSOded Gabbay } 364e65e175bSOded Gabbay 365e65e175bSOded Gabbay out: 366e65e175bSOded Gabbay hl_cb_put(cb); 367e65e175bSOded Gabbay return rc; 368e65e175bSOded Gabbay } 369e65e175bSOded Gabbay 370e65e175bSOded Gabbay int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) 371e65e175bSOded Gabbay { 372e65e175bSOded Gabbay union hl_cb_args *args = data; 373e65e175bSOded Gabbay struct hl_device *hdev = hpriv->hdev; 374e65e175bSOded Gabbay u64 handle = 0, device_va = 0; 375e65e175bSOded Gabbay enum hl_device_status status; 376e65e175bSOded Gabbay u32 usage_cnt = 0; 377e65e175bSOded Gabbay int rc; 378e65e175bSOded Gabbay 379e65e175bSOded Gabbay if (!hl_device_operational(hdev, &status)) { 380e1e8e747SOded Gabbay dev_dbg_ratelimited(hdev->dev, 381e65e175bSOded Gabbay "Device is %s. Can't execute CB IOCTL\n", 382e65e175bSOded Gabbay hdev->status[status]); 383e65e175bSOded Gabbay return -EBUSY; 384e65e175bSOded Gabbay } 385e65e175bSOded Gabbay 386e65e175bSOded Gabbay switch (args->in.op) { 387e65e175bSOded Gabbay case HL_CB_OP_CREATE: 388e65e175bSOded Gabbay if (args->in.cb_size > HL_MAX_CB_SIZE) { 389e65e175bSOded Gabbay dev_err(hdev->dev, 390e65e175bSOded Gabbay "User requested CB size %d must be less than %d\n", 391e65e175bSOded Gabbay args->in.cb_size, HL_MAX_CB_SIZE); 392e65e175bSOded Gabbay rc = -EINVAL; 393e65e175bSOded Gabbay } else { 394e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hpriv->mem_mgr, hpriv->ctx, 395e65e175bSOded Gabbay args->in.cb_size, false, 396e65e175bSOded Gabbay !!(args->in.flags & HL_CB_FLAGS_MAP), 397e65e175bSOded Gabbay &handle); 398e65e175bSOded Gabbay } 399e65e175bSOded Gabbay 400e65e175bSOded Gabbay memset(args, 0, sizeof(*args)); 401e65e175bSOded Gabbay args->out.cb_handle = handle; 402e65e175bSOded Gabbay break; 403e65e175bSOded Gabbay 404e65e175bSOded Gabbay case HL_CB_OP_DESTROY: 405e65e175bSOded Gabbay rc = hl_cb_destroy(&hpriv->mem_mgr, 406e65e175bSOded Gabbay args->in.cb_handle); 407e65e175bSOded Gabbay break; 408e65e175bSOded Gabbay 409e65e175bSOded Gabbay case HL_CB_OP_INFO: 410e65e175bSOded Gabbay rc = hl_cb_info(&hpriv->mem_mgr, args->in.cb_handle, 411e65e175bSOded Gabbay args->in.flags, 412e65e175bSOded Gabbay &usage_cnt, 413e65e175bSOded Gabbay &device_va); 414e65e175bSOded Gabbay if (rc) 415e65e175bSOded Gabbay break; 416e65e175bSOded Gabbay 417e65e175bSOded Gabbay memset(&args->out, 0, sizeof(args->out)); 418e65e175bSOded Gabbay 419e65e175bSOded Gabbay if (args->in.flags & HL_CB_FLAGS_GET_DEVICE_VA) 420e65e175bSOded Gabbay args->out.device_va = device_va; 421e65e175bSOded Gabbay else 422e65e175bSOded Gabbay args->out.usage_cnt = usage_cnt; 423e65e175bSOded Gabbay break; 424e65e175bSOded Gabbay 425e65e175bSOded Gabbay default: 426e65e175bSOded Gabbay rc = -EINVAL; 427e65e175bSOded Gabbay break; 428e65e175bSOded Gabbay } 429e65e175bSOded Gabbay 430e65e175bSOded Gabbay return rc; 431e65e175bSOded Gabbay } 432e65e175bSOded Gabbay 433e65e175bSOded Gabbay struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle) 434e65e175bSOded Gabbay { 435e65e175bSOded Gabbay struct hl_mmap_mem_buf *buf; 436e65e175bSOded Gabbay 437e65e175bSOded Gabbay buf = hl_mmap_mem_buf_get(mmg, handle); 438e65e175bSOded Gabbay if (!buf) 439e65e175bSOded Gabbay return NULL; 440e65e175bSOded Gabbay return buf->private; 441e65e175bSOded Gabbay 442e65e175bSOded Gabbay } 443e65e175bSOded Gabbay 444e65e175bSOded Gabbay void hl_cb_put(struct hl_cb *cb) 445e65e175bSOded Gabbay { 446e65e175bSOded Gabbay hl_mmap_mem_buf_put(cb->buf); 447e65e175bSOded Gabbay } 448e65e175bSOded Gabbay 449e65e175bSOded Gabbay struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, 450e65e175bSOded Gabbay bool internal_cb) 451e65e175bSOded Gabbay { 452e65e175bSOded Gabbay u64 cb_handle; 453e65e175bSOded Gabbay struct hl_cb *cb; 454e65e175bSOded Gabbay int rc; 455e65e175bSOded Gabbay 456e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, cb_size, 457e65e175bSOded Gabbay internal_cb, false, &cb_handle); 458e65e175bSOded Gabbay if (rc) { 459e65e175bSOded Gabbay dev_err(hdev->dev, 460e65e175bSOded Gabbay "Failed to allocate CB for the kernel driver %d\n", rc); 461e65e175bSOded Gabbay return NULL; 462e65e175bSOded Gabbay } 463e65e175bSOded Gabbay 464e65e175bSOded Gabbay cb = hl_cb_get(&hdev->kernel_mem_mgr, cb_handle); 465e65e175bSOded Gabbay /* hl_cb_get should never fail here */ 466e65e175bSOded Gabbay if (!cb) { 467e65e175bSOded Gabbay dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n", 468e65e175bSOded Gabbay (u32) cb_handle); 469e65e175bSOded Gabbay goto destroy_cb; 470e65e175bSOded Gabbay } 471e65e175bSOded Gabbay 472e65e175bSOded Gabbay return cb; 473e65e175bSOded Gabbay 474e65e175bSOded Gabbay destroy_cb: 475e65e175bSOded Gabbay hl_cb_destroy(&hdev->kernel_mem_mgr, cb_handle); 476e65e175bSOded Gabbay 477e65e175bSOded Gabbay return NULL; 478e65e175bSOded Gabbay } 479e65e175bSOded Gabbay 480e65e175bSOded Gabbay int hl_cb_pool_init(struct hl_device *hdev) 481e65e175bSOded Gabbay { 482e65e175bSOded Gabbay struct hl_cb *cb; 483e65e175bSOded Gabbay int i; 484e65e175bSOded Gabbay 485e65e175bSOded Gabbay INIT_LIST_HEAD(&hdev->cb_pool); 486e65e175bSOded Gabbay spin_lock_init(&hdev->cb_pool_lock); 487e65e175bSOded Gabbay 488e65e175bSOded Gabbay for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) { 489e65e175bSOded Gabbay cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size, 490e65e175bSOded Gabbay HL_KERNEL_ASID_ID, false); 491e65e175bSOded Gabbay if (cb) { 492e65e175bSOded Gabbay cb->is_pool = true; 493e65e175bSOded Gabbay list_add(&cb->pool_list, &hdev->cb_pool); 494e65e175bSOded Gabbay } else { 495e65e175bSOded Gabbay hl_cb_pool_fini(hdev); 496e65e175bSOded Gabbay return -ENOMEM; 497e65e175bSOded Gabbay } 498e65e175bSOded Gabbay } 499e65e175bSOded Gabbay 500e65e175bSOded Gabbay return 0; 501e65e175bSOded Gabbay } 502e65e175bSOded Gabbay 503e65e175bSOded Gabbay int hl_cb_pool_fini(struct hl_device *hdev) 504e65e175bSOded Gabbay { 505e65e175bSOded Gabbay struct hl_cb *cb, *tmp; 506e65e175bSOded Gabbay 507e65e175bSOded Gabbay list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) { 508e65e175bSOded Gabbay list_del(&cb->pool_list); 509e65e175bSOded Gabbay cb_fini(hdev, cb); 510e65e175bSOded Gabbay } 511e65e175bSOded Gabbay 512e65e175bSOded Gabbay return 0; 513e65e175bSOded Gabbay } 514e65e175bSOded Gabbay 515e65e175bSOded Gabbay int hl_cb_va_pool_init(struct hl_ctx *ctx) 516e65e175bSOded Gabbay { 517e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 518e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 519e65e175bSOded Gabbay int rc; 520e65e175bSOded Gabbay 521e65e175bSOded Gabbay if (!hdev->supports_cb_mapping) 522e65e175bSOded Gabbay return 0; 523e65e175bSOded Gabbay 524e65e175bSOded Gabbay ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1); 525e65e175bSOded Gabbay if (!ctx->cb_va_pool) { 526e65e175bSOded Gabbay dev_err(hdev->dev, 527e65e175bSOded Gabbay "Failed to create VA gen pool for CB mapping\n"); 528e65e175bSOded Gabbay return -ENOMEM; 529e65e175bSOded Gabbay } 530e65e175bSOded Gabbay 531e65e175bSOded Gabbay ctx->cb_va_pool_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 532e65e175bSOded Gabbay CB_VA_POOL_SIZE, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 533e65e175bSOded Gabbay if (!ctx->cb_va_pool_base) { 534e65e175bSOded Gabbay rc = -ENOMEM; 535e65e175bSOded Gabbay goto err_pool_destroy; 536e65e175bSOded Gabbay } 537e65e175bSOded Gabbay rc = gen_pool_add(ctx->cb_va_pool, ctx->cb_va_pool_base, CB_VA_POOL_SIZE, -1); 538e65e175bSOded Gabbay if (rc) { 539e65e175bSOded Gabbay dev_err(hdev->dev, 540e65e175bSOded Gabbay "Failed to add memory to VA gen pool for CB mapping\n"); 541e65e175bSOded Gabbay goto err_unreserve_va_block; 542e65e175bSOded Gabbay } 543e65e175bSOded Gabbay 544e65e175bSOded Gabbay return 0; 545e65e175bSOded Gabbay 546e65e175bSOded Gabbay err_unreserve_va_block: 547e65e175bSOded Gabbay hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 548e65e175bSOded Gabbay err_pool_destroy: 549e65e175bSOded Gabbay gen_pool_destroy(ctx->cb_va_pool); 550e65e175bSOded Gabbay 551e65e175bSOded Gabbay return rc; 552e65e175bSOded Gabbay } 553e65e175bSOded Gabbay 554e65e175bSOded Gabbay void hl_cb_va_pool_fini(struct hl_ctx *ctx) 555e65e175bSOded Gabbay { 556e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 557e65e175bSOded Gabbay 558e65e175bSOded Gabbay if (!hdev->supports_cb_mapping) 559e65e175bSOded Gabbay return; 560e65e175bSOded Gabbay 561e65e175bSOded Gabbay gen_pool_destroy(ctx->cb_va_pool); 562e65e175bSOded Gabbay hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 563e65e175bSOded Gabbay } 564