1e65e175bSOded Gabbay // SPDX-License-Identifier: GPL-2.0 2e65e175bSOded Gabbay 3e65e175bSOded Gabbay /* 4e65e175bSOded Gabbay * Copyright 2016-2019 HabanaLabs, Ltd. 5e65e175bSOded Gabbay * All Rights Reserved. 6e65e175bSOded Gabbay */ 7e65e175bSOded Gabbay 8e65e175bSOded Gabbay #include <uapi/drm/habanalabs_accel.h> 9e65e175bSOded Gabbay #include "habanalabs.h" 10e65e175bSOded Gabbay 11e65e175bSOded Gabbay #include <linux/mm.h> 12e65e175bSOded Gabbay #include <linux/slab.h> 13e65e175bSOded Gabbay #include <linux/uaccess.h> 14e65e175bSOded Gabbay 15e65e175bSOded Gabbay #define CB_VA_POOL_SIZE (4UL * SZ_1G) 16e65e175bSOded Gabbay 17e65e175bSOded Gabbay static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb) 18e65e175bSOded Gabbay { 19e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 20e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 21e65e175bSOded Gabbay u32 page_size = prop->pmmu.page_size; 22e65e175bSOded Gabbay int rc; 23e65e175bSOded Gabbay 24e65e175bSOded Gabbay if (!hdev->supports_cb_mapping) { 25e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, 26e65e175bSOded Gabbay "Mapping a CB to the device's MMU is not supported\n"); 27e65e175bSOded Gabbay return -EINVAL; 28e65e175bSOded Gabbay } 29e65e175bSOded Gabbay 30e65e175bSOded Gabbay if (!hdev->mmu_enable) { 31e65e175bSOded Gabbay dev_err_ratelimited(hdev->dev, 32e65e175bSOded Gabbay "Cannot map CB because MMU is disabled\n"); 33e65e175bSOded Gabbay return -EINVAL; 34e65e175bSOded Gabbay } 35e65e175bSOded Gabbay 36e65e175bSOded Gabbay if (cb->is_mmu_mapped) 37e65e175bSOded Gabbay return 0; 38e65e175bSOded Gabbay 39e65e175bSOded Gabbay cb->roundup_size = roundup(cb->size, page_size); 40e65e175bSOded Gabbay 41e65e175bSOded Gabbay cb->virtual_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, cb->roundup_size); 42e65e175bSOded Gabbay if (!cb->virtual_addr) { 43e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to allocate device virtual address for CB\n"); 44e65e175bSOded Gabbay return -ENOMEM; 45e65e175bSOded Gabbay } 46e65e175bSOded Gabbay 47e65e175bSOded Gabbay mutex_lock(&hdev->mmu_lock); 48e65e175bSOded Gabbay rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size); 49e65e175bSOded Gabbay if (rc) { 50e65e175bSOded Gabbay dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr); 51e65e175bSOded Gabbay goto err_va_umap; 52e65e175bSOded Gabbay } 53e65e175bSOded Gabbay rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV); 54e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock); 55e65e175bSOded Gabbay 56e65e175bSOded Gabbay cb->is_mmu_mapped = true; 57e65e175bSOded Gabbay return rc; 58e65e175bSOded Gabbay 59e65e175bSOded Gabbay err_va_umap: 60e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock); 61e65e175bSOded Gabbay gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 62e65e175bSOded Gabbay return rc; 63e65e175bSOded Gabbay } 64e65e175bSOded Gabbay 65e65e175bSOded Gabbay static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb) 66e65e175bSOded Gabbay { 67e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 68e65e175bSOded Gabbay 69e65e175bSOded Gabbay mutex_lock(&hdev->mmu_lock); 70e65e175bSOded Gabbay hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); 71e65e175bSOded Gabbay hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); 72e65e175bSOded Gabbay mutex_unlock(&hdev->mmu_lock); 73e65e175bSOded Gabbay 74e65e175bSOded Gabbay gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); 75e65e175bSOded Gabbay } 76e65e175bSOded Gabbay 77e65e175bSOded Gabbay static void cb_fini(struct hl_device *hdev, struct hl_cb *cb) 78e65e175bSOded Gabbay { 79e65e175bSOded Gabbay if (cb->is_internal) 80e65e175bSOded Gabbay gen_pool_free(hdev->internal_cb_pool, 81e65e175bSOded Gabbay (uintptr_t)cb->kernel_address, cb->size); 82e65e175bSOded Gabbay else 83e65e175bSOded Gabbay hl_asic_dma_free_coherent(hdev, cb->size, cb->kernel_address, cb->bus_address); 84e65e175bSOded Gabbay 85e65e175bSOded Gabbay kfree(cb); 86e65e175bSOded Gabbay } 87e65e175bSOded Gabbay 88e65e175bSOded Gabbay static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb) 89e65e175bSOded Gabbay { 90e65e175bSOded Gabbay if (cb->is_pool) { 91e2a079a2STomer Tayar atomic_set(&cb->is_handle_destroyed, 0); 92e65e175bSOded Gabbay spin_lock(&hdev->cb_pool_lock); 93e65e175bSOded Gabbay list_add(&cb->pool_list, &hdev->cb_pool); 94e65e175bSOded Gabbay spin_unlock(&hdev->cb_pool_lock); 95e65e175bSOded Gabbay } else { 96e65e175bSOded Gabbay cb_fini(hdev, cb); 97e65e175bSOded Gabbay } 98e65e175bSOded Gabbay } 99e65e175bSOded Gabbay 100e65e175bSOded Gabbay static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size, 101e65e175bSOded Gabbay int ctx_id, bool internal_cb) 102e65e175bSOded Gabbay { 103e65e175bSOded Gabbay struct hl_cb *cb = NULL; 104e65e175bSOded Gabbay u32 cb_offset; 105e65e175bSOded Gabbay void *p; 106e65e175bSOded Gabbay 107e65e175bSOded Gabbay /* 108e65e175bSOded Gabbay * We use of GFP_ATOMIC here because this function can be called from 109e65e175bSOded Gabbay * the latency-sensitive code path for command submission. Due to H/W 110e65e175bSOded Gabbay * limitations in some of the ASICs, the kernel must copy the user CB 111e65e175bSOded Gabbay * that is designated for an external queue and actually enqueue 112e65e175bSOded Gabbay * the kernel's copy. Hence, we must never sleep in this code section 113e65e175bSOded Gabbay * and must use GFP_ATOMIC for all memory allocations. 114e65e175bSOded Gabbay */ 115e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID && !hdev->disabled) 116e65e175bSOded Gabbay cb = kzalloc(sizeof(*cb), GFP_ATOMIC); 117e65e175bSOded Gabbay 118e65e175bSOded Gabbay if (!cb) 119e65e175bSOded Gabbay cb = kzalloc(sizeof(*cb), GFP_KERNEL); 120e65e175bSOded Gabbay 121e65e175bSOded Gabbay if (!cb) 122e65e175bSOded Gabbay return NULL; 123e65e175bSOded Gabbay 124e65e175bSOded Gabbay if (internal_cb) { 125e65e175bSOded Gabbay p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size); 126e65e175bSOded Gabbay if (!p) { 127e65e175bSOded Gabbay kfree(cb); 128e65e175bSOded Gabbay return NULL; 129e65e175bSOded Gabbay } 130e65e175bSOded Gabbay 131e65e175bSOded Gabbay cb_offset = p - hdev->internal_cb_pool_virt_addr; 132e65e175bSOded Gabbay cb->is_internal = true; 133e65e175bSOded Gabbay cb->bus_address = hdev->internal_cb_va_base + cb_offset; 134e65e175bSOded Gabbay } else if (ctx_id == HL_KERNEL_ASID_ID) { 135e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_ATOMIC); 136e65e175bSOded Gabbay if (!p) 137e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, GFP_KERNEL); 138e65e175bSOded Gabbay } else { 139e65e175bSOded Gabbay p = hl_asic_dma_alloc_coherent(hdev, cb_size, &cb->bus_address, 140e65e175bSOded Gabbay GFP_USER | __GFP_ZERO); 141e65e175bSOded Gabbay } 142e65e175bSOded Gabbay 143e65e175bSOded Gabbay if (!p) { 144e65e175bSOded Gabbay dev_err(hdev->dev, 145e65e175bSOded Gabbay "failed to allocate %d of dma memory for CB\n", 146e65e175bSOded Gabbay cb_size); 147e65e175bSOded Gabbay kfree(cb); 148e65e175bSOded Gabbay return NULL; 149e65e175bSOded Gabbay } 150e65e175bSOded Gabbay 151e65e175bSOded Gabbay cb->kernel_address = p; 152e65e175bSOded Gabbay cb->size = cb_size; 153e65e175bSOded Gabbay 154e65e175bSOded Gabbay return cb; 155e65e175bSOded Gabbay } 156e65e175bSOded Gabbay 157e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args { 158e65e175bSOded Gabbay struct hl_device *hdev; 159e65e175bSOded Gabbay struct hl_ctx *ctx; 160e65e175bSOded Gabbay u32 cb_size; 161e65e175bSOded Gabbay bool internal_cb; 162e65e175bSOded Gabbay bool map_cb; 163e65e175bSOded Gabbay }; 164e65e175bSOded Gabbay 165e65e175bSOded Gabbay static void hl_cb_mmap_mem_release(struct hl_mmap_mem_buf *buf) 166e65e175bSOded Gabbay { 167e65e175bSOded Gabbay struct hl_cb *cb = buf->private; 168e65e175bSOded Gabbay 169e65e175bSOded Gabbay hl_debugfs_remove_cb(cb); 170e65e175bSOded Gabbay 171e65e175bSOded Gabbay if (cb->is_mmu_mapped) 172e65e175bSOded Gabbay cb_unmap_mem(cb->ctx, cb); 173e65e175bSOded Gabbay 174e65e175bSOded Gabbay hl_ctx_put(cb->ctx); 175e65e175bSOded Gabbay 176e65e175bSOded Gabbay cb_do_release(cb->hdev, cb); 177e65e175bSOded Gabbay } 178e65e175bSOded Gabbay 179e65e175bSOded Gabbay static int hl_cb_mmap_mem_alloc(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args) 180e65e175bSOded Gabbay { 181e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args *cb_args = args; 182e65e175bSOded Gabbay struct hl_cb *cb; 183e65e175bSOded Gabbay int rc, ctx_id = cb_args->ctx->asid; 184e65e175bSOded Gabbay bool alloc_new_cb = true; 185e65e175bSOded Gabbay 186e65e175bSOded Gabbay if (!cb_args->internal_cb) { 187e65e175bSOded Gabbay /* Minimum allocation must be PAGE SIZE */ 188e65e175bSOded Gabbay if (cb_args->cb_size < PAGE_SIZE) 189e65e175bSOded Gabbay cb_args->cb_size = PAGE_SIZE; 190e65e175bSOded Gabbay 191e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID && 192e65e175bSOded Gabbay cb_args->cb_size <= cb_args->hdev->asic_prop.cb_pool_cb_size) { 193e65e175bSOded Gabbay 194e65e175bSOded Gabbay spin_lock(&cb_args->hdev->cb_pool_lock); 195e65e175bSOded Gabbay if (!list_empty(&cb_args->hdev->cb_pool)) { 196e65e175bSOded Gabbay cb = list_first_entry(&cb_args->hdev->cb_pool, 197e65e175bSOded Gabbay typeof(*cb), pool_list); 198e65e175bSOded Gabbay list_del(&cb->pool_list); 199e65e175bSOded Gabbay spin_unlock(&cb_args->hdev->cb_pool_lock); 200e65e175bSOded Gabbay alloc_new_cb = false; 201e65e175bSOded Gabbay } else { 202e65e175bSOded Gabbay spin_unlock(&cb_args->hdev->cb_pool_lock); 203e65e175bSOded Gabbay dev_dbg(cb_args->hdev->dev, "CB pool is empty\n"); 204e65e175bSOded Gabbay } 205e65e175bSOded Gabbay } 206e65e175bSOded Gabbay } 207e65e175bSOded Gabbay 208e65e175bSOded Gabbay if (alloc_new_cb) { 209e65e175bSOded Gabbay cb = hl_cb_alloc(cb_args->hdev, cb_args->cb_size, ctx_id, cb_args->internal_cb); 210e65e175bSOded Gabbay if (!cb) 211e65e175bSOded Gabbay return -ENOMEM; 212e65e175bSOded Gabbay } 213e65e175bSOded Gabbay 214e65e175bSOded Gabbay cb->hdev = cb_args->hdev; 215e65e175bSOded Gabbay cb->ctx = cb_args->ctx; 216e65e175bSOded Gabbay cb->buf = buf; 217e65e175bSOded Gabbay cb->buf->mappable_size = cb->size; 218e65e175bSOded Gabbay cb->buf->private = cb; 219e65e175bSOded Gabbay 220e65e175bSOded Gabbay hl_ctx_get(cb->ctx); 221e65e175bSOded Gabbay 222e65e175bSOded Gabbay if (cb_args->map_cb) { 223e65e175bSOded Gabbay if (ctx_id == HL_KERNEL_ASID_ID) { 224e65e175bSOded Gabbay dev_err(cb_args->hdev->dev, 225e65e175bSOded Gabbay "CB mapping is not supported for kernel context\n"); 226e65e175bSOded Gabbay rc = -EINVAL; 227e65e175bSOded Gabbay goto release_cb; 228e65e175bSOded Gabbay } 229e65e175bSOded Gabbay 230e65e175bSOded Gabbay rc = cb_map_mem(cb_args->ctx, cb); 231e65e175bSOded Gabbay if (rc) 232e65e175bSOded Gabbay goto release_cb; 233e65e175bSOded Gabbay } 234e65e175bSOded Gabbay 235e65e175bSOded Gabbay hl_debugfs_add_cb(cb); 236e65e175bSOded Gabbay 237e65e175bSOded Gabbay return 0; 238e65e175bSOded Gabbay 239e65e175bSOded Gabbay release_cb: 240e65e175bSOded Gabbay hl_ctx_put(cb->ctx); 241e65e175bSOded Gabbay cb_do_release(cb_args->hdev, cb); 242e65e175bSOded Gabbay 243e65e175bSOded Gabbay return rc; 244e65e175bSOded Gabbay } 245e65e175bSOded Gabbay 246e65e175bSOded Gabbay static int hl_cb_mmap(struct hl_mmap_mem_buf *buf, 247e65e175bSOded Gabbay struct vm_area_struct *vma, void *args) 248e65e175bSOded Gabbay { 249e65e175bSOded Gabbay struct hl_cb *cb = buf->private; 250e65e175bSOded Gabbay 251e65e175bSOded Gabbay return cb->hdev->asic_funcs->mmap(cb->hdev, vma, cb->kernel_address, 252e65e175bSOded Gabbay cb->bus_address, cb->size); 253e65e175bSOded Gabbay } 254e65e175bSOded Gabbay 255e65e175bSOded Gabbay static struct hl_mmap_mem_buf_behavior cb_behavior = { 256e65e175bSOded Gabbay .topic = "CB", 257e65e175bSOded Gabbay .mem_id = HL_MMAP_TYPE_CB, 258e65e175bSOded Gabbay .alloc = hl_cb_mmap_mem_alloc, 259e65e175bSOded Gabbay .release = hl_cb_mmap_mem_release, 260e65e175bSOded Gabbay .mmap = hl_cb_mmap, 261e65e175bSOded Gabbay }; 262e65e175bSOded Gabbay 263e65e175bSOded Gabbay int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, 264e65e175bSOded Gabbay struct hl_ctx *ctx, u32 cb_size, bool internal_cb, 265e65e175bSOded Gabbay bool map_cb, u64 *handle) 266e65e175bSOded Gabbay { 267e65e175bSOded Gabbay struct hl_cb_mmap_mem_alloc_args args = { 268e65e175bSOded Gabbay .hdev = hdev, 269e65e175bSOded Gabbay .ctx = ctx, 270e65e175bSOded Gabbay .cb_size = cb_size, 271e65e175bSOded Gabbay .internal_cb = internal_cb, 272e65e175bSOded Gabbay .map_cb = map_cb, 273e65e175bSOded Gabbay }; 274e65e175bSOded Gabbay struct hl_mmap_mem_buf *buf; 275e65e175bSOded Gabbay int ctx_id = ctx->asid; 276e65e175bSOded Gabbay 277e65e175bSOded Gabbay if ((hdev->disabled) || (hdev->reset_info.in_reset && (ctx_id != HL_KERNEL_ASID_ID))) { 278e65e175bSOded Gabbay dev_warn_ratelimited(hdev->dev, 279e65e175bSOded Gabbay "Device is disabled or in reset. Can't create new CBs\n"); 280e65e175bSOded Gabbay return -EBUSY; 281e65e175bSOded Gabbay } 282e65e175bSOded Gabbay 283e65e175bSOded Gabbay if (cb_size > SZ_2M) { 284e65e175bSOded Gabbay dev_err(hdev->dev, "CB size %d must be less than %d\n", 285e65e175bSOded Gabbay cb_size, SZ_2M); 286e65e175bSOded Gabbay return -EINVAL; 287e65e175bSOded Gabbay } 288e65e175bSOded Gabbay 289e65e175bSOded Gabbay buf = hl_mmap_mem_buf_alloc( 290e65e175bSOded Gabbay mmg, &cb_behavior, 291e65e175bSOded Gabbay ctx_id == HL_KERNEL_ASID_ID ? GFP_ATOMIC : GFP_KERNEL, &args); 292e65e175bSOded Gabbay if (!buf) 293e65e175bSOded Gabbay return -ENOMEM; 294e65e175bSOded Gabbay 295e65e175bSOded Gabbay *handle = buf->handle; 296e65e175bSOded Gabbay 297e65e175bSOded Gabbay return 0; 298e65e175bSOded Gabbay } 299e65e175bSOded Gabbay 300e65e175bSOded Gabbay int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle) 301e65e175bSOded Gabbay { 302e65e175bSOded Gabbay struct hl_cb *cb; 303e65e175bSOded Gabbay int rc; 304e65e175bSOded Gabbay 305e65e175bSOded Gabbay cb = hl_cb_get(mmg, cb_handle); 306e65e175bSOded Gabbay if (!cb) { 307e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB destroy failed, no CB was found for handle %#llx\n", 308e65e175bSOded Gabbay cb_handle); 309e2a079a2STomer Tayar return -EINVAL; 310e65e175bSOded Gabbay } 311e65e175bSOded Gabbay 312e2a079a2STomer Tayar /* Make sure that CB handle isn't destroyed more than once */ 313e65e175bSOded Gabbay rc = atomic_cmpxchg(&cb->is_handle_destroyed, 0, 1); 314e65e175bSOded Gabbay hl_cb_put(cb); 315e65e175bSOded Gabbay if (rc) { 316e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB destroy failed, handle %#llx was already destroyed\n", 317e65e175bSOded Gabbay cb_handle); 318e2a079a2STomer Tayar return -EINVAL; 319e65e175bSOded Gabbay } 320e65e175bSOded Gabbay 321e65e175bSOded Gabbay rc = hl_mmap_mem_buf_put_handle(mmg, cb_handle); 322e65e175bSOded Gabbay if (rc < 0) 323e65e175bSOded Gabbay return rc; /* Invalid handle */ 324e65e175bSOded Gabbay 325e65e175bSOded Gabbay if (rc == 0) 326e65e175bSOded Gabbay dev_dbg(mmg->dev, "CB 0x%llx is destroyed while still in use\n", cb_handle); 327e65e175bSOded Gabbay 328e65e175bSOded Gabbay return 0; 329e65e175bSOded Gabbay } 330e65e175bSOded Gabbay 331e65e175bSOded Gabbay static int hl_cb_info(struct hl_mem_mgr *mmg, 332e65e175bSOded Gabbay u64 handle, u32 flags, u32 *usage_cnt, u64 *device_va) 333e65e175bSOded Gabbay { 334e65e175bSOded Gabbay struct hl_cb *cb; 335e65e175bSOded Gabbay int rc = 0; 336e65e175bSOded Gabbay 337e65e175bSOded Gabbay cb = hl_cb_get(mmg, handle); 338e65e175bSOded Gabbay if (!cb) { 339e65e175bSOded Gabbay dev_err(mmg->dev, 340e65e175bSOded Gabbay "CB info failed, no match to handle 0x%llx\n", handle); 341e65e175bSOded Gabbay return -EINVAL; 342e65e175bSOded Gabbay } 343e65e175bSOded Gabbay 344e65e175bSOded Gabbay if (flags & HL_CB_FLAGS_GET_DEVICE_VA) { 345e65e175bSOded Gabbay if (cb->is_mmu_mapped) { 346e65e175bSOded Gabbay *device_va = cb->virtual_addr; 347e65e175bSOded Gabbay } else { 348e65e175bSOded Gabbay dev_err(mmg->dev, "CB is not mapped to the device's MMU\n"); 349e65e175bSOded Gabbay rc = -EINVAL; 350e65e175bSOded Gabbay goto out; 351e65e175bSOded Gabbay } 352e65e175bSOded Gabbay } else { 353e65e175bSOded Gabbay *usage_cnt = atomic_read(&cb->cs_cnt); 354e65e175bSOded Gabbay } 355e65e175bSOded Gabbay 356e65e175bSOded Gabbay out: 357e65e175bSOded Gabbay hl_cb_put(cb); 358e65e175bSOded Gabbay return rc; 359e65e175bSOded Gabbay } 360e65e175bSOded Gabbay 361e65e175bSOded Gabbay int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) 362e65e175bSOded Gabbay { 363e65e175bSOded Gabbay union hl_cb_args *args = data; 364e65e175bSOded Gabbay struct hl_device *hdev = hpriv->hdev; 365e65e175bSOded Gabbay u64 handle = 0, device_va = 0; 366e65e175bSOded Gabbay enum hl_device_status status; 367e65e175bSOded Gabbay u32 usage_cnt = 0; 368e65e175bSOded Gabbay int rc; 369e65e175bSOded Gabbay 370e65e175bSOded Gabbay if (!hl_device_operational(hdev, &status)) { 371*e1e8e747SOded Gabbay dev_dbg_ratelimited(hdev->dev, 372e65e175bSOded Gabbay "Device is %s. Can't execute CB IOCTL\n", 373e65e175bSOded Gabbay hdev->status[status]); 374e65e175bSOded Gabbay return -EBUSY; 375e65e175bSOded Gabbay } 376e65e175bSOded Gabbay 377e65e175bSOded Gabbay switch (args->in.op) { 378e65e175bSOded Gabbay case HL_CB_OP_CREATE: 379e65e175bSOded Gabbay if (args->in.cb_size > HL_MAX_CB_SIZE) { 380e65e175bSOded Gabbay dev_err(hdev->dev, 381e65e175bSOded Gabbay "User requested CB size %d must be less than %d\n", 382e65e175bSOded Gabbay args->in.cb_size, HL_MAX_CB_SIZE); 383e65e175bSOded Gabbay rc = -EINVAL; 384e65e175bSOded Gabbay } else { 385e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hpriv->mem_mgr, hpriv->ctx, 386e65e175bSOded Gabbay args->in.cb_size, false, 387e65e175bSOded Gabbay !!(args->in.flags & HL_CB_FLAGS_MAP), 388e65e175bSOded Gabbay &handle); 389e65e175bSOded Gabbay } 390e65e175bSOded Gabbay 391e65e175bSOded Gabbay memset(args, 0, sizeof(*args)); 392e65e175bSOded Gabbay args->out.cb_handle = handle; 393e65e175bSOded Gabbay break; 394e65e175bSOded Gabbay 395e65e175bSOded Gabbay case HL_CB_OP_DESTROY: 396e65e175bSOded Gabbay rc = hl_cb_destroy(&hpriv->mem_mgr, 397e65e175bSOded Gabbay args->in.cb_handle); 398e65e175bSOded Gabbay break; 399e65e175bSOded Gabbay 400e65e175bSOded Gabbay case HL_CB_OP_INFO: 401e65e175bSOded Gabbay rc = hl_cb_info(&hpriv->mem_mgr, args->in.cb_handle, 402e65e175bSOded Gabbay args->in.flags, 403e65e175bSOded Gabbay &usage_cnt, 404e65e175bSOded Gabbay &device_va); 405e65e175bSOded Gabbay if (rc) 406e65e175bSOded Gabbay break; 407e65e175bSOded Gabbay 408e65e175bSOded Gabbay memset(&args->out, 0, sizeof(args->out)); 409e65e175bSOded Gabbay 410e65e175bSOded Gabbay if (args->in.flags & HL_CB_FLAGS_GET_DEVICE_VA) 411e65e175bSOded Gabbay args->out.device_va = device_va; 412e65e175bSOded Gabbay else 413e65e175bSOded Gabbay args->out.usage_cnt = usage_cnt; 414e65e175bSOded Gabbay break; 415e65e175bSOded Gabbay 416e65e175bSOded Gabbay default: 417e65e175bSOded Gabbay rc = -EINVAL; 418e65e175bSOded Gabbay break; 419e65e175bSOded Gabbay } 420e65e175bSOded Gabbay 421e65e175bSOded Gabbay return rc; 422e65e175bSOded Gabbay } 423e65e175bSOded Gabbay 424e65e175bSOded Gabbay struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle) 425e65e175bSOded Gabbay { 426e65e175bSOded Gabbay struct hl_mmap_mem_buf *buf; 427e65e175bSOded Gabbay 428e65e175bSOded Gabbay buf = hl_mmap_mem_buf_get(mmg, handle); 429e65e175bSOded Gabbay if (!buf) 430e65e175bSOded Gabbay return NULL; 431e65e175bSOded Gabbay return buf->private; 432e65e175bSOded Gabbay 433e65e175bSOded Gabbay } 434e65e175bSOded Gabbay 435e65e175bSOded Gabbay void hl_cb_put(struct hl_cb *cb) 436e65e175bSOded Gabbay { 437e65e175bSOded Gabbay hl_mmap_mem_buf_put(cb->buf); 438e65e175bSOded Gabbay } 439e65e175bSOded Gabbay 440e65e175bSOded Gabbay struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, 441e65e175bSOded Gabbay bool internal_cb) 442e65e175bSOded Gabbay { 443e65e175bSOded Gabbay u64 cb_handle; 444e65e175bSOded Gabbay struct hl_cb *cb; 445e65e175bSOded Gabbay int rc; 446e65e175bSOded Gabbay 447e65e175bSOded Gabbay rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, cb_size, 448e65e175bSOded Gabbay internal_cb, false, &cb_handle); 449e65e175bSOded Gabbay if (rc) { 450e65e175bSOded Gabbay dev_err(hdev->dev, 451e65e175bSOded Gabbay "Failed to allocate CB for the kernel driver %d\n", rc); 452e65e175bSOded Gabbay return NULL; 453e65e175bSOded Gabbay } 454e65e175bSOded Gabbay 455e65e175bSOded Gabbay cb = hl_cb_get(&hdev->kernel_mem_mgr, cb_handle); 456e65e175bSOded Gabbay /* hl_cb_get should never fail here */ 457e65e175bSOded Gabbay if (!cb) { 458e65e175bSOded Gabbay dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n", 459e65e175bSOded Gabbay (u32) cb_handle); 460e65e175bSOded Gabbay goto destroy_cb; 461e65e175bSOded Gabbay } 462e65e175bSOded Gabbay 463e65e175bSOded Gabbay return cb; 464e65e175bSOded Gabbay 465e65e175bSOded Gabbay destroy_cb: 466e65e175bSOded Gabbay hl_cb_destroy(&hdev->kernel_mem_mgr, cb_handle); 467e65e175bSOded Gabbay 468e65e175bSOded Gabbay return NULL; 469e65e175bSOded Gabbay } 470e65e175bSOded Gabbay 471e65e175bSOded Gabbay int hl_cb_pool_init(struct hl_device *hdev) 472e65e175bSOded Gabbay { 473e65e175bSOded Gabbay struct hl_cb *cb; 474e65e175bSOded Gabbay int i; 475e65e175bSOded Gabbay 476e65e175bSOded Gabbay INIT_LIST_HEAD(&hdev->cb_pool); 477e65e175bSOded Gabbay spin_lock_init(&hdev->cb_pool_lock); 478e65e175bSOded Gabbay 479e65e175bSOded Gabbay for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) { 480e65e175bSOded Gabbay cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size, 481e65e175bSOded Gabbay HL_KERNEL_ASID_ID, false); 482e65e175bSOded Gabbay if (cb) { 483e65e175bSOded Gabbay cb->is_pool = true; 484e65e175bSOded Gabbay list_add(&cb->pool_list, &hdev->cb_pool); 485e65e175bSOded Gabbay } else { 486e65e175bSOded Gabbay hl_cb_pool_fini(hdev); 487e65e175bSOded Gabbay return -ENOMEM; 488e65e175bSOded Gabbay } 489e65e175bSOded Gabbay } 490e65e175bSOded Gabbay 491e65e175bSOded Gabbay return 0; 492e65e175bSOded Gabbay } 493e65e175bSOded Gabbay 494e65e175bSOded Gabbay int hl_cb_pool_fini(struct hl_device *hdev) 495e65e175bSOded Gabbay { 496e65e175bSOded Gabbay struct hl_cb *cb, *tmp; 497e65e175bSOded Gabbay 498e65e175bSOded Gabbay list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) { 499e65e175bSOded Gabbay list_del(&cb->pool_list); 500e65e175bSOded Gabbay cb_fini(hdev, cb); 501e65e175bSOded Gabbay } 502e65e175bSOded Gabbay 503e65e175bSOded Gabbay return 0; 504e65e175bSOded Gabbay } 505e65e175bSOded Gabbay 506e65e175bSOded Gabbay int hl_cb_va_pool_init(struct hl_ctx *ctx) 507e65e175bSOded Gabbay { 508e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 509e65e175bSOded Gabbay struct asic_fixed_properties *prop = &hdev->asic_prop; 510e65e175bSOded Gabbay int rc; 511e65e175bSOded Gabbay 512e65e175bSOded Gabbay if (!hdev->supports_cb_mapping) 513e65e175bSOded Gabbay return 0; 514e65e175bSOded Gabbay 515e65e175bSOded Gabbay ctx->cb_va_pool = gen_pool_create(__ffs(prop->pmmu.page_size), -1); 516e65e175bSOded Gabbay if (!ctx->cb_va_pool) { 517e65e175bSOded Gabbay dev_err(hdev->dev, 518e65e175bSOded Gabbay "Failed to create VA gen pool for CB mapping\n"); 519e65e175bSOded Gabbay return -ENOMEM; 520e65e175bSOded Gabbay } 521e65e175bSOded Gabbay 522e65e175bSOded Gabbay ctx->cb_va_pool_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, 523e65e175bSOded Gabbay CB_VA_POOL_SIZE, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); 524e65e175bSOded Gabbay if (!ctx->cb_va_pool_base) { 525e65e175bSOded Gabbay rc = -ENOMEM; 526e65e175bSOded Gabbay goto err_pool_destroy; 527e65e175bSOded Gabbay } 528e65e175bSOded Gabbay rc = gen_pool_add(ctx->cb_va_pool, ctx->cb_va_pool_base, CB_VA_POOL_SIZE, -1); 529e65e175bSOded Gabbay if (rc) { 530e65e175bSOded Gabbay dev_err(hdev->dev, 531e65e175bSOded Gabbay "Failed to add memory to VA gen pool for CB mapping\n"); 532e65e175bSOded Gabbay goto err_unreserve_va_block; 533e65e175bSOded Gabbay } 534e65e175bSOded Gabbay 535e65e175bSOded Gabbay return 0; 536e65e175bSOded Gabbay 537e65e175bSOded Gabbay err_unreserve_va_block: 538e65e175bSOded Gabbay hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 539e65e175bSOded Gabbay err_pool_destroy: 540e65e175bSOded Gabbay gen_pool_destroy(ctx->cb_va_pool); 541e65e175bSOded Gabbay 542e65e175bSOded Gabbay return rc; 543e65e175bSOded Gabbay } 544e65e175bSOded Gabbay 545e65e175bSOded Gabbay void hl_cb_va_pool_fini(struct hl_ctx *ctx) 546e65e175bSOded Gabbay { 547e65e175bSOded Gabbay struct hl_device *hdev = ctx->hdev; 548e65e175bSOded Gabbay 549e65e175bSOded Gabbay if (!hdev->supports_cb_mapping) 550e65e175bSOded Gabbay return; 551e65e175bSOded Gabbay 552e65e175bSOded Gabbay gen_pool_destroy(ctx->cb_va_pool); 553e65e175bSOded Gabbay hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); 554e65e175bSOded Gabbay } 555