1ff13be83SJeffrey Hugo // SPDX-License-Identifier: GPL-2.0-only 2ff13be83SJeffrey Hugo 3ff13be83SJeffrey Hugo /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ 4ff13be83SJeffrey Hugo /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ 5ff13be83SJeffrey Hugo 6ff13be83SJeffrey Hugo #include <linux/bitfield.h> 7ff13be83SJeffrey Hugo #include <linux/bits.h> 8ff13be83SJeffrey Hugo #include <linux/completion.h> 9ff13be83SJeffrey Hugo #include <linux/delay.h> 10ff13be83SJeffrey Hugo #include <linux/dma-buf.h> 11ff13be83SJeffrey Hugo #include <linux/dma-mapping.h> 12ff13be83SJeffrey Hugo #include <linux/interrupt.h> 13ff13be83SJeffrey Hugo #include <linux/kref.h> 14ff13be83SJeffrey Hugo #include <linux/list.h> 15ff13be83SJeffrey Hugo #include <linux/math64.h> 16ff13be83SJeffrey Hugo #include <linux/mm.h> 17ff13be83SJeffrey Hugo #include <linux/moduleparam.h> 18ff13be83SJeffrey Hugo #include <linux/scatterlist.h> 19ff13be83SJeffrey Hugo #include <linux/spinlock.h> 20ff13be83SJeffrey Hugo #include <linux/srcu.h> 21ff13be83SJeffrey Hugo #include <linux/types.h> 22ff13be83SJeffrey Hugo #include <linux/uaccess.h> 23ff13be83SJeffrey Hugo #include <linux/wait.h> 24ff13be83SJeffrey Hugo #include <drm/drm_file.h> 25ff13be83SJeffrey Hugo #include <drm/drm_gem.h> 26ff13be83SJeffrey Hugo #include <drm/drm_print.h> 27ff13be83SJeffrey Hugo #include <uapi/drm/qaic_accel.h> 28ff13be83SJeffrey Hugo 29ff13be83SJeffrey Hugo #include "qaic.h" 30ff13be83SJeffrey Hugo 31ff13be83SJeffrey Hugo #define SEM_VAL_MASK GENMASK_ULL(11, 0) 32ff13be83SJeffrey Hugo #define SEM_INDEX_MASK GENMASK_ULL(4, 0) 33ff13be83SJeffrey Hugo #define BULK_XFER BIT(3) 34ff13be83SJeffrey Hugo #define GEN_COMPLETION BIT(4) 35ff13be83SJeffrey Hugo #define INBOUND_XFER 1 36ff13be83SJeffrey Hugo #define OUTBOUND_XFER 2 37ff13be83SJeffrey Hugo #define REQHP_OFF 0x0 /* we read this */ 38ff13be83SJeffrey Hugo #define REQTP_OFF 0x4 /* we write this */ 39ff13be83SJeffrey Hugo #define RSPHP_OFF 0x8 /* we write this */ 40ff13be83SJeffrey Hugo #define RSPTP_OFF 0xc /* we read this */ 41ff13be83SJeffrey Hugo 42ff13be83SJeffrey Hugo #define ENCODE_SEM(val, index, sync, cmd, flags) \ 43ff13be83SJeffrey Hugo ({ \ 44ff13be83SJeffrey Hugo FIELD_PREP(GENMASK(11, 0), (val)) | \ 45ff13be83SJeffrey Hugo FIELD_PREP(GENMASK(20, 16), (index)) | \ 46ff13be83SJeffrey Hugo FIELD_PREP(BIT(22), (sync)) | \ 47ff13be83SJeffrey Hugo FIELD_PREP(GENMASK(26, 24), (cmd)) | \ 48ff13be83SJeffrey Hugo FIELD_PREP(GENMASK(30, 29), (flags)) | \ 49ff13be83SJeffrey Hugo FIELD_PREP(BIT(31), (cmd) ? 1 : 0); \ 50ff13be83SJeffrey Hugo }) 51ff13be83SJeffrey Hugo #define NUM_EVENTS 128 52ff13be83SJeffrey Hugo #define NUM_DELAYS 10 53ff13be83SJeffrey Hugo 54ff13be83SJeffrey Hugo static unsigned int wait_exec_default_timeout_ms = 5000; /* 5 sec default */ 55ff13be83SJeffrey Hugo module_param(wait_exec_default_timeout_ms, uint, 0600); 56ff13be83SJeffrey Hugo MODULE_PARM_DESC(wait_exec_default_timeout_ms, "Default timeout for DRM_IOCTL_QAIC_WAIT_BO"); 57ff13be83SJeffrey Hugo 58ff13be83SJeffrey Hugo static unsigned int datapath_poll_interval_us = 100; /* 100 usec default */ 59ff13be83SJeffrey Hugo module_param(datapath_poll_interval_us, uint, 0600); 60ff13be83SJeffrey Hugo MODULE_PARM_DESC(datapath_poll_interval_us, 61ff13be83SJeffrey Hugo "Amount of time to sleep between activity when datapath polling is enabled"); 62ff13be83SJeffrey Hugo 63ff13be83SJeffrey Hugo struct dbc_req { 64ff13be83SJeffrey Hugo /* 65ff13be83SJeffrey Hugo * A request ID is assigned to each memory handle going in DMA queue. 66ff13be83SJeffrey Hugo * As a single memory handle can enqueue multiple elements in DMA queue 67ff13be83SJeffrey Hugo * all of them will have the same request ID. 68ff13be83SJeffrey Hugo */ 69ff13be83SJeffrey Hugo __le16 req_id; 70ff13be83SJeffrey Hugo /* Future use */ 71ff13be83SJeffrey Hugo __u8 seq_id; 72ff13be83SJeffrey Hugo /* 73ff13be83SJeffrey Hugo * Special encoded variable 74ff13be83SJeffrey Hugo * 7 0 - Do not force to generate MSI after DMA is completed 75ff13be83SJeffrey Hugo * 1 - Force to generate MSI after DMA is completed 76ff13be83SJeffrey Hugo * 6:5 Reserved 77ff13be83SJeffrey Hugo * 4 1 - Generate completion element in the response queue 78ff13be83SJeffrey Hugo * 0 - No Completion Code 79ff13be83SJeffrey Hugo * 3 0 - DMA request is a Link list transfer 80ff13be83SJeffrey Hugo * 1 - DMA request is a Bulk transfer 81ff13be83SJeffrey Hugo * 2 Reserved 82ff13be83SJeffrey Hugo * 1:0 00 - No DMA transfer involved 83ff13be83SJeffrey Hugo * 01 - DMA transfer is part of inbound transfer 84ff13be83SJeffrey Hugo * 10 - DMA transfer has outbound transfer 85ff13be83SJeffrey Hugo * 11 - NA 86ff13be83SJeffrey Hugo */ 87ff13be83SJeffrey Hugo __u8 cmd; 88ff13be83SJeffrey Hugo __le32 resv; 89ff13be83SJeffrey Hugo /* Source address for the transfer */ 90ff13be83SJeffrey Hugo __le64 src_addr; 91ff13be83SJeffrey Hugo /* Destination address for the transfer */ 92ff13be83SJeffrey Hugo __le64 dest_addr; 93ff13be83SJeffrey Hugo /* Length of transfer request */ 94ff13be83SJeffrey Hugo __le32 len; 95ff13be83SJeffrey Hugo __le32 resv2; 96ff13be83SJeffrey Hugo /* Doorbell address */ 97ff13be83SJeffrey Hugo __le64 db_addr; 98ff13be83SJeffrey Hugo /* 99ff13be83SJeffrey Hugo * Special encoded variable 100ff13be83SJeffrey Hugo * 7 1 - Doorbell(db) write 101ff13be83SJeffrey Hugo * 0 - No doorbell write 102ff13be83SJeffrey Hugo * 6:2 Reserved 103ff13be83SJeffrey Hugo * 1:0 00 - 32 bit access, db address must be aligned to 32bit-boundary 104ff13be83SJeffrey Hugo * 01 - 16 bit access, db address must be aligned to 16bit-boundary 105ff13be83SJeffrey Hugo * 10 - 8 bit access, db address must be aligned to 8bit-boundary 106ff13be83SJeffrey Hugo * 11 - Reserved 107ff13be83SJeffrey Hugo */ 108ff13be83SJeffrey Hugo __u8 db_len; 109ff13be83SJeffrey Hugo __u8 resv3; 110ff13be83SJeffrey Hugo __le16 resv4; 111ff13be83SJeffrey Hugo /* 32 bit data written to doorbell address */ 112ff13be83SJeffrey Hugo __le32 db_data; 113ff13be83SJeffrey Hugo /* 114ff13be83SJeffrey Hugo * Special encoded variable 115ff13be83SJeffrey Hugo * All the fields of sem_cmdX are passed from user and all are ORed 116ff13be83SJeffrey Hugo * together to form sem_cmd. 117ff13be83SJeffrey Hugo * 0:11 Semaphore value 118ff13be83SJeffrey Hugo * 15:12 Reserved 119ff13be83SJeffrey Hugo * 20:16 Semaphore index 120ff13be83SJeffrey Hugo * 21 Reserved 121ff13be83SJeffrey Hugo * 22 Semaphore Sync 122ff13be83SJeffrey Hugo * 23 Reserved 123ff13be83SJeffrey Hugo * 26:24 Semaphore command 124ff13be83SJeffrey Hugo * 28:27 Reserved 125ff13be83SJeffrey Hugo * 29 Semaphore DMA out bound sync fence 126ff13be83SJeffrey Hugo * 30 Semaphore DMA in bound sync fence 127ff13be83SJeffrey Hugo * 31 Enable semaphore command 128ff13be83SJeffrey Hugo */ 129ff13be83SJeffrey Hugo __le32 sem_cmd0; 130ff13be83SJeffrey Hugo __le32 sem_cmd1; 131ff13be83SJeffrey Hugo __le32 sem_cmd2; 132ff13be83SJeffrey Hugo __le32 sem_cmd3; 133ff13be83SJeffrey Hugo } __packed; 134ff13be83SJeffrey Hugo 135ff13be83SJeffrey Hugo struct dbc_rsp { 136ff13be83SJeffrey Hugo /* Request ID of the memory handle whose DMA transaction is completed */ 137ff13be83SJeffrey Hugo __le16 req_id; 138ff13be83SJeffrey Hugo /* Status of the DMA transaction. 0 : Success otherwise failure */ 139ff13be83SJeffrey Hugo __le16 status; 140ff13be83SJeffrey Hugo } __packed; 141ff13be83SJeffrey Hugo 142ff13be83SJeffrey Hugo inline int get_dbc_req_elem_size(void) 143ff13be83SJeffrey Hugo { 144ff13be83SJeffrey Hugo return sizeof(struct dbc_req); 145ff13be83SJeffrey Hugo } 146ff13be83SJeffrey Hugo 147ff13be83SJeffrey Hugo inline int get_dbc_rsp_elem_size(void) 148ff13be83SJeffrey Hugo { 149ff13be83SJeffrey Hugo return sizeof(struct dbc_rsp); 150ff13be83SJeffrey Hugo } 151ff13be83SJeffrey Hugo 152ff13be83SJeffrey Hugo static void free_slice(struct kref *kref) 153ff13be83SJeffrey Hugo { 154ff13be83SJeffrey Hugo struct bo_slice *slice = container_of(kref, struct bo_slice, ref_count); 155ff13be83SJeffrey Hugo 156ff13be83SJeffrey Hugo list_del(&slice->slice); 157ff13be83SJeffrey Hugo drm_gem_object_put(&slice->bo->base); 158ff13be83SJeffrey Hugo sg_free_table(slice->sgt); 159ff13be83SJeffrey Hugo kfree(slice->sgt); 160ff13be83SJeffrey Hugo kfree(slice->reqs); 161ff13be83SJeffrey Hugo kfree(slice); 162ff13be83SJeffrey Hugo } 163ff13be83SJeffrey Hugo 164ff13be83SJeffrey Hugo static int clone_range_of_sgt_for_slice(struct qaic_device *qdev, struct sg_table **sgt_out, 165ff13be83SJeffrey Hugo struct sg_table *sgt_in, u64 size, u64 offset) 166ff13be83SJeffrey Hugo { 167ff13be83SJeffrey Hugo int total_len, len, nents, offf = 0, offl = 0; 168ff13be83SJeffrey Hugo struct scatterlist *sg, *sgn, *sgf, *sgl; 169ff13be83SJeffrey Hugo struct sg_table *sgt; 170ff13be83SJeffrey Hugo int ret, j; 171ff13be83SJeffrey Hugo 172ff13be83SJeffrey Hugo /* find out number of relevant nents needed for this mem */ 173ff13be83SJeffrey Hugo total_len = 0; 174ff13be83SJeffrey Hugo sgf = NULL; 175ff13be83SJeffrey Hugo sgl = NULL; 176ff13be83SJeffrey Hugo nents = 0; 177ff13be83SJeffrey Hugo 178ff13be83SJeffrey Hugo size = size ? size : PAGE_SIZE; 179ff13be83SJeffrey Hugo for (sg = sgt_in->sgl; sg; sg = sg_next(sg)) { 180ff13be83SJeffrey Hugo len = sg_dma_len(sg); 181ff13be83SJeffrey Hugo 182ff13be83SJeffrey Hugo if (!len) 183ff13be83SJeffrey Hugo continue; 184ff13be83SJeffrey Hugo if (offset >= total_len && offset < total_len + len) { 185ff13be83SJeffrey Hugo sgf = sg; 186ff13be83SJeffrey Hugo offf = offset - total_len; 187ff13be83SJeffrey Hugo } 188ff13be83SJeffrey Hugo if (sgf) 189ff13be83SJeffrey Hugo nents++; 190ff13be83SJeffrey Hugo if (offset + size >= total_len && 191ff13be83SJeffrey Hugo offset + size <= total_len + len) { 192ff13be83SJeffrey Hugo sgl = sg; 193ff13be83SJeffrey Hugo offl = offset + size - total_len; 194ff13be83SJeffrey Hugo break; 195ff13be83SJeffrey Hugo } 196ff13be83SJeffrey Hugo total_len += len; 197ff13be83SJeffrey Hugo } 198ff13be83SJeffrey Hugo 199ff13be83SJeffrey Hugo if (!sgf || !sgl) { 200ff13be83SJeffrey Hugo ret = -EINVAL; 201ff13be83SJeffrey Hugo goto out; 202ff13be83SJeffrey Hugo } 203ff13be83SJeffrey Hugo 204ff13be83SJeffrey Hugo sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 205ff13be83SJeffrey Hugo if (!sgt) { 206ff13be83SJeffrey Hugo ret = -ENOMEM; 207ff13be83SJeffrey Hugo goto out; 208ff13be83SJeffrey Hugo } 209ff13be83SJeffrey Hugo 210ff13be83SJeffrey Hugo ret = sg_alloc_table(sgt, nents, GFP_KERNEL); 211ff13be83SJeffrey Hugo if (ret) 212ff13be83SJeffrey Hugo goto free_sgt; 213ff13be83SJeffrey Hugo 214ff13be83SJeffrey Hugo /* copy relevant sg node and fix page and length */ 215ff13be83SJeffrey Hugo sgn = sgf; 216ff13be83SJeffrey Hugo for_each_sgtable_sg(sgt, sg, j) { 217ff13be83SJeffrey Hugo memcpy(sg, sgn, sizeof(*sg)); 218ff13be83SJeffrey Hugo if (sgn == sgf) { 219ff13be83SJeffrey Hugo sg_dma_address(sg) += offf; 220ff13be83SJeffrey Hugo sg_dma_len(sg) -= offf; 221ff13be83SJeffrey Hugo sg_set_page(sg, sg_page(sgn), sg_dma_len(sg), offf); 222ff13be83SJeffrey Hugo } else { 223ff13be83SJeffrey Hugo offf = 0; 224ff13be83SJeffrey Hugo } 225ff13be83SJeffrey Hugo if (sgn == sgl) { 226ff13be83SJeffrey Hugo sg_dma_len(sg) = offl - offf; 227ff13be83SJeffrey Hugo sg_set_page(sg, sg_page(sgn), offl - offf, offf); 228ff13be83SJeffrey Hugo sg_mark_end(sg); 229ff13be83SJeffrey Hugo break; 230ff13be83SJeffrey Hugo } 231ff13be83SJeffrey Hugo sgn = sg_next(sgn); 232ff13be83SJeffrey Hugo } 233ff13be83SJeffrey Hugo 234ff13be83SJeffrey Hugo *sgt_out = sgt; 235ff13be83SJeffrey Hugo return ret; 236ff13be83SJeffrey Hugo 237ff13be83SJeffrey Hugo free_sgt: 238ff13be83SJeffrey Hugo kfree(sgt); 239ff13be83SJeffrey Hugo out: 240ff13be83SJeffrey Hugo *sgt_out = NULL; 241ff13be83SJeffrey Hugo return ret; 242ff13be83SJeffrey Hugo } 243ff13be83SJeffrey Hugo 244ff13be83SJeffrey Hugo static int encode_reqs(struct qaic_device *qdev, struct bo_slice *slice, 245ff13be83SJeffrey Hugo struct qaic_attach_slice_entry *req) 246ff13be83SJeffrey Hugo { 247ff13be83SJeffrey Hugo __le64 db_addr = cpu_to_le64(req->db_addr); 248ff13be83SJeffrey Hugo __le32 db_data = cpu_to_le32(req->db_data); 249ff13be83SJeffrey Hugo struct scatterlist *sg; 250ff13be83SJeffrey Hugo __u8 cmd = BULK_XFER; 251ff13be83SJeffrey Hugo int presync_sem; 252ff13be83SJeffrey Hugo u64 dev_addr; 253ff13be83SJeffrey Hugo __u8 db_len; 254ff13be83SJeffrey Hugo int i; 255ff13be83SJeffrey Hugo 256ff13be83SJeffrey Hugo if (!slice->no_xfer) 257ff13be83SJeffrey Hugo cmd |= (slice->dir == DMA_TO_DEVICE ? INBOUND_XFER : OUTBOUND_XFER); 258ff13be83SJeffrey Hugo 259ff13be83SJeffrey Hugo if (req->db_len && !IS_ALIGNED(req->db_addr, req->db_len / 8)) 260ff13be83SJeffrey Hugo return -EINVAL; 261ff13be83SJeffrey Hugo 262ff13be83SJeffrey Hugo presync_sem = req->sem0.presync + req->sem1.presync + req->sem2.presync + req->sem3.presync; 263ff13be83SJeffrey Hugo if (presync_sem > 1) 264ff13be83SJeffrey Hugo return -EINVAL; 265ff13be83SJeffrey Hugo 266ff13be83SJeffrey Hugo presync_sem = req->sem0.presync << 0 | req->sem1.presync << 1 | 267ff13be83SJeffrey Hugo req->sem2.presync << 2 | req->sem3.presync << 3; 268ff13be83SJeffrey Hugo 269ff13be83SJeffrey Hugo switch (req->db_len) { 270ff13be83SJeffrey Hugo case 32: 271ff13be83SJeffrey Hugo db_len = BIT(7); 272ff13be83SJeffrey Hugo break; 273ff13be83SJeffrey Hugo case 16: 274ff13be83SJeffrey Hugo db_len = BIT(7) | 1; 275ff13be83SJeffrey Hugo break; 276ff13be83SJeffrey Hugo case 8: 277ff13be83SJeffrey Hugo db_len = BIT(7) | 2; 278ff13be83SJeffrey Hugo break; 279ff13be83SJeffrey Hugo case 0: 280ff13be83SJeffrey Hugo db_len = 0; /* doorbell is not active for this command */ 281ff13be83SJeffrey Hugo break; 282ff13be83SJeffrey Hugo default: 283ff13be83SJeffrey Hugo return -EINVAL; /* should never hit this */ 284ff13be83SJeffrey Hugo } 285ff13be83SJeffrey Hugo 286ff13be83SJeffrey Hugo /* 287ff13be83SJeffrey Hugo * When we end up splitting up a single request (ie a buf slice) into 288ff13be83SJeffrey Hugo * multiple DMA requests, we have to manage the sync data carefully. 289ff13be83SJeffrey Hugo * There can only be one presync sem. That needs to be on every xfer 290ff13be83SJeffrey Hugo * so that the DMA engine doesn't transfer data before the receiver is 291ff13be83SJeffrey Hugo * ready. We only do the doorbell and postsync sems after the xfer. 292ff13be83SJeffrey Hugo * To guarantee previous xfers for the request are complete, we use a 293ff13be83SJeffrey Hugo * fence. 294ff13be83SJeffrey Hugo */ 295ff13be83SJeffrey Hugo dev_addr = req->dev_addr; 296ff13be83SJeffrey Hugo for_each_sgtable_sg(slice->sgt, sg, i) { 297ff13be83SJeffrey Hugo slice->reqs[i].cmd = cmd; 298ff13be83SJeffrey Hugo slice->reqs[i].src_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ? 299ff13be83SJeffrey Hugo sg_dma_address(sg) : dev_addr); 300ff13be83SJeffrey Hugo slice->reqs[i].dest_addr = cpu_to_le64(slice->dir == DMA_TO_DEVICE ? 301ff13be83SJeffrey Hugo dev_addr : sg_dma_address(sg)); 302ff13be83SJeffrey Hugo /* 303ff13be83SJeffrey Hugo * sg_dma_len(sg) returns size of a DMA segment, maximum DMA 304ff13be83SJeffrey Hugo * segment size is set to UINT_MAX by qaic and hence return 305ff13be83SJeffrey Hugo * values of sg_dma_len(sg) can never exceed u32 range. So, 306ff13be83SJeffrey Hugo * by down sizing we are not corrupting the value. 307ff13be83SJeffrey Hugo */ 308ff13be83SJeffrey Hugo slice->reqs[i].len = cpu_to_le32((u32)sg_dma_len(sg)); 309ff13be83SJeffrey Hugo switch (presync_sem) { 310ff13be83SJeffrey Hugo case BIT(0): 311ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val, 312ff13be83SJeffrey Hugo req->sem0.index, 313ff13be83SJeffrey Hugo req->sem0.presync, 314ff13be83SJeffrey Hugo req->sem0.cmd, 315ff13be83SJeffrey Hugo req->sem0.flags)); 316ff13be83SJeffrey Hugo break; 317ff13be83SJeffrey Hugo case BIT(1): 318ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val, 319ff13be83SJeffrey Hugo req->sem1.index, 320ff13be83SJeffrey Hugo req->sem1.presync, 321ff13be83SJeffrey Hugo req->sem1.cmd, 322ff13be83SJeffrey Hugo req->sem1.flags)); 323ff13be83SJeffrey Hugo break; 324ff13be83SJeffrey Hugo case BIT(2): 325ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val, 326ff13be83SJeffrey Hugo req->sem2.index, 327ff13be83SJeffrey Hugo req->sem2.presync, 328ff13be83SJeffrey Hugo req->sem2.cmd, 329ff13be83SJeffrey Hugo req->sem2.flags)); 330ff13be83SJeffrey Hugo break; 331ff13be83SJeffrey Hugo case BIT(3): 332ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val, 333ff13be83SJeffrey Hugo req->sem3.index, 334ff13be83SJeffrey Hugo req->sem3.presync, 335ff13be83SJeffrey Hugo req->sem3.cmd, 336ff13be83SJeffrey Hugo req->sem3.flags)); 337ff13be83SJeffrey Hugo break; 338ff13be83SJeffrey Hugo } 339ff13be83SJeffrey Hugo dev_addr += sg_dma_len(sg); 340ff13be83SJeffrey Hugo } 341ff13be83SJeffrey Hugo /* add post transfer stuff to last segment */ 342ff13be83SJeffrey Hugo i--; 343ff13be83SJeffrey Hugo slice->reqs[i].cmd |= GEN_COMPLETION; 344ff13be83SJeffrey Hugo slice->reqs[i].db_addr = db_addr; 345ff13be83SJeffrey Hugo slice->reqs[i].db_len = db_len; 346ff13be83SJeffrey Hugo slice->reqs[i].db_data = db_data; 347ff13be83SJeffrey Hugo /* 348ff13be83SJeffrey Hugo * Add a fence if we have more than one request going to the hardware 349ff13be83SJeffrey Hugo * representing the entirety of the user request, and the user request 350ff13be83SJeffrey Hugo * has no presync condition. 351ff13be83SJeffrey Hugo * Fences are expensive, so we try to avoid them. We rely on the 352ff13be83SJeffrey Hugo * hardware behavior to avoid needing one when there is a presync 353ff13be83SJeffrey Hugo * condition. When a presync exists, all requests for that same 354ff13be83SJeffrey Hugo * presync will be queued into a fifo. Thus, since we queue the 355ff13be83SJeffrey Hugo * post xfer activity only on the last request we queue, the hardware 356ff13be83SJeffrey Hugo * will ensure that the last queued request is processed last, thus 357ff13be83SJeffrey Hugo * making sure the post xfer activity happens at the right time without 358ff13be83SJeffrey Hugo * a fence. 359ff13be83SJeffrey Hugo */ 360ff13be83SJeffrey Hugo if (i && !presync_sem) 361ff13be83SJeffrey Hugo req->sem0.flags |= (slice->dir == DMA_TO_DEVICE ? 362ff13be83SJeffrey Hugo QAIC_SEM_INSYNCFENCE : QAIC_SEM_OUTSYNCFENCE); 363ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd0 = cpu_to_le32(ENCODE_SEM(req->sem0.val, req->sem0.index, 364ff13be83SJeffrey Hugo req->sem0.presync, req->sem0.cmd, 365ff13be83SJeffrey Hugo req->sem0.flags)); 366ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd1 = cpu_to_le32(ENCODE_SEM(req->sem1.val, req->sem1.index, 367ff13be83SJeffrey Hugo req->sem1.presync, req->sem1.cmd, 368ff13be83SJeffrey Hugo req->sem1.flags)); 369ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd2 = cpu_to_le32(ENCODE_SEM(req->sem2.val, req->sem2.index, 370ff13be83SJeffrey Hugo req->sem2.presync, req->sem2.cmd, 371ff13be83SJeffrey Hugo req->sem2.flags)); 372ff13be83SJeffrey Hugo slice->reqs[i].sem_cmd3 = cpu_to_le32(ENCODE_SEM(req->sem3.val, req->sem3.index, 373ff13be83SJeffrey Hugo req->sem3.presync, req->sem3.cmd, 374ff13be83SJeffrey Hugo req->sem3.flags)); 375ff13be83SJeffrey Hugo 376ff13be83SJeffrey Hugo return 0; 377ff13be83SJeffrey Hugo } 378ff13be83SJeffrey Hugo 379ff13be83SJeffrey Hugo static int qaic_map_one_slice(struct qaic_device *qdev, struct qaic_bo *bo, 380ff13be83SJeffrey Hugo struct qaic_attach_slice_entry *slice_ent) 381ff13be83SJeffrey Hugo { 382ff13be83SJeffrey Hugo struct sg_table *sgt = NULL; 383ff13be83SJeffrey Hugo struct bo_slice *slice; 384ff13be83SJeffrey Hugo int ret; 385ff13be83SJeffrey Hugo 386ff13be83SJeffrey Hugo ret = clone_range_of_sgt_for_slice(qdev, &sgt, bo->sgt, slice_ent->size, slice_ent->offset); 387ff13be83SJeffrey Hugo if (ret) 388ff13be83SJeffrey Hugo goto out; 389ff13be83SJeffrey Hugo 390ff13be83SJeffrey Hugo slice = kmalloc(sizeof(*slice), GFP_KERNEL); 391ff13be83SJeffrey Hugo if (!slice) { 392ff13be83SJeffrey Hugo ret = -ENOMEM; 393ff13be83SJeffrey Hugo goto free_sgt; 394ff13be83SJeffrey Hugo } 395ff13be83SJeffrey Hugo 396ff13be83SJeffrey Hugo slice->reqs = kcalloc(sgt->nents, sizeof(*slice->reqs), GFP_KERNEL); 397ff13be83SJeffrey Hugo if (!slice->reqs) { 398ff13be83SJeffrey Hugo ret = -ENOMEM; 399ff13be83SJeffrey Hugo goto free_slice; 400ff13be83SJeffrey Hugo } 401ff13be83SJeffrey Hugo 402ff13be83SJeffrey Hugo slice->no_xfer = !slice_ent->size; 403ff13be83SJeffrey Hugo slice->sgt = sgt; 404ff13be83SJeffrey Hugo slice->nents = sgt->nents; 405ff13be83SJeffrey Hugo slice->dir = bo->dir; 406ff13be83SJeffrey Hugo slice->bo = bo; 407ff13be83SJeffrey Hugo slice->size = slice_ent->size; 408ff13be83SJeffrey Hugo slice->offset = slice_ent->offset; 409ff13be83SJeffrey Hugo 410ff13be83SJeffrey Hugo ret = encode_reqs(qdev, slice, slice_ent); 411ff13be83SJeffrey Hugo if (ret) 412ff13be83SJeffrey Hugo goto free_req; 413ff13be83SJeffrey Hugo 414ff13be83SJeffrey Hugo bo->total_slice_nents += sgt->nents; 415ff13be83SJeffrey Hugo kref_init(&slice->ref_count); 416ff13be83SJeffrey Hugo drm_gem_object_get(&bo->base); 417ff13be83SJeffrey Hugo list_add_tail(&slice->slice, &bo->slices); 418ff13be83SJeffrey Hugo 419ff13be83SJeffrey Hugo return 0; 420ff13be83SJeffrey Hugo 421ff13be83SJeffrey Hugo free_req: 422ff13be83SJeffrey Hugo kfree(slice->reqs); 423ff13be83SJeffrey Hugo free_slice: 424ff13be83SJeffrey Hugo kfree(slice); 425ff13be83SJeffrey Hugo free_sgt: 426ff13be83SJeffrey Hugo sg_free_table(sgt); 427ff13be83SJeffrey Hugo kfree(sgt); 428ff13be83SJeffrey Hugo out: 429ff13be83SJeffrey Hugo return ret; 430ff13be83SJeffrey Hugo } 431ff13be83SJeffrey Hugo 432ff13be83SJeffrey Hugo static int create_sgt(struct qaic_device *qdev, struct sg_table **sgt_out, u64 size) 433ff13be83SJeffrey Hugo { 434ff13be83SJeffrey Hugo struct scatterlist *sg; 435ff13be83SJeffrey Hugo struct sg_table *sgt; 436ff13be83SJeffrey Hugo struct page **pages; 437ff13be83SJeffrey Hugo int *pages_order; 438ff13be83SJeffrey Hugo int buf_extra; 439ff13be83SJeffrey Hugo int max_order; 440ff13be83SJeffrey Hugo int nr_pages; 441ff13be83SJeffrey Hugo int ret = 0; 442ff13be83SJeffrey Hugo int i, j, k; 443ff13be83SJeffrey Hugo int order; 444ff13be83SJeffrey Hugo 445ff13be83SJeffrey Hugo if (size) { 446ff13be83SJeffrey Hugo nr_pages = DIV_ROUND_UP(size, PAGE_SIZE); 447ff13be83SJeffrey Hugo /* 448ff13be83SJeffrey Hugo * calculate how much extra we are going to allocate, to remove 449ff13be83SJeffrey Hugo * later 450ff13be83SJeffrey Hugo */ 451ff13be83SJeffrey Hugo buf_extra = (PAGE_SIZE - size % PAGE_SIZE) % PAGE_SIZE; 452ff13be83SJeffrey Hugo max_order = min(MAX_ORDER - 1, get_order(size)); 453ff13be83SJeffrey Hugo } else { 454ff13be83SJeffrey Hugo /* allocate a single page for book keeping */ 455ff13be83SJeffrey Hugo nr_pages = 1; 456ff13be83SJeffrey Hugo buf_extra = 0; 457ff13be83SJeffrey Hugo max_order = 0; 458ff13be83SJeffrey Hugo } 459ff13be83SJeffrey Hugo 460ff13be83SJeffrey Hugo pages = kvmalloc_array(nr_pages, sizeof(*pages) + sizeof(*pages_order), GFP_KERNEL); 461ff13be83SJeffrey Hugo if (!pages) { 462ff13be83SJeffrey Hugo ret = -ENOMEM; 463ff13be83SJeffrey Hugo goto out; 464ff13be83SJeffrey Hugo } 465ff13be83SJeffrey Hugo pages_order = (void *)pages + sizeof(*pages) * nr_pages; 466ff13be83SJeffrey Hugo 467ff13be83SJeffrey Hugo /* 468ff13be83SJeffrey Hugo * Allocate requested memory using alloc_pages. It is possible to allocate 469ff13be83SJeffrey Hugo * the requested memory in multiple chunks by calling alloc_pages 470ff13be83SJeffrey Hugo * multiple times. Use SG table to handle multiple allocated pages. 471ff13be83SJeffrey Hugo */ 472ff13be83SJeffrey Hugo i = 0; 473ff13be83SJeffrey Hugo while (nr_pages > 0) { 474ff13be83SJeffrey Hugo order = min(get_order(nr_pages * PAGE_SIZE), max_order); 475ff13be83SJeffrey Hugo while (1) { 476ff13be83SJeffrey Hugo pages[i] = alloc_pages(GFP_KERNEL | GFP_HIGHUSER | 477ff13be83SJeffrey Hugo __GFP_NOWARN | __GFP_ZERO | 478ff13be83SJeffrey Hugo (order ? __GFP_NORETRY : __GFP_RETRY_MAYFAIL), 479ff13be83SJeffrey Hugo order); 480ff13be83SJeffrey Hugo if (pages[i]) 481ff13be83SJeffrey Hugo break; 482ff13be83SJeffrey Hugo if (!order--) { 483ff13be83SJeffrey Hugo ret = -ENOMEM; 484ff13be83SJeffrey Hugo goto free_partial_alloc; 485ff13be83SJeffrey Hugo } 486ff13be83SJeffrey Hugo } 487ff13be83SJeffrey Hugo 488ff13be83SJeffrey Hugo max_order = order; 489ff13be83SJeffrey Hugo pages_order[i] = order; 490ff13be83SJeffrey Hugo 491ff13be83SJeffrey Hugo nr_pages -= 1 << order; 492ff13be83SJeffrey Hugo if (nr_pages <= 0) 493ff13be83SJeffrey Hugo /* account for over allocation */ 494ff13be83SJeffrey Hugo buf_extra += abs(nr_pages) * PAGE_SIZE; 495ff13be83SJeffrey Hugo i++; 496ff13be83SJeffrey Hugo } 497ff13be83SJeffrey Hugo 498ff13be83SJeffrey Hugo sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); 499ff13be83SJeffrey Hugo if (!sgt) { 500ff13be83SJeffrey Hugo ret = -ENOMEM; 501ff13be83SJeffrey Hugo goto free_partial_alloc; 502ff13be83SJeffrey Hugo } 503ff13be83SJeffrey Hugo 504ff13be83SJeffrey Hugo if (sg_alloc_table(sgt, i, GFP_KERNEL)) { 505ff13be83SJeffrey Hugo ret = -ENOMEM; 506ff13be83SJeffrey Hugo goto free_sgt; 507ff13be83SJeffrey Hugo } 508ff13be83SJeffrey Hugo 509ff13be83SJeffrey Hugo /* Populate the SG table with the allocated memory pages */ 510ff13be83SJeffrey Hugo sg = sgt->sgl; 511ff13be83SJeffrey Hugo for (k = 0; k < i; k++, sg = sg_next(sg)) { 512ff13be83SJeffrey Hugo /* Last entry requires special handling */ 513ff13be83SJeffrey Hugo if (k < i - 1) { 514ff13be83SJeffrey Hugo sg_set_page(sg, pages[k], PAGE_SIZE << pages_order[k], 0); 515ff13be83SJeffrey Hugo } else { 516ff13be83SJeffrey Hugo sg_set_page(sg, pages[k], (PAGE_SIZE << pages_order[k]) - buf_extra, 0); 517ff13be83SJeffrey Hugo sg_mark_end(sg); 518ff13be83SJeffrey Hugo } 519ff13be83SJeffrey Hugo } 520ff13be83SJeffrey Hugo 521ff13be83SJeffrey Hugo kvfree(pages); 522ff13be83SJeffrey Hugo *sgt_out = sgt; 523ff13be83SJeffrey Hugo return ret; 524ff13be83SJeffrey Hugo 525ff13be83SJeffrey Hugo free_sgt: 526ff13be83SJeffrey Hugo kfree(sgt); 527ff13be83SJeffrey Hugo free_partial_alloc: 528ff13be83SJeffrey Hugo for (j = 0; j < i; j++) 529ff13be83SJeffrey Hugo __free_pages(pages[j], pages_order[j]); 530ff13be83SJeffrey Hugo kvfree(pages); 531ff13be83SJeffrey Hugo out: 532ff13be83SJeffrey Hugo *sgt_out = NULL; 533ff13be83SJeffrey Hugo return ret; 534ff13be83SJeffrey Hugo } 535ff13be83SJeffrey Hugo 536ff13be83SJeffrey Hugo static bool invalid_sem(struct qaic_sem *sem) 537ff13be83SJeffrey Hugo { 538ff13be83SJeffrey Hugo if (sem->val & ~SEM_VAL_MASK || sem->index & ~SEM_INDEX_MASK || 539ff13be83SJeffrey Hugo !(sem->presync == 0 || sem->presync == 1) || sem->pad || 540ff13be83SJeffrey Hugo sem->flags & ~(QAIC_SEM_INSYNCFENCE | QAIC_SEM_OUTSYNCFENCE) || 541ff13be83SJeffrey Hugo sem->cmd > QAIC_SEM_WAIT_GT_0) 542ff13be83SJeffrey Hugo return true; 543ff13be83SJeffrey Hugo return false; 544ff13be83SJeffrey Hugo } 545ff13be83SJeffrey Hugo 546ff13be83SJeffrey Hugo static int qaic_validate_req(struct qaic_device *qdev, struct qaic_attach_slice_entry *slice_ent, 547ff13be83SJeffrey Hugo u32 count, u64 total_size) 548ff13be83SJeffrey Hugo { 549ff13be83SJeffrey Hugo int i; 550ff13be83SJeffrey Hugo 551ff13be83SJeffrey Hugo for (i = 0; i < count; i++) { 552ff13be83SJeffrey Hugo if (!(slice_ent[i].db_len == 32 || slice_ent[i].db_len == 16 || 553ff13be83SJeffrey Hugo slice_ent[i].db_len == 8 || slice_ent[i].db_len == 0) || 554ff13be83SJeffrey Hugo invalid_sem(&slice_ent[i].sem0) || invalid_sem(&slice_ent[i].sem1) || 555ff13be83SJeffrey Hugo invalid_sem(&slice_ent[i].sem2) || invalid_sem(&slice_ent[i].sem3)) 556ff13be83SJeffrey Hugo return -EINVAL; 557ff13be83SJeffrey Hugo 558ff13be83SJeffrey Hugo if (slice_ent[i].offset + slice_ent[i].size > total_size) 559ff13be83SJeffrey Hugo return -EINVAL; 560ff13be83SJeffrey Hugo } 561ff13be83SJeffrey Hugo 562ff13be83SJeffrey Hugo return 0; 563ff13be83SJeffrey Hugo } 564ff13be83SJeffrey Hugo 565ff13be83SJeffrey Hugo static void qaic_free_sgt(struct sg_table *sgt) 566ff13be83SJeffrey Hugo { 567ff13be83SJeffrey Hugo struct scatterlist *sg; 568ff13be83SJeffrey Hugo 569ff13be83SJeffrey Hugo for (sg = sgt->sgl; sg; sg = sg_next(sg)) 570ff13be83SJeffrey Hugo if (sg_page(sg)) 571ff13be83SJeffrey Hugo __free_pages(sg_page(sg), get_order(sg->length)); 572ff13be83SJeffrey Hugo sg_free_table(sgt); 573ff13be83SJeffrey Hugo kfree(sgt); 574ff13be83SJeffrey Hugo } 575ff13be83SJeffrey Hugo 576ff13be83SJeffrey Hugo static void qaic_gem_print_info(struct drm_printer *p, unsigned int indent, 577ff13be83SJeffrey Hugo const struct drm_gem_object *obj) 578ff13be83SJeffrey Hugo { 579ff13be83SJeffrey Hugo struct qaic_bo *bo = to_qaic_bo(obj); 580ff13be83SJeffrey Hugo 581ff13be83SJeffrey Hugo drm_printf_indent(p, indent, "user requested size=%llu\n", bo->size); 582ff13be83SJeffrey Hugo } 583ff13be83SJeffrey Hugo 584ff13be83SJeffrey Hugo static const struct vm_operations_struct drm_vm_ops = { 585ff13be83SJeffrey Hugo .open = drm_gem_vm_open, 586ff13be83SJeffrey Hugo .close = drm_gem_vm_close, 587ff13be83SJeffrey Hugo }; 588ff13be83SJeffrey Hugo 589ff13be83SJeffrey Hugo static int qaic_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 590ff13be83SJeffrey Hugo { 591ff13be83SJeffrey Hugo struct qaic_bo *bo = to_qaic_bo(obj); 592ff13be83SJeffrey Hugo unsigned long offset = 0; 593ff13be83SJeffrey Hugo struct scatterlist *sg; 594*0e163e54STom Rix int ret = 0; 595ff13be83SJeffrey Hugo 596ff13be83SJeffrey Hugo if (obj->import_attach) 597ff13be83SJeffrey Hugo return -EINVAL; 598ff13be83SJeffrey Hugo 599ff13be83SJeffrey Hugo for (sg = bo->sgt->sgl; sg; sg = sg_next(sg)) { 600ff13be83SJeffrey Hugo if (sg_page(sg)) { 601ff13be83SJeffrey Hugo ret = remap_pfn_range(vma, vma->vm_start + offset, page_to_pfn(sg_page(sg)), 602ff13be83SJeffrey Hugo sg->length, vma->vm_page_prot); 603ff13be83SJeffrey Hugo if (ret) 604ff13be83SJeffrey Hugo goto out; 605ff13be83SJeffrey Hugo offset += sg->length; 606ff13be83SJeffrey Hugo } 607ff13be83SJeffrey Hugo } 608ff13be83SJeffrey Hugo 609ff13be83SJeffrey Hugo out: 610ff13be83SJeffrey Hugo return ret; 611ff13be83SJeffrey Hugo } 612ff13be83SJeffrey Hugo 613ff13be83SJeffrey Hugo static void qaic_free_object(struct drm_gem_object *obj) 614ff13be83SJeffrey Hugo { 615ff13be83SJeffrey Hugo struct qaic_bo *bo = to_qaic_bo(obj); 616ff13be83SJeffrey Hugo 617ff13be83SJeffrey Hugo if (obj->import_attach) { 618ff13be83SJeffrey Hugo /* DMABUF/PRIME Path */ 619ff13be83SJeffrey Hugo dma_buf_detach(obj->import_attach->dmabuf, obj->import_attach); 620ff13be83SJeffrey Hugo dma_buf_put(obj->import_attach->dmabuf); 621ff13be83SJeffrey Hugo } else { 622ff13be83SJeffrey Hugo /* Private buffer allocation path */ 623ff13be83SJeffrey Hugo qaic_free_sgt(bo->sgt); 624ff13be83SJeffrey Hugo } 625ff13be83SJeffrey Hugo 626ff13be83SJeffrey Hugo drm_gem_object_release(obj); 627ff13be83SJeffrey Hugo kfree(bo); 628ff13be83SJeffrey Hugo } 629ff13be83SJeffrey Hugo 630ff13be83SJeffrey Hugo static const struct drm_gem_object_funcs qaic_gem_funcs = { 631ff13be83SJeffrey Hugo .free = qaic_free_object, 632ff13be83SJeffrey Hugo .print_info = qaic_gem_print_info, 633ff13be83SJeffrey Hugo .mmap = qaic_gem_object_mmap, 634ff13be83SJeffrey Hugo .vm_ops = &drm_vm_ops, 635ff13be83SJeffrey Hugo }; 636ff13be83SJeffrey Hugo 637ff13be83SJeffrey Hugo static struct qaic_bo *qaic_alloc_init_bo(void) 638ff13be83SJeffrey Hugo { 639ff13be83SJeffrey Hugo struct qaic_bo *bo; 640ff13be83SJeffrey Hugo 641ff13be83SJeffrey Hugo bo = kzalloc(sizeof(*bo), GFP_KERNEL); 642ff13be83SJeffrey Hugo if (!bo) 643ff13be83SJeffrey Hugo return ERR_PTR(-ENOMEM); 644ff13be83SJeffrey Hugo 645ff13be83SJeffrey Hugo INIT_LIST_HEAD(&bo->slices); 646ff13be83SJeffrey Hugo init_completion(&bo->xfer_done); 647ff13be83SJeffrey Hugo complete_all(&bo->xfer_done); 648ff13be83SJeffrey Hugo 649ff13be83SJeffrey Hugo return bo; 650ff13be83SJeffrey Hugo } 651ff13be83SJeffrey Hugo 652ff13be83SJeffrey Hugo int qaic_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 653ff13be83SJeffrey Hugo { 654ff13be83SJeffrey Hugo struct qaic_create_bo *args = data; 655ff13be83SJeffrey Hugo int usr_rcu_id, qdev_rcu_id; 656ff13be83SJeffrey Hugo struct drm_gem_object *obj; 657ff13be83SJeffrey Hugo struct qaic_device *qdev; 658ff13be83SJeffrey Hugo struct qaic_user *usr; 659ff13be83SJeffrey Hugo struct qaic_bo *bo; 660ff13be83SJeffrey Hugo size_t size; 661ff13be83SJeffrey Hugo int ret; 662ff13be83SJeffrey Hugo 663ff13be83SJeffrey Hugo if (args->pad) 664ff13be83SJeffrey Hugo return -EINVAL; 665ff13be83SJeffrey Hugo 666ff13be83SJeffrey Hugo usr = file_priv->driver_priv; 667ff13be83SJeffrey Hugo usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 668ff13be83SJeffrey Hugo if (!usr->qddev) { 669ff13be83SJeffrey Hugo ret = -ENODEV; 670ff13be83SJeffrey Hugo goto unlock_usr_srcu; 671ff13be83SJeffrey Hugo } 672ff13be83SJeffrey Hugo 673ff13be83SJeffrey Hugo qdev = usr->qddev->qdev; 674ff13be83SJeffrey Hugo qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 675ff13be83SJeffrey Hugo if (qdev->in_reset) { 676ff13be83SJeffrey Hugo ret = -ENODEV; 677ff13be83SJeffrey Hugo goto unlock_dev_srcu; 678ff13be83SJeffrey Hugo } 679ff13be83SJeffrey Hugo 680ff13be83SJeffrey Hugo size = PAGE_ALIGN(args->size); 681ff13be83SJeffrey Hugo if (size == 0) { 682ff13be83SJeffrey Hugo ret = -EINVAL; 683ff13be83SJeffrey Hugo goto unlock_dev_srcu; 684ff13be83SJeffrey Hugo } 685ff13be83SJeffrey Hugo 686ff13be83SJeffrey Hugo bo = qaic_alloc_init_bo(); 687ff13be83SJeffrey Hugo if (IS_ERR(bo)) { 688ff13be83SJeffrey Hugo ret = PTR_ERR(bo); 689ff13be83SJeffrey Hugo goto unlock_dev_srcu; 690ff13be83SJeffrey Hugo } 691ff13be83SJeffrey Hugo obj = &bo->base; 692ff13be83SJeffrey Hugo 693ff13be83SJeffrey Hugo drm_gem_private_object_init(dev, obj, size); 694ff13be83SJeffrey Hugo 695ff13be83SJeffrey Hugo obj->funcs = &qaic_gem_funcs; 696ff13be83SJeffrey Hugo ret = create_sgt(qdev, &bo->sgt, size); 697ff13be83SJeffrey Hugo if (ret) 698ff13be83SJeffrey Hugo goto free_bo; 699ff13be83SJeffrey Hugo 700ff13be83SJeffrey Hugo bo->size = args->size; 701ff13be83SJeffrey Hugo 702ff13be83SJeffrey Hugo ret = drm_gem_handle_create(file_priv, obj, &args->handle); 703ff13be83SJeffrey Hugo if (ret) 704ff13be83SJeffrey Hugo goto free_sgt; 705ff13be83SJeffrey Hugo 706ff13be83SJeffrey Hugo bo->handle = args->handle; 707ff13be83SJeffrey Hugo drm_gem_object_put(obj); 708ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 709ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 710ff13be83SJeffrey Hugo 711ff13be83SJeffrey Hugo return 0; 712ff13be83SJeffrey Hugo 713ff13be83SJeffrey Hugo free_sgt: 714ff13be83SJeffrey Hugo qaic_free_sgt(bo->sgt); 715ff13be83SJeffrey Hugo free_bo: 716ff13be83SJeffrey Hugo kfree(bo); 717ff13be83SJeffrey Hugo unlock_dev_srcu: 718ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 719ff13be83SJeffrey Hugo unlock_usr_srcu: 720ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 721ff13be83SJeffrey Hugo return ret; 722ff13be83SJeffrey Hugo } 723ff13be83SJeffrey Hugo 724ff13be83SJeffrey Hugo int qaic_mmap_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 725ff13be83SJeffrey Hugo { 726ff13be83SJeffrey Hugo struct qaic_mmap_bo *args = data; 727ff13be83SJeffrey Hugo int usr_rcu_id, qdev_rcu_id; 728ff13be83SJeffrey Hugo struct drm_gem_object *obj; 729ff13be83SJeffrey Hugo struct qaic_device *qdev; 730ff13be83SJeffrey Hugo struct qaic_user *usr; 731ff13be83SJeffrey Hugo int ret; 732ff13be83SJeffrey Hugo 733ff13be83SJeffrey Hugo usr = file_priv->driver_priv; 734ff13be83SJeffrey Hugo usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 735ff13be83SJeffrey Hugo if (!usr->qddev) { 736ff13be83SJeffrey Hugo ret = -ENODEV; 737ff13be83SJeffrey Hugo goto unlock_usr_srcu; 738ff13be83SJeffrey Hugo } 739ff13be83SJeffrey Hugo 740ff13be83SJeffrey Hugo qdev = usr->qddev->qdev; 741ff13be83SJeffrey Hugo qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 742ff13be83SJeffrey Hugo if (qdev->in_reset) { 743ff13be83SJeffrey Hugo ret = -ENODEV; 744ff13be83SJeffrey Hugo goto unlock_dev_srcu; 745ff13be83SJeffrey Hugo } 746ff13be83SJeffrey Hugo 747ff13be83SJeffrey Hugo obj = drm_gem_object_lookup(file_priv, args->handle); 748ff13be83SJeffrey Hugo if (!obj) { 749ff13be83SJeffrey Hugo ret = -ENOENT; 750ff13be83SJeffrey Hugo goto unlock_dev_srcu; 751ff13be83SJeffrey Hugo } 752ff13be83SJeffrey Hugo 753ff13be83SJeffrey Hugo ret = drm_gem_create_mmap_offset(obj); 754ff13be83SJeffrey Hugo if (ret == 0) 755ff13be83SJeffrey Hugo args->offset = drm_vma_node_offset_addr(&obj->vma_node); 756ff13be83SJeffrey Hugo 757ff13be83SJeffrey Hugo drm_gem_object_put(obj); 758ff13be83SJeffrey Hugo 759ff13be83SJeffrey Hugo unlock_dev_srcu: 760ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 761ff13be83SJeffrey Hugo unlock_usr_srcu: 762ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 763ff13be83SJeffrey Hugo return ret; 764ff13be83SJeffrey Hugo } 765ff13be83SJeffrey Hugo 766ff13be83SJeffrey Hugo struct drm_gem_object *qaic_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) 767ff13be83SJeffrey Hugo { 768ff13be83SJeffrey Hugo struct dma_buf_attachment *attach; 769ff13be83SJeffrey Hugo struct drm_gem_object *obj; 770ff13be83SJeffrey Hugo struct qaic_bo *bo; 771ff13be83SJeffrey Hugo size_t size; 772ff13be83SJeffrey Hugo int ret; 773ff13be83SJeffrey Hugo 774ff13be83SJeffrey Hugo bo = qaic_alloc_init_bo(); 775ff13be83SJeffrey Hugo if (IS_ERR(bo)) { 776ff13be83SJeffrey Hugo ret = PTR_ERR(bo); 777ff13be83SJeffrey Hugo goto out; 778ff13be83SJeffrey Hugo } 779ff13be83SJeffrey Hugo 780ff13be83SJeffrey Hugo obj = &bo->base; 781ff13be83SJeffrey Hugo get_dma_buf(dma_buf); 782ff13be83SJeffrey Hugo 783ff13be83SJeffrey Hugo attach = dma_buf_attach(dma_buf, dev->dev); 784ff13be83SJeffrey Hugo if (IS_ERR(attach)) { 785ff13be83SJeffrey Hugo ret = PTR_ERR(attach); 786ff13be83SJeffrey Hugo goto attach_fail; 787ff13be83SJeffrey Hugo } 788ff13be83SJeffrey Hugo 789ff13be83SJeffrey Hugo size = PAGE_ALIGN(attach->dmabuf->size); 790ff13be83SJeffrey Hugo if (size == 0) { 791ff13be83SJeffrey Hugo ret = -EINVAL; 792ff13be83SJeffrey Hugo goto size_align_fail; 793ff13be83SJeffrey Hugo } 794ff13be83SJeffrey Hugo 795ff13be83SJeffrey Hugo drm_gem_private_object_init(dev, obj, size); 796ff13be83SJeffrey Hugo /* 797ff13be83SJeffrey Hugo * skipping dma_buf_map_attachment() as we do not know the direction 798ff13be83SJeffrey Hugo * just yet. Once the direction is known in the subsequent IOCTL to 799ff13be83SJeffrey Hugo * attach slicing, we can do it then. 800ff13be83SJeffrey Hugo */ 801ff13be83SJeffrey Hugo 802ff13be83SJeffrey Hugo obj->funcs = &qaic_gem_funcs; 803ff13be83SJeffrey Hugo obj->import_attach = attach; 804ff13be83SJeffrey Hugo obj->resv = dma_buf->resv; 805ff13be83SJeffrey Hugo 806ff13be83SJeffrey Hugo return obj; 807ff13be83SJeffrey Hugo 808ff13be83SJeffrey Hugo size_align_fail: 809ff13be83SJeffrey Hugo dma_buf_detach(dma_buf, attach); 810ff13be83SJeffrey Hugo attach_fail: 811ff13be83SJeffrey Hugo dma_buf_put(dma_buf); 812ff13be83SJeffrey Hugo kfree(bo); 813ff13be83SJeffrey Hugo out: 814ff13be83SJeffrey Hugo return ERR_PTR(ret); 815ff13be83SJeffrey Hugo } 816ff13be83SJeffrey Hugo 817ff13be83SJeffrey Hugo static int qaic_prepare_import_bo(struct qaic_bo *bo, struct qaic_attach_slice_hdr *hdr) 818ff13be83SJeffrey Hugo { 819ff13be83SJeffrey Hugo struct drm_gem_object *obj = &bo->base; 820ff13be83SJeffrey Hugo struct sg_table *sgt; 821ff13be83SJeffrey Hugo int ret; 822ff13be83SJeffrey Hugo 823ff13be83SJeffrey Hugo if (obj->import_attach->dmabuf->size < hdr->size) 824ff13be83SJeffrey Hugo return -EINVAL; 825ff13be83SJeffrey Hugo 826ff13be83SJeffrey Hugo sgt = dma_buf_map_attachment(obj->import_attach, hdr->dir); 827ff13be83SJeffrey Hugo if (IS_ERR(sgt)) { 828ff13be83SJeffrey Hugo ret = PTR_ERR(sgt); 829ff13be83SJeffrey Hugo return ret; 830ff13be83SJeffrey Hugo } 831ff13be83SJeffrey Hugo 832ff13be83SJeffrey Hugo bo->sgt = sgt; 833ff13be83SJeffrey Hugo bo->size = hdr->size; 834ff13be83SJeffrey Hugo 835ff13be83SJeffrey Hugo return 0; 836ff13be83SJeffrey Hugo } 837ff13be83SJeffrey Hugo 838ff13be83SJeffrey Hugo static int qaic_prepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo, 839ff13be83SJeffrey Hugo struct qaic_attach_slice_hdr *hdr) 840ff13be83SJeffrey Hugo { 841ff13be83SJeffrey Hugo int ret; 842ff13be83SJeffrey Hugo 843ff13be83SJeffrey Hugo if (bo->size != hdr->size) 844ff13be83SJeffrey Hugo return -EINVAL; 845ff13be83SJeffrey Hugo 846ff13be83SJeffrey Hugo ret = dma_map_sgtable(&qdev->pdev->dev, bo->sgt, hdr->dir, 0); 847ff13be83SJeffrey Hugo if (ret) 848ff13be83SJeffrey Hugo return -EFAULT; 849ff13be83SJeffrey Hugo 850ff13be83SJeffrey Hugo return 0; 851ff13be83SJeffrey Hugo } 852ff13be83SJeffrey Hugo 853ff13be83SJeffrey Hugo static int qaic_prepare_bo(struct qaic_device *qdev, struct qaic_bo *bo, 854ff13be83SJeffrey Hugo struct qaic_attach_slice_hdr *hdr) 855ff13be83SJeffrey Hugo { 856ff13be83SJeffrey Hugo int ret; 857ff13be83SJeffrey Hugo 858ff13be83SJeffrey Hugo if (bo->base.import_attach) 859ff13be83SJeffrey Hugo ret = qaic_prepare_import_bo(bo, hdr); 860ff13be83SJeffrey Hugo else 861ff13be83SJeffrey Hugo ret = qaic_prepare_export_bo(qdev, bo, hdr); 862ff13be83SJeffrey Hugo 863ff13be83SJeffrey Hugo if (ret == 0) 864ff13be83SJeffrey Hugo bo->dir = hdr->dir; 865ff13be83SJeffrey Hugo 866ff13be83SJeffrey Hugo return ret; 867ff13be83SJeffrey Hugo } 868ff13be83SJeffrey Hugo 869ff13be83SJeffrey Hugo static void qaic_unprepare_import_bo(struct qaic_bo *bo) 870ff13be83SJeffrey Hugo { 871ff13be83SJeffrey Hugo dma_buf_unmap_attachment(bo->base.import_attach, bo->sgt, bo->dir); 872ff13be83SJeffrey Hugo bo->sgt = NULL; 873ff13be83SJeffrey Hugo bo->size = 0; 874ff13be83SJeffrey Hugo } 875ff13be83SJeffrey Hugo 876ff13be83SJeffrey Hugo static void qaic_unprepare_export_bo(struct qaic_device *qdev, struct qaic_bo *bo) 877ff13be83SJeffrey Hugo { 878ff13be83SJeffrey Hugo dma_unmap_sgtable(&qdev->pdev->dev, bo->sgt, bo->dir, 0); 879ff13be83SJeffrey Hugo } 880ff13be83SJeffrey Hugo 881ff13be83SJeffrey Hugo static void qaic_unprepare_bo(struct qaic_device *qdev, struct qaic_bo *bo) 882ff13be83SJeffrey Hugo { 883ff13be83SJeffrey Hugo if (bo->base.import_attach) 884ff13be83SJeffrey Hugo qaic_unprepare_import_bo(bo); 885ff13be83SJeffrey Hugo else 886ff13be83SJeffrey Hugo qaic_unprepare_export_bo(qdev, bo); 887ff13be83SJeffrey Hugo 888ff13be83SJeffrey Hugo bo->dir = 0; 889ff13be83SJeffrey Hugo } 890ff13be83SJeffrey Hugo 891ff13be83SJeffrey Hugo static void qaic_free_slices_bo(struct qaic_bo *bo) 892ff13be83SJeffrey Hugo { 893ff13be83SJeffrey Hugo struct bo_slice *slice, *temp; 894ff13be83SJeffrey Hugo 895ff13be83SJeffrey Hugo list_for_each_entry_safe(slice, temp, &bo->slices, slice) 896ff13be83SJeffrey Hugo kref_put(&slice->ref_count, free_slice); 897ff13be83SJeffrey Hugo } 898ff13be83SJeffrey Hugo 899ff13be83SJeffrey Hugo static int qaic_attach_slicing_bo(struct qaic_device *qdev, struct qaic_bo *bo, 900ff13be83SJeffrey Hugo struct qaic_attach_slice_hdr *hdr, 901ff13be83SJeffrey Hugo struct qaic_attach_slice_entry *slice_ent) 902ff13be83SJeffrey Hugo { 903ff13be83SJeffrey Hugo int ret, i; 904ff13be83SJeffrey Hugo 905ff13be83SJeffrey Hugo for (i = 0; i < hdr->count; i++) { 906ff13be83SJeffrey Hugo ret = qaic_map_one_slice(qdev, bo, &slice_ent[i]); 907ff13be83SJeffrey Hugo if (ret) { 908ff13be83SJeffrey Hugo qaic_free_slices_bo(bo); 909ff13be83SJeffrey Hugo return ret; 910ff13be83SJeffrey Hugo } 911ff13be83SJeffrey Hugo } 912ff13be83SJeffrey Hugo 913ff13be83SJeffrey Hugo if (bo->total_slice_nents > qdev->dbc[hdr->dbc_id].nelem) { 914ff13be83SJeffrey Hugo qaic_free_slices_bo(bo); 915ff13be83SJeffrey Hugo return -ENOSPC; 916ff13be83SJeffrey Hugo } 917ff13be83SJeffrey Hugo 918ff13be83SJeffrey Hugo bo->sliced = true; 919ff13be83SJeffrey Hugo bo->nr_slice = hdr->count; 920ff13be83SJeffrey Hugo list_add_tail(&bo->bo_list, &qdev->dbc[hdr->dbc_id].bo_lists); 921ff13be83SJeffrey Hugo 922ff13be83SJeffrey Hugo return 0; 923ff13be83SJeffrey Hugo } 924ff13be83SJeffrey Hugo 925ff13be83SJeffrey Hugo int qaic_attach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 926ff13be83SJeffrey Hugo { 927ff13be83SJeffrey Hugo struct qaic_attach_slice_entry *slice_ent; 928ff13be83SJeffrey Hugo struct qaic_attach_slice *args = data; 929ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc; 930ff13be83SJeffrey Hugo int usr_rcu_id, qdev_rcu_id; 931ff13be83SJeffrey Hugo struct drm_gem_object *obj; 932ff13be83SJeffrey Hugo struct qaic_device *qdev; 933ff13be83SJeffrey Hugo unsigned long arg_size; 934ff13be83SJeffrey Hugo struct qaic_user *usr; 935ff13be83SJeffrey Hugo u8 __user *user_data; 936ff13be83SJeffrey Hugo struct qaic_bo *bo; 937ff13be83SJeffrey Hugo int ret; 938ff13be83SJeffrey Hugo 939ff13be83SJeffrey Hugo usr = file_priv->driver_priv; 940ff13be83SJeffrey Hugo usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 941ff13be83SJeffrey Hugo if (!usr->qddev) { 942ff13be83SJeffrey Hugo ret = -ENODEV; 943ff13be83SJeffrey Hugo goto unlock_usr_srcu; 944ff13be83SJeffrey Hugo } 945ff13be83SJeffrey Hugo 946ff13be83SJeffrey Hugo qdev = usr->qddev->qdev; 947ff13be83SJeffrey Hugo qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 948ff13be83SJeffrey Hugo if (qdev->in_reset) { 949ff13be83SJeffrey Hugo ret = -ENODEV; 950ff13be83SJeffrey Hugo goto unlock_dev_srcu; 951ff13be83SJeffrey Hugo } 952ff13be83SJeffrey Hugo 953ff13be83SJeffrey Hugo if (args->hdr.count == 0) { 954ff13be83SJeffrey Hugo ret = -EINVAL; 955ff13be83SJeffrey Hugo goto unlock_dev_srcu; 956ff13be83SJeffrey Hugo } 957ff13be83SJeffrey Hugo 958ff13be83SJeffrey Hugo arg_size = args->hdr.count * sizeof(*slice_ent); 959ff13be83SJeffrey Hugo if (arg_size / args->hdr.count != sizeof(*slice_ent)) { 960ff13be83SJeffrey Hugo ret = -EINVAL; 961ff13be83SJeffrey Hugo goto unlock_dev_srcu; 962ff13be83SJeffrey Hugo } 963ff13be83SJeffrey Hugo 964ff13be83SJeffrey Hugo if (args->hdr.dbc_id >= qdev->num_dbc) { 965ff13be83SJeffrey Hugo ret = -EINVAL; 966ff13be83SJeffrey Hugo goto unlock_dev_srcu; 967ff13be83SJeffrey Hugo } 968ff13be83SJeffrey Hugo 969ff13be83SJeffrey Hugo if (args->hdr.size == 0) { 970ff13be83SJeffrey Hugo ret = -EINVAL; 971ff13be83SJeffrey Hugo goto unlock_dev_srcu; 972ff13be83SJeffrey Hugo } 973ff13be83SJeffrey Hugo 974ff13be83SJeffrey Hugo if (!(args->hdr.dir == DMA_TO_DEVICE || args->hdr.dir == DMA_FROM_DEVICE)) { 975ff13be83SJeffrey Hugo ret = -EINVAL; 976ff13be83SJeffrey Hugo goto unlock_dev_srcu; 977ff13be83SJeffrey Hugo } 978ff13be83SJeffrey Hugo 979ff13be83SJeffrey Hugo dbc = &qdev->dbc[args->hdr.dbc_id]; 980ff13be83SJeffrey Hugo if (dbc->usr != usr) { 981ff13be83SJeffrey Hugo ret = -EINVAL; 982ff13be83SJeffrey Hugo goto unlock_dev_srcu; 983ff13be83SJeffrey Hugo } 984ff13be83SJeffrey Hugo 985ff13be83SJeffrey Hugo if (args->data == 0) { 986ff13be83SJeffrey Hugo ret = -EINVAL; 987ff13be83SJeffrey Hugo goto unlock_dev_srcu; 988ff13be83SJeffrey Hugo } 989ff13be83SJeffrey Hugo 990ff13be83SJeffrey Hugo user_data = u64_to_user_ptr(args->data); 991ff13be83SJeffrey Hugo 992ff13be83SJeffrey Hugo slice_ent = kzalloc(arg_size, GFP_KERNEL); 993ff13be83SJeffrey Hugo if (!slice_ent) { 994ff13be83SJeffrey Hugo ret = -EINVAL; 995ff13be83SJeffrey Hugo goto unlock_dev_srcu; 996ff13be83SJeffrey Hugo } 997ff13be83SJeffrey Hugo 998ff13be83SJeffrey Hugo ret = copy_from_user(slice_ent, user_data, arg_size); 999ff13be83SJeffrey Hugo if (ret) { 1000ff13be83SJeffrey Hugo ret = -EFAULT; 1001ff13be83SJeffrey Hugo goto free_slice_ent; 1002ff13be83SJeffrey Hugo } 1003ff13be83SJeffrey Hugo 1004ff13be83SJeffrey Hugo ret = qaic_validate_req(qdev, slice_ent, args->hdr.count, args->hdr.size); 1005ff13be83SJeffrey Hugo if (ret) 1006ff13be83SJeffrey Hugo goto free_slice_ent; 1007ff13be83SJeffrey Hugo 1008ff13be83SJeffrey Hugo obj = drm_gem_object_lookup(file_priv, args->hdr.handle); 1009ff13be83SJeffrey Hugo if (!obj) { 1010ff13be83SJeffrey Hugo ret = -ENOENT; 1011ff13be83SJeffrey Hugo goto free_slice_ent; 1012ff13be83SJeffrey Hugo } 1013ff13be83SJeffrey Hugo 1014ff13be83SJeffrey Hugo bo = to_qaic_bo(obj); 1015ff13be83SJeffrey Hugo 1016ff13be83SJeffrey Hugo ret = qaic_prepare_bo(qdev, bo, &args->hdr); 1017ff13be83SJeffrey Hugo if (ret) 1018ff13be83SJeffrey Hugo goto put_bo; 1019ff13be83SJeffrey Hugo 1020ff13be83SJeffrey Hugo ret = qaic_attach_slicing_bo(qdev, bo, &args->hdr, slice_ent); 1021ff13be83SJeffrey Hugo if (ret) 1022ff13be83SJeffrey Hugo goto unprepare_bo; 1023ff13be83SJeffrey Hugo 1024ff13be83SJeffrey Hugo if (args->hdr.dir == DMA_TO_DEVICE) 1025ff13be83SJeffrey Hugo dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, args->hdr.dir); 1026ff13be83SJeffrey Hugo 1027ff13be83SJeffrey Hugo bo->dbc = dbc; 1028ff13be83SJeffrey Hugo drm_gem_object_put(obj); 1029ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1030ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1031ff13be83SJeffrey Hugo 1032ff13be83SJeffrey Hugo return 0; 1033ff13be83SJeffrey Hugo 1034ff13be83SJeffrey Hugo unprepare_bo: 1035ff13be83SJeffrey Hugo qaic_unprepare_bo(qdev, bo); 1036ff13be83SJeffrey Hugo put_bo: 1037ff13be83SJeffrey Hugo drm_gem_object_put(obj); 1038ff13be83SJeffrey Hugo free_slice_ent: 1039ff13be83SJeffrey Hugo kfree(slice_ent); 1040ff13be83SJeffrey Hugo unlock_dev_srcu: 1041ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1042ff13be83SJeffrey Hugo unlock_usr_srcu: 1043ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1044ff13be83SJeffrey Hugo return ret; 1045ff13be83SJeffrey Hugo } 1046ff13be83SJeffrey Hugo 1047ff13be83SJeffrey Hugo static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id, 1048ff13be83SJeffrey Hugo u32 head, u32 *ptail) 1049ff13be83SJeffrey Hugo { 1050ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id]; 1051ff13be83SJeffrey Hugo struct dbc_req *reqs = slice->reqs; 1052ff13be83SJeffrey Hugo u32 tail = *ptail; 1053ff13be83SJeffrey Hugo u32 avail; 1054ff13be83SJeffrey Hugo 1055ff13be83SJeffrey Hugo avail = head - tail; 1056ff13be83SJeffrey Hugo if (head <= tail) 1057ff13be83SJeffrey Hugo avail += dbc->nelem; 1058ff13be83SJeffrey Hugo 1059ff13be83SJeffrey Hugo --avail; 1060ff13be83SJeffrey Hugo 1061ff13be83SJeffrey Hugo if (avail < slice->nents) 1062ff13be83SJeffrey Hugo return -EAGAIN; 1063ff13be83SJeffrey Hugo 1064ff13be83SJeffrey Hugo if (tail + slice->nents > dbc->nelem) { 1065ff13be83SJeffrey Hugo avail = dbc->nelem - tail; 1066ff13be83SJeffrey Hugo avail = min_t(u32, avail, slice->nents); 1067ff13be83SJeffrey Hugo memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1068ff13be83SJeffrey Hugo sizeof(*reqs) * avail); 1069ff13be83SJeffrey Hugo reqs += avail; 1070ff13be83SJeffrey Hugo avail = slice->nents - avail; 1071ff13be83SJeffrey Hugo if (avail) 1072ff13be83SJeffrey Hugo memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail); 1073ff13be83SJeffrey Hugo } else { 1074ff13be83SJeffrey Hugo memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1075ff13be83SJeffrey Hugo sizeof(*reqs) * slice->nents); 1076ff13be83SJeffrey Hugo } 1077ff13be83SJeffrey Hugo 1078ff13be83SJeffrey Hugo *ptail = (tail + slice->nents) % dbc->nelem; 1079ff13be83SJeffrey Hugo 1080ff13be83SJeffrey Hugo return 0; 1081ff13be83SJeffrey Hugo } 1082ff13be83SJeffrey Hugo 1083ff13be83SJeffrey Hugo /* 1084ff13be83SJeffrey Hugo * Based on the value of resize we may only need to transmit first_n 1085ff13be83SJeffrey Hugo * entries and the last entry, with last_bytes to send from the last entry. 1086ff13be83SJeffrey Hugo * Note that first_n could be 0. 1087ff13be83SJeffrey Hugo */ 1088ff13be83SJeffrey Hugo static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, 1089ff13be83SJeffrey Hugo u64 resize, u32 dbc_id, u32 head, u32 *ptail) 1090ff13be83SJeffrey Hugo { 1091ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id]; 1092ff13be83SJeffrey Hugo struct dbc_req *reqs = slice->reqs; 1093ff13be83SJeffrey Hugo struct dbc_req *last_req; 1094ff13be83SJeffrey Hugo u32 tail = *ptail; 1095ff13be83SJeffrey Hugo u64 total_bytes; 1096ff13be83SJeffrey Hugo u64 last_bytes; 1097ff13be83SJeffrey Hugo u32 first_n; 1098ff13be83SJeffrey Hugo u32 avail; 1099ff13be83SJeffrey Hugo int ret; 1100ff13be83SJeffrey Hugo int i; 1101ff13be83SJeffrey Hugo 1102ff13be83SJeffrey Hugo avail = head - tail; 1103ff13be83SJeffrey Hugo if (head <= tail) 1104ff13be83SJeffrey Hugo avail += dbc->nelem; 1105ff13be83SJeffrey Hugo 1106ff13be83SJeffrey Hugo --avail; 1107ff13be83SJeffrey Hugo 1108ff13be83SJeffrey Hugo total_bytes = 0; 1109ff13be83SJeffrey Hugo for (i = 0; i < slice->nents; i++) { 1110ff13be83SJeffrey Hugo total_bytes += le32_to_cpu(reqs[i].len); 1111ff13be83SJeffrey Hugo if (total_bytes >= resize) 1112ff13be83SJeffrey Hugo break; 1113ff13be83SJeffrey Hugo } 1114ff13be83SJeffrey Hugo 1115ff13be83SJeffrey Hugo if (total_bytes < resize) { 1116ff13be83SJeffrey Hugo /* User space should have used the full buffer path. */ 1117ff13be83SJeffrey Hugo ret = -EINVAL; 1118ff13be83SJeffrey Hugo return ret; 1119ff13be83SJeffrey Hugo } 1120ff13be83SJeffrey Hugo 1121ff13be83SJeffrey Hugo first_n = i; 1122ff13be83SJeffrey Hugo last_bytes = i ? resize + le32_to_cpu(reqs[i].len) - total_bytes : resize; 1123ff13be83SJeffrey Hugo 1124ff13be83SJeffrey Hugo if (avail < (first_n + 1)) 1125ff13be83SJeffrey Hugo return -EAGAIN; 1126ff13be83SJeffrey Hugo 1127ff13be83SJeffrey Hugo if (first_n) { 1128ff13be83SJeffrey Hugo if (tail + first_n > dbc->nelem) { 1129ff13be83SJeffrey Hugo avail = dbc->nelem - tail; 1130ff13be83SJeffrey Hugo avail = min_t(u32, avail, first_n); 1131ff13be83SJeffrey Hugo memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1132ff13be83SJeffrey Hugo sizeof(*reqs) * avail); 1133ff13be83SJeffrey Hugo last_req = reqs + avail; 1134ff13be83SJeffrey Hugo avail = first_n - avail; 1135ff13be83SJeffrey Hugo if (avail) 1136ff13be83SJeffrey Hugo memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail); 1137ff13be83SJeffrey Hugo } else { 1138ff13be83SJeffrey Hugo memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, 1139ff13be83SJeffrey Hugo sizeof(*reqs) * first_n); 1140ff13be83SJeffrey Hugo } 1141ff13be83SJeffrey Hugo } 1142ff13be83SJeffrey Hugo 1143ff13be83SJeffrey Hugo /* Copy over the last entry. Here we need to adjust len to the left over 1144ff13be83SJeffrey Hugo * size, and set src and dst to the entry it is copied to. 1145ff13be83SJeffrey Hugo */ 1146ff13be83SJeffrey Hugo last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size(); 1147ff13be83SJeffrey Hugo memcpy(last_req, reqs + slice->nents - 1, sizeof(*reqs)); 1148ff13be83SJeffrey Hugo 1149ff13be83SJeffrey Hugo /* 1150ff13be83SJeffrey Hugo * last_bytes holds size of a DMA segment, maximum DMA segment size is 1151ff13be83SJeffrey Hugo * set to UINT_MAX by qaic and hence last_bytes can never exceed u32 1152ff13be83SJeffrey Hugo * range. So, by down sizing we are not corrupting the value. 1153ff13be83SJeffrey Hugo */ 1154ff13be83SJeffrey Hugo last_req->len = cpu_to_le32((u32)last_bytes); 1155ff13be83SJeffrey Hugo last_req->src_addr = reqs[first_n].src_addr; 1156ff13be83SJeffrey Hugo last_req->dest_addr = reqs[first_n].dest_addr; 1157ff13be83SJeffrey Hugo 1158ff13be83SJeffrey Hugo *ptail = (tail + first_n + 1) % dbc->nelem; 1159ff13be83SJeffrey Hugo 1160ff13be83SJeffrey Hugo return 0; 1161ff13be83SJeffrey Hugo } 1162ff13be83SJeffrey Hugo 1163ff13be83SJeffrey Hugo static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *file_priv, 1164ff13be83SJeffrey Hugo struct qaic_execute_entry *exec, unsigned int count, 1165ff13be83SJeffrey Hugo bool is_partial, struct dma_bridge_chan *dbc, u32 head, 1166ff13be83SJeffrey Hugo u32 *tail) 1167ff13be83SJeffrey Hugo { 1168ff13be83SJeffrey Hugo struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec; 1169ff13be83SJeffrey Hugo struct drm_gem_object *obj; 1170ff13be83SJeffrey Hugo struct bo_slice *slice; 1171ff13be83SJeffrey Hugo unsigned long flags; 1172ff13be83SJeffrey Hugo struct qaic_bo *bo; 1173ff13be83SJeffrey Hugo bool queued; 1174ff13be83SJeffrey Hugo int i, j; 1175ff13be83SJeffrey Hugo int ret; 1176ff13be83SJeffrey Hugo 1177ff13be83SJeffrey Hugo for (i = 0; i < count; i++) { 1178ff13be83SJeffrey Hugo /* 1179ff13be83SJeffrey Hugo * ref count will be decremented when the transfer of this 1180ff13be83SJeffrey Hugo * buffer is complete. It is inside dbc_irq_threaded_fn(). 1181ff13be83SJeffrey Hugo */ 1182ff13be83SJeffrey Hugo obj = drm_gem_object_lookup(file_priv, 1183ff13be83SJeffrey Hugo is_partial ? pexec[i].handle : exec[i].handle); 1184ff13be83SJeffrey Hugo if (!obj) { 1185ff13be83SJeffrey Hugo ret = -ENOENT; 1186ff13be83SJeffrey Hugo goto failed_to_send_bo; 1187ff13be83SJeffrey Hugo } 1188ff13be83SJeffrey Hugo 1189ff13be83SJeffrey Hugo bo = to_qaic_bo(obj); 1190ff13be83SJeffrey Hugo 1191ff13be83SJeffrey Hugo if (!bo->sliced) { 1192ff13be83SJeffrey Hugo ret = -EINVAL; 1193ff13be83SJeffrey Hugo goto failed_to_send_bo; 1194ff13be83SJeffrey Hugo } 1195ff13be83SJeffrey Hugo 1196ff13be83SJeffrey Hugo if (is_partial && pexec[i].resize > bo->size) { 1197ff13be83SJeffrey Hugo ret = -EINVAL; 1198ff13be83SJeffrey Hugo goto failed_to_send_bo; 1199ff13be83SJeffrey Hugo } 1200ff13be83SJeffrey Hugo 1201ff13be83SJeffrey Hugo spin_lock_irqsave(&dbc->xfer_lock, flags); 1202ff13be83SJeffrey Hugo queued = bo->queued; 1203ff13be83SJeffrey Hugo bo->queued = true; 1204ff13be83SJeffrey Hugo if (queued) { 1205ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1206ff13be83SJeffrey Hugo ret = -EINVAL; 1207ff13be83SJeffrey Hugo goto failed_to_send_bo; 1208ff13be83SJeffrey Hugo } 1209ff13be83SJeffrey Hugo 1210ff13be83SJeffrey Hugo bo->req_id = dbc->next_req_id++; 1211ff13be83SJeffrey Hugo 1212ff13be83SJeffrey Hugo list_for_each_entry(slice, &bo->slices, slice) { 1213ff13be83SJeffrey Hugo /* 1214ff13be83SJeffrey Hugo * If this slice does not fall under the given 1215ff13be83SJeffrey Hugo * resize then skip this slice and continue the loop 1216ff13be83SJeffrey Hugo */ 1217ff13be83SJeffrey Hugo if (is_partial && pexec[i].resize && pexec[i].resize <= slice->offset) 1218ff13be83SJeffrey Hugo continue; 1219ff13be83SJeffrey Hugo 1220ff13be83SJeffrey Hugo for (j = 0; j < slice->nents; j++) 1221ff13be83SJeffrey Hugo slice->reqs[j].req_id = cpu_to_le16(bo->req_id); 1222ff13be83SJeffrey Hugo 1223ff13be83SJeffrey Hugo /* 1224ff13be83SJeffrey Hugo * If it is a partial execute ioctl call then check if 1225ff13be83SJeffrey Hugo * resize has cut this slice short then do a partial copy 1226ff13be83SJeffrey Hugo * else do complete copy 1227ff13be83SJeffrey Hugo */ 1228ff13be83SJeffrey Hugo if (is_partial && pexec[i].resize && 1229ff13be83SJeffrey Hugo pexec[i].resize < slice->offset + slice->size) 1230ff13be83SJeffrey Hugo ret = copy_partial_exec_reqs(qdev, slice, 1231ff13be83SJeffrey Hugo pexec[i].resize - slice->offset, 1232ff13be83SJeffrey Hugo dbc->id, head, tail); 1233ff13be83SJeffrey Hugo else 1234ff13be83SJeffrey Hugo ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail); 1235ff13be83SJeffrey Hugo if (ret) { 1236ff13be83SJeffrey Hugo bo->queued = false; 1237ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1238ff13be83SJeffrey Hugo goto failed_to_send_bo; 1239ff13be83SJeffrey Hugo } 1240ff13be83SJeffrey Hugo } 1241ff13be83SJeffrey Hugo reinit_completion(&bo->xfer_done); 1242ff13be83SJeffrey Hugo list_add_tail(&bo->xfer_list, &dbc->xfer_list); 1243ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1244ff13be83SJeffrey Hugo dma_sync_sgtable_for_device(&qdev->pdev->dev, bo->sgt, bo->dir); 1245ff13be83SJeffrey Hugo } 1246ff13be83SJeffrey Hugo 1247ff13be83SJeffrey Hugo return 0; 1248ff13be83SJeffrey Hugo 1249ff13be83SJeffrey Hugo failed_to_send_bo: 1250ff13be83SJeffrey Hugo if (likely(obj)) 1251ff13be83SJeffrey Hugo drm_gem_object_put(obj); 1252ff13be83SJeffrey Hugo for (j = 0; j < i; j++) { 1253ff13be83SJeffrey Hugo spin_lock_irqsave(&dbc->xfer_lock, flags); 1254ff13be83SJeffrey Hugo bo = list_last_entry(&dbc->xfer_list, struct qaic_bo, xfer_list); 1255ff13be83SJeffrey Hugo obj = &bo->base; 1256ff13be83SJeffrey Hugo bo->queued = false; 1257ff13be83SJeffrey Hugo list_del(&bo->xfer_list); 1258ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1259ff13be83SJeffrey Hugo dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); 1260ff13be83SJeffrey Hugo drm_gem_object_put(obj); 1261ff13be83SJeffrey Hugo } 1262ff13be83SJeffrey Hugo return ret; 1263ff13be83SJeffrey Hugo } 1264ff13be83SJeffrey Hugo 1265ff13be83SJeffrey Hugo static void update_profiling_data(struct drm_file *file_priv, 1266ff13be83SJeffrey Hugo struct qaic_execute_entry *exec, unsigned int count, 1267ff13be83SJeffrey Hugo bool is_partial, u64 received_ts, u64 submit_ts, u32 queue_level) 1268ff13be83SJeffrey Hugo { 1269ff13be83SJeffrey Hugo struct qaic_partial_execute_entry *pexec = (struct qaic_partial_execute_entry *)exec; 1270ff13be83SJeffrey Hugo struct drm_gem_object *obj; 1271ff13be83SJeffrey Hugo struct qaic_bo *bo; 1272ff13be83SJeffrey Hugo int i; 1273ff13be83SJeffrey Hugo 1274ff13be83SJeffrey Hugo for (i = 0; i < count; i++) { 1275ff13be83SJeffrey Hugo /* 1276ff13be83SJeffrey Hugo * Since we already committed the BO to hardware, the only way 1277ff13be83SJeffrey Hugo * this should fail is a pending signal. We can't cancel the 1278ff13be83SJeffrey Hugo * submit to hardware, so we have to just skip the profiling 1279ff13be83SJeffrey Hugo * data. In case the signal is not fatal to the process, we 1280ff13be83SJeffrey Hugo * return success so that the user doesn't try to resubmit. 1281ff13be83SJeffrey Hugo */ 1282ff13be83SJeffrey Hugo obj = drm_gem_object_lookup(file_priv, 1283ff13be83SJeffrey Hugo is_partial ? pexec[i].handle : exec[i].handle); 1284ff13be83SJeffrey Hugo if (!obj) 1285ff13be83SJeffrey Hugo break; 1286ff13be83SJeffrey Hugo bo = to_qaic_bo(obj); 1287ff13be83SJeffrey Hugo bo->perf_stats.req_received_ts = received_ts; 1288ff13be83SJeffrey Hugo bo->perf_stats.req_submit_ts = submit_ts; 1289ff13be83SJeffrey Hugo bo->perf_stats.queue_level_before = queue_level; 1290ff13be83SJeffrey Hugo queue_level += bo->total_slice_nents; 1291ff13be83SJeffrey Hugo drm_gem_object_put(obj); 1292ff13be83SJeffrey Hugo } 1293ff13be83SJeffrey Hugo } 1294ff13be83SJeffrey Hugo 1295ff13be83SJeffrey Hugo static int __qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv, 1296ff13be83SJeffrey Hugo bool is_partial) 1297ff13be83SJeffrey Hugo { 1298ff13be83SJeffrey Hugo struct qaic_partial_execute_entry *pexec; 1299ff13be83SJeffrey Hugo struct qaic_execute *args = data; 1300ff13be83SJeffrey Hugo struct qaic_execute_entry *exec; 1301ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc; 1302ff13be83SJeffrey Hugo int usr_rcu_id, qdev_rcu_id; 1303ff13be83SJeffrey Hugo struct qaic_device *qdev; 1304ff13be83SJeffrey Hugo struct qaic_user *usr; 1305ff13be83SJeffrey Hugo u8 __user *user_data; 1306ff13be83SJeffrey Hugo unsigned long n; 1307ff13be83SJeffrey Hugo u64 received_ts; 1308ff13be83SJeffrey Hugo u32 queue_level; 1309ff13be83SJeffrey Hugo u64 submit_ts; 1310ff13be83SJeffrey Hugo int rcu_id; 1311ff13be83SJeffrey Hugo u32 head; 1312ff13be83SJeffrey Hugo u32 tail; 1313ff13be83SJeffrey Hugo u64 size; 1314ff13be83SJeffrey Hugo int ret; 1315ff13be83SJeffrey Hugo 1316ff13be83SJeffrey Hugo received_ts = ktime_get_ns(); 1317ff13be83SJeffrey Hugo 1318ff13be83SJeffrey Hugo size = is_partial ? sizeof(*pexec) : sizeof(*exec); 1319ff13be83SJeffrey Hugo 1320ff13be83SJeffrey Hugo n = (unsigned long)size * args->hdr.count; 1321ff13be83SJeffrey Hugo if (args->hdr.count == 0 || n / args->hdr.count != size) 1322ff13be83SJeffrey Hugo return -EINVAL; 1323ff13be83SJeffrey Hugo 1324ff13be83SJeffrey Hugo user_data = u64_to_user_ptr(args->data); 1325ff13be83SJeffrey Hugo 1326ff13be83SJeffrey Hugo exec = kcalloc(args->hdr.count, size, GFP_KERNEL); 1327ff13be83SJeffrey Hugo pexec = (struct qaic_partial_execute_entry *)exec; 1328ff13be83SJeffrey Hugo if (!exec) 1329ff13be83SJeffrey Hugo return -ENOMEM; 1330ff13be83SJeffrey Hugo 1331ff13be83SJeffrey Hugo if (copy_from_user(exec, user_data, n)) { 1332ff13be83SJeffrey Hugo ret = -EFAULT; 1333ff13be83SJeffrey Hugo goto free_exec; 1334ff13be83SJeffrey Hugo } 1335ff13be83SJeffrey Hugo 1336ff13be83SJeffrey Hugo usr = file_priv->driver_priv; 1337ff13be83SJeffrey Hugo usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 1338ff13be83SJeffrey Hugo if (!usr->qddev) { 1339ff13be83SJeffrey Hugo ret = -ENODEV; 1340ff13be83SJeffrey Hugo goto unlock_usr_srcu; 1341ff13be83SJeffrey Hugo } 1342ff13be83SJeffrey Hugo 1343ff13be83SJeffrey Hugo qdev = usr->qddev->qdev; 1344ff13be83SJeffrey Hugo qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 1345ff13be83SJeffrey Hugo if (qdev->in_reset) { 1346ff13be83SJeffrey Hugo ret = -ENODEV; 1347ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1348ff13be83SJeffrey Hugo } 1349ff13be83SJeffrey Hugo 1350ff13be83SJeffrey Hugo if (args->hdr.dbc_id >= qdev->num_dbc) { 1351ff13be83SJeffrey Hugo ret = -EINVAL; 1352ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1353ff13be83SJeffrey Hugo } 1354ff13be83SJeffrey Hugo 1355ff13be83SJeffrey Hugo dbc = &qdev->dbc[args->hdr.dbc_id]; 1356ff13be83SJeffrey Hugo 1357ff13be83SJeffrey Hugo rcu_id = srcu_read_lock(&dbc->ch_lock); 1358ff13be83SJeffrey Hugo if (!dbc->usr || dbc->usr->handle != usr->handle) { 1359ff13be83SJeffrey Hugo ret = -EPERM; 1360ff13be83SJeffrey Hugo goto release_ch_rcu; 1361ff13be83SJeffrey Hugo } 1362ff13be83SJeffrey Hugo 1363ff13be83SJeffrey Hugo head = readl(dbc->dbc_base + REQHP_OFF); 1364ff13be83SJeffrey Hugo tail = readl(dbc->dbc_base + REQTP_OFF); 1365ff13be83SJeffrey Hugo 1366ff13be83SJeffrey Hugo if (head == U32_MAX || tail == U32_MAX) { 1367ff13be83SJeffrey Hugo /* PCI link error */ 1368ff13be83SJeffrey Hugo ret = -ENODEV; 1369ff13be83SJeffrey Hugo goto release_ch_rcu; 1370ff13be83SJeffrey Hugo } 1371ff13be83SJeffrey Hugo 1372ff13be83SJeffrey Hugo queue_level = head <= tail ? tail - head : dbc->nelem - (head - tail); 1373ff13be83SJeffrey Hugo 1374ff13be83SJeffrey Hugo ret = send_bo_list_to_device(qdev, file_priv, exec, args->hdr.count, is_partial, dbc, 1375ff13be83SJeffrey Hugo head, &tail); 1376ff13be83SJeffrey Hugo if (ret) 1377ff13be83SJeffrey Hugo goto release_ch_rcu; 1378ff13be83SJeffrey Hugo 1379ff13be83SJeffrey Hugo /* Finalize commit to hardware */ 1380ff13be83SJeffrey Hugo submit_ts = ktime_get_ns(); 1381ff13be83SJeffrey Hugo writel(tail, dbc->dbc_base + REQTP_OFF); 1382ff13be83SJeffrey Hugo 1383ff13be83SJeffrey Hugo update_profiling_data(file_priv, exec, args->hdr.count, is_partial, received_ts, 1384ff13be83SJeffrey Hugo submit_ts, queue_level); 1385ff13be83SJeffrey Hugo 1386ff13be83SJeffrey Hugo if (datapath_polling) 1387ff13be83SJeffrey Hugo schedule_work(&dbc->poll_work); 1388ff13be83SJeffrey Hugo 1389ff13be83SJeffrey Hugo release_ch_rcu: 1390ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1391ff13be83SJeffrey Hugo unlock_dev_srcu: 1392ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1393ff13be83SJeffrey Hugo unlock_usr_srcu: 1394ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1395ff13be83SJeffrey Hugo free_exec: 1396ff13be83SJeffrey Hugo kfree(exec); 1397ff13be83SJeffrey Hugo return ret; 1398ff13be83SJeffrey Hugo } 1399ff13be83SJeffrey Hugo 1400ff13be83SJeffrey Hugo int qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1401ff13be83SJeffrey Hugo { 1402ff13be83SJeffrey Hugo return __qaic_execute_bo_ioctl(dev, data, file_priv, false); 1403ff13be83SJeffrey Hugo } 1404ff13be83SJeffrey Hugo 1405ff13be83SJeffrey Hugo int qaic_partial_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1406ff13be83SJeffrey Hugo { 1407ff13be83SJeffrey Hugo return __qaic_execute_bo_ioctl(dev, data, file_priv, true); 1408ff13be83SJeffrey Hugo } 1409ff13be83SJeffrey Hugo 1410ff13be83SJeffrey Hugo /* 1411ff13be83SJeffrey Hugo * Our interrupt handling is a bit more complicated than a simple ideal, but 1412ff13be83SJeffrey Hugo * sadly necessary. 1413ff13be83SJeffrey Hugo * 1414ff13be83SJeffrey Hugo * Each dbc has a completion queue. Entries in the queue correspond to DMA 1415ff13be83SJeffrey Hugo * requests which the device has processed. The hardware already has a built 1416ff13be83SJeffrey Hugo * in irq mitigation. When the device puts an entry into the queue, it will 1417ff13be83SJeffrey Hugo * only trigger an interrupt if the queue was empty. Therefore, when adding 1418ff13be83SJeffrey Hugo * the Nth event to a non-empty queue, the hardware doesn't trigger an 1419ff13be83SJeffrey Hugo * interrupt. This means the host doesn't get additional interrupts signaling 1420ff13be83SJeffrey Hugo * the same thing - the queue has something to process. 1421ff13be83SJeffrey Hugo * This behavior can be overridden in the DMA request. 1422ff13be83SJeffrey Hugo * This means that when the host receives an interrupt, it is required to 1423ff13be83SJeffrey Hugo * drain the queue. 1424ff13be83SJeffrey Hugo * 1425ff13be83SJeffrey Hugo * This behavior is what NAPI attempts to accomplish, although we can't use 1426ff13be83SJeffrey Hugo * NAPI as we don't have a netdev. We use threaded irqs instead. 1427ff13be83SJeffrey Hugo * 1428ff13be83SJeffrey Hugo * However, there is a situation where the host drains the queue fast enough 1429ff13be83SJeffrey Hugo * that every event causes an interrupt. Typically this is not a problem as 1430ff13be83SJeffrey Hugo * the rate of events would be low. However, that is not the case with 1431ff13be83SJeffrey Hugo * lprnet for example. On an Intel Xeon D-2191 where we run 8 instances of 1432ff13be83SJeffrey Hugo * lprnet, the host receives roughly 80k interrupts per second from the device 1433ff13be83SJeffrey Hugo * (per /proc/interrupts). While NAPI documentation indicates the host should 1434ff13be83SJeffrey Hugo * just chug along, sadly that behavior causes instability in some hosts. 1435ff13be83SJeffrey Hugo * 1436ff13be83SJeffrey Hugo * Therefore, we implement an interrupt disable scheme similar to NAPI. The 1437ff13be83SJeffrey Hugo * key difference is that we will delay after draining the queue for a small 1438ff13be83SJeffrey Hugo * time to allow additional events to come in via polling. Using the above 1439ff13be83SJeffrey Hugo * lprnet workload, this reduces the number of interrupts processed from 1440ff13be83SJeffrey Hugo * ~80k/sec to about 64 in 5 minutes and appears to solve the system 1441ff13be83SJeffrey Hugo * instability. 1442ff13be83SJeffrey Hugo */ 1443ff13be83SJeffrey Hugo irqreturn_t dbc_irq_handler(int irq, void *data) 1444ff13be83SJeffrey Hugo { 1445ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc = data; 1446ff13be83SJeffrey Hugo int rcu_id; 1447ff13be83SJeffrey Hugo u32 head; 1448ff13be83SJeffrey Hugo u32 tail; 1449ff13be83SJeffrey Hugo 1450ff13be83SJeffrey Hugo rcu_id = srcu_read_lock(&dbc->ch_lock); 1451ff13be83SJeffrey Hugo 1452ff13be83SJeffrey Hugo if (!dbc->usr) { 1453ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1454ff13be83SJeffrey Hugo return IRQ_HANDLED; 1455ff13be83SJeffrey Hugo } 1456ff13be83SJeffrey Hugo 1457ff13be83SJeffrey Hugo head = readl(dbc->dbc_base + RSPHP_OFF); 1458ff13be83SJeffrey Hugo if (head == U32_MAX) { /* PCI link error */ 1459ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1460ff13be83SJeffrey Hugo return IRQ_NONE; 1461ff13be83SJeffrey Hugo } 1462ff13be83SJeffrey Hugo 1463ff13be83SJeffrey Hugo tail = readl(dbc->dbc_base + RSPTP_OFF); 1464ff13be83SJeffrey Hugo if (tail == U32_MAX) { /* PCI link error */ 1465ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1466ff13be83SJeffrey Hugo return IRQ_NONE; 1467ff13be83SJeffrey Hugo } 1468ff13be83SJeffrey Hugo 1469ff13be83SJeffrey Hugo if (head == tail) { /* queue empty */ 1470ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1471ff13be83SJeffrey Hugo return IRQ_NONE; 1472ff13be83SJeffrey Hugo } 1473ff13be83SJeffrey Hugo 1474ff13be83SJeffrey Hugo disable_irq_nosync(irq); 1475ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1476ff13be83SJeffrey Hugo return IRQ_WAKE_THREAD; 1477ff13be83SJeffrey Hugo } 1478ff13be83SJeffrey Hugo 1479ff13be83SJeffrey Hugo void irq_polling_work(struct work_struct *work) 1480ff13be83SJeffrey Hugo { 1481ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc = container_of(work, struct dma_bridge_chan, poll_work); 1482ff13be83SJeffrey Hugo unsigned long flags; 1483ff13be83SJeffrey Hugo int rcu_id; 1484ff13be83SJeffrey Hugo u32 head; 1485ff13be83SJeffrey Hugo u32 tail; 1486ff13be83SJeffrey Hugo 1487ff13be83SJeffrey Hugo rcu_id = srcu_read_lock(&dbc->ch_lock); 1488ff13be83SJeffrey Hugo 1489ff13be83SJeffrey Hugo while (1) { 1490ff13be83SJeffrey Hugo if (dbc->qdev->in_reset) { 1491ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1492ff13be83SJeffrey Hugo return; 1493ff13be83SJeffrey Hugo } 1494ff13be83SJeffrey Hugo if (!dbc->usr) { 1495ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1496ff13be83SJeffrey Hugo return; 1497ff13be83SJeffrey Hugo } 1498ff13be83SJeffrey Hugo spin_lock_irqsave(&dbc->xfer_lock, flags); 1499ff13be83SJeffrey Hugo if (list_empty(&dbc->xfer_list)) { 1500ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1501ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1502ff13be83SJeffrey Hugo return; 1503ff13be83SJeffrey Hugo } 1504ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1505ff13be83SJeffrey Hugo 1506ff13be83SJeffrey Hugo head = readl(dbc->dbc_base + RSPHP_OFF); 1507ff13be83SJeffrey Hugo if (head == U32_MAX) { /* PCI link error */ 1508ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1509ff13be83SJeffrey Hugo return; 1510ff13be83SJeffrey Hugo } 1511ff13be83SJeffrey Hugo 1512ff13be83SJeffrey Hugo tail = readl(dbc->dbc_base + RSPTP_OFF); 1513ff13be83SJeffrey Hugo if (tail == U32_MAX) { /* PCI link error */ 1514ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1515ff13be83SJeffrey Hugo return; 1516ff13be83SJeffrey Hugo } 1517ff13be83SJeffrey Hugo 1518ff13be83SJeffrey Hugo if (head != tail) { 1519ff13be83SJeffrey Hugo irq_wake_thread(dbc->irq, dbc); 1520ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1521ff13be83SJeffrey Hugo return; 1522ff13be83SJeffrey Hugo } 1523ff13be83SJeffrey Hugo 1524ff13be83SJeffrey Hugo cond_resched(); 1525ff13be83SJeffrey Hugo usleep_range(datapath_poll_interval_us, 2 * datapath_poll_interval_us); 1526ff13be83SJeffrey Hugo } 1527ff13be83SJeffrey Hugo } 1528ff13be83SJeffrey Hugo 1529ff13be83SJeffrey Hugo irqreturn_t dbc_irq_threaded_fn(int irq, void *data) 1530ff13be83SJeffrey Hugo { 1531ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc = data; 1532ff13be83SJeffrey Hugo int event_count = NUM_EVENTS; 1533ff13be83SJeffrey Hugo int delay_count = NUM_DELAYS; 1534ff13be83SJeffrey Hugo struct qaic_device *qdev; 1535ff13be83SJeffrey Hugo struct qaic_bo *bo, *i; 1536ff13be83SJeffrey Hugo struct dbc_rsp *rsp; 1537ff13be83SJeffrey Hugo unsigned long flags; 1538ff13be83SJeffrey Hugo int rcu_id; 1539ff13be83SJeffrey Hugo u16 status; 1540ff13be83SJeffrey Hugo u16 req_id; 1541ff13be83SJeffrey Hugo u32 head; 1542ff13be83SJeffrey Hugo u32 tail; 1543ff13be83SJeffrey Hugo 1544ff13be83SJeffrey Hugo rcu_id = srcu_read_lock(&dbc->ch_lock); 1545ff13be83SJeffrey Hugo 1546ff13be83SJeffrey Hugo head = readl(dbc->dbc_base + RSPHP_OFF); 1547ff13be83SJeffrey Hugo if (head == U32_MAX) /* PCI link error */ 1548ff13be83SJeffrey Hugo goto error_out; 1549ff13be83SJeffrey Hugo 1550ff13be83SJeffrey Hugo qdev = dbc->qdev; 1551ff13be83SJeffrey Hugo read_fifo: 1552ff13be83SJeffrey Hugo 1553ff13be83SJeffrey Hugo if (!event_count) { 1554ff13be83SJeffrey Hugo event_count = NUM_EVENTS; 1555ff13be83SJeffrey Hugo cond_resched(); 1556ff13be83SJeffrey Hugo } 1557ff13be83SJeffrey Hugo 1558ff13be83SJeffrey Hugo /* 1559ff13be83SJeffrey Hugo * if this channel isn't assigned or gets unassigned during processing 1560ff13be83SJeffrey Hugo * we have nothing further to do 1561ff13be83SJeffrey Hugo */ 1562ff13be83SJeffrey Hugo if (!dbc->usr) 1563ff13be83SJeffrey Hugo goto error_out; 1564ff13be83SJeffrey Hugo 1565ff13be83SJeffrey Hugo tail = readl(dbc->dbc_base + RSPTP_OFF); 1566ff13be83SJeffrey Hugo if (tail == U32_MAX) /* PCI link error */ 1567ff13be83SJeffrey Hugo goto error_out; 1568ff13be83SJeffrey Hugo 1569ff13be83SJeffrey Hugo if (head == tail) { /* queue empty */ 1570ff13be83SJeffrey Hugo if (delay_count) { 1571ff13be83SJeffrey Hugo --delay_count; 1572ff13be83SJeffrey Hugo usleep_range(100, 200); 1573ff13be83SJeffrey Hugo goto read_fifo; /* check for a new event */ 1574ff13be83SJeffrey Hugo } 1575ff13be83SJeffrey Hugo goto normal_out; 1576ff13be83SJeffrey Hugo } 1577ff13be83SJeffrey Hugo 1578ff13be83SJeffrey Hugo delay_count = NUM_DELAYS; 1579ff13be83SJeffrey Hugo while (head != tail) { 1580ff13be83SJeffrey Hugo if (!event_count) 1581ff13be83SJeffrey Hugo break; 1582ff13be83SJeffrey Hugo --event_count; 1583ff13be83SJeffrey Hugo rsp = dbc->rsp_q_base + head * sizeof(*rsp); 1584ff13be83SJeffrey Hugo req_id = le16_to_cpu(rsp->req_id); 1585ff13be83SJeffrey Hugo status = le16_to_cpu(rsp->status); 1586ff13be83SJeffrey Hugo if (status) 1587ff13be83SJeffrey Hugo pci_dbg(qdev->pdev, "req_id %d failed with status %d\n", req_id, status); 1588ff13be83SJeffrey Hugo spin_lock_irqsave(&dbc->xfer_lock, flags); 1589ff13be83SJeffrey Hugo /* 1590ff13be83SJeffrey Hugo * A BO can receive multiple interrupts, since a BO can be 1591ff13be83SJeffrey Hugo * divided into multiple slices and a buffer receives as many 1592ff13be83SJeffrey Hugo * interrupts as slices. So until it receives interrupts for 1593ff13be83SJeffrey Hugo * all the slices we cannot mark that buffer complete. 1594ff13be83SJeffrey Hugo */ 1595ff13be83SJeffrey Hugo list_for_each_entry_safe(bo, i, &dbc->xfer_list, xfer_list) { 1596ff13be83SJeffrey Hugo if (bo->req_id == req_id) 1597ff13be83SJeffrey Hugo bo->nr_slice_xfer_done++; 1598ff13be83SJeffrey Hugo else 1599ff13be83SJeffrey Hugo continue; 1600ff13be83SJeffrey Hugo 1601ff13be83SJeffrey Hugo if (bo->nr_slice_xfer_done < bo->nr_slice) 1602ff13be83SJeffrey Hugo break; 1603ff13be83SJeffrey Hugo 1604ff13be83SJeffrey Hugo /* 1605ff13be83SJeffrey Hugo * At this point we have received all the interrupts for 1606ff13be83SJeffrey Hugo * BO, which means BO execution is complete. 1607ff13be83SJeffrey Hugo */ 1608ff13be83SJeffrey Hugo dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); 1609ff13be83SJeffrey Hugo bo->nr_slice_xfer_done = 0; 1610ff13be83SJeffrey Hugo bo->queued = false; 1611ff13be83SJeffrey Hugo list_del(&bo->xfer_list); 1612ff13be83SJeffrey Hugo bo->perf_stats.req_processed_ts = ktime_get_ns(); 1613ff13be83SJeffrey Hugo complete_all(&bo->xfer_done); 1614ff13be83SJeffrey Hugo drm_gem_object_put(&bo->base); 1615ff13be83SJeffrey Hugo break; 1616ff13be83SJeffrey Hugo } 1617ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1618ff13be83SJeffrey Hugo head = (head + 1) % dbc->nelem; 1619ff13be83SJeffrey Hugo } 1620ff13be83SJeffrey Hugo 1621ff13be83SJeffrey Hugo /* 1622ff13be83SJeffrey Hugo * Update the head pointer of response queue and let the device know 1623ff13be83SJeffrey Hugo * that we have consumed elements from the queue. 1624ff13be83SJeffrey Hugo */ 1625ff13be83SJeffrey Hugo writel(head, dbc->dbc_base + RSPHP_OFF); 1626ff13be83SJeffrey Hugo 1627ff13be83SJeffrey Hugo /* elements might have been put in the queue while we were processing */ 1628ff13be83SJeffrey Hugo goto read_fifo; 1629ff13be83SJeffrey Hugo 1630ff13be83SJeffrey Hugo normal_out: 1631ff13be83SJeffrey Hugo if (likely(!datapath_polling)) 1632ff13be83SJeffrey Hugo enable_irq(irq); 1633ff13be83SJeffrey Hugo else 1634ff13be83SJeffrey Hugo schedule_work(&dbc->poll_work); 1635ff13be83SJeffrey Hugo /* checking the fifo and enabling irqs is a race, missed event check */ 1636ff13be83SJeffrey Hugo tail = readl(dbc->dbc_base + RSPTP_OFF); 1637ff13be83SJeffrey Hugo if (tail != U32_MAX && head != tail) { 1638ff13be83SJeffrey Hugo if (likely(!datapath_polling)) 1639ff13be83SJeffrey Hugo disable_irq_nosync(irq); 1640ff13be83SJeffrey Hugo goto read_fifo; 1641ff13be83SJeffrey Hugo } 1642ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1643ff13be83SJeffrey Hugo return IRQ_HANDLED; 1644ff13be83SJeffrey Hugo 1645ff13be83SJeffrey Hugo error_out: 1646ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1647ff13be83SJeffrey Hugo if (likely(!datapath_polling)) 1648ff13be83SJeffrey Hugo enable_irq(irq); 1649ff13be83SJeffrey Hugo else 1650ff13be83SJeffrey Hugo schedule_work(&dbc->poll_work); 1651ff13be83SJeffrey Hugo 1652ff13be83SJeffrey Hugo return IRQ_HANDLED; 1653ff13be83SJeffrey Hugo } 1654ff13be83SJeffrey Hugo 1655ff13be83SJeffrey Hugo int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1656ff13be83SJeffrey Hugo { 1657ff13be83SJeffrey Hugo struct qaic_wait *args = data; 1658ff13be83SJeffrey Hugo int usr_rcu_id, qdev_rcu_id; 1659ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc; 1660ff13be83SJeffrey Hugo struct drm_gem_object *obj; 1661ff13be83SJeffrey Hugo struct qaic_device *qdev; 1662ff13be83SJeffrey Hugo unsigned long timeout; 1663ff13be83SJeffrey Hugo struct qaic_user *usr; 1664ff13be83SJeffrey Hugo struct qaic_bo *bo; 1665ff13be83SJeffrey Hugo int rcu_id; 1666ff13be83SJeffrey Hugo int ret; 1667ff13be83SJeffrey Hugo 1668ff13be83SJeffrey Hugo usr = file_priv->driver_priv; 1669ff13be83SJeffrey Hugo usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 1670ff13be83SJeffrey Hugo if (!usr->qddev) { 1671ff13be83SJeffrey Hugo ret = -ENODEV; 1672ff13be83SJeffrey Hugo goto unlock_usr_srcu; 1673ff13be83SJeffrey Hugo } 1674ff13be83SJeffrey Hugo 1675ff13be83SJeffrey Hugo qdev = usr->qddev->qdev; 1676ff13be83SJeffrey Hugo qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 1677ff13be83SJeffrey Hugo if (qdev->in_reset) { 1678ff13be83SJeffrey Hugo ret = -ENODEV; 1679ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1680ff13be83SJeffrey Hugo } 1681ff13be83SJeffrey Hugo 1682ff13be83SJeffrey Hugo if (args->pad != 0) { 1683ff13be83SJeffrey Hugo ret = -EINVAL; 1684ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1685ff13be83SJeffrey Hugo } 1686ff13be83SJeffrey Hugo 1687ff13be83SJeffrey Hugo if (args->dbc_id >= qdev->num_dbc) { 1688ff13be83SJeffrey Hugo ret = -EINVAL; 1689ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1690ff13be83SJeffrey Hugo } 1691ff13be83SJeffrey Hugo 1692ff13be83SJeffrey Hugo dbc = &qdev->dbc[args->dbc_id]; 1693ff13be83SJeffrey Hugo 1694ff13be83SJeffrey Hugo rcu_id = srcu_read_lock(&dbc->ch_lock); 1695ff13be83SJeffrey Hugo if (dbc->usr != usr) { 1696ff13be83SJeffrey Hugo ret = -EPERM; 1697ff13be83SJeffrey Hugo goto unlock_ch_srcu; 1698ff13be83SJeffrey Hugo } 1699ff13be83SJeffrey Hugo 1700ff13be83SJeffrey Hugo obj = drm_gem_object_lookup(file_priv, args->handle); 1701ff13be83SJeffrey Hugo if (!obj) { 1702ff13be83SJeffrey Hugo ret = -ENOENT; 1703ff13be83SJeffrey Hugo goto unlock_ch_srcu; 1704ff13be83SJeffrey Hugo } 1705ff13be83SJeffrey Hugo 1706ff13be83SJeffrey Hugo bo = to_qaic_bo(obj); 1707ff13be83SJeffrey Hugo timeout = args->timeout ? args->timeout : wait_exec_default_timeout_ms; 1708ff13be83SJeffrey Hugo timeout = msecs_to_jiffies(timeout); 1709ff13be83SJeffrey Hugo ret = wait_for_completion_interruptible_timeout(&bo->xfer_done, timeout); 1710ff13be83SJeffrey Hugo if (!ret) { 1711ff13be83SJeffrey Hugo ret = -ETIMEDOUT; 1712ff13be83SJeffrey Hugo goto put_obj; 1713ff13be83SJeffrey Hugo } 1714ff13be83SJeffrey Hugo if (ret > 0) 1715ff13be83SJeffrey Hugo ret = 0; 1716ff13be83SJeffrey Hugo 1717ff13be83SJeffrey Hugo if (!dbc->usr) 1718ff13be83SJeffrey Hugo ret = -EPERM; 1719ff13be83SJeffrey Hugo 1720ff13be83SJeffrey Hugo put_obj: 1721ff13be83SJeffrey Hugo drm_gem_object_put(obj); 1722ff13be83SJeffrey Hugo unlock_ch_srcu: 1723ff13be83SJeffrey Hugo srcu_read_unlock(&dbc->ch_lock, rcu_id); 1724ff13be83SJeffrey Hugo unlock_dev_srcu: 1725ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1726ff13be83SJeffrey Hugo unlock_usr_srcu: 1727ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1728ff13be83SJeffrey Hugo return ret; 1729ff13be83SJeffrey Hugo } 1730ff13be83SJeffrey Hugo 1731ff13be83SJeffrey Hugo int qaic_perf_stats_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 1732ff13be83SJeffrey Hugo { 1733ff13be83SJeffrey Hugo struct qaic_perf_stats_entry *ent = NULL; 1734ff13be83SJeffrey Hugo struct qaic_perf_stats *args = data; 1735ff13be83SJeffrey Hugo int usr_rcu_id, qdev_rcu_id; 1736ff13be83SJeffrey Hugo struct drm_gem_object *obj; 1737ff13be83SJeffrey Hugo struct qaic_device *qdev; 1738ff13be83SJeffrey Hugo struct qaic_user *usr; 1739ff13be83SJeffrey Hugo struct qaic_bo *bo; 1740ff13be83SJeffrey Hugo int ret, i; 1741ff13be83SJeffrey Hugo 1742ff13be83SJeffrey Hugo usr = file_priv->driver_priv; 1743ff13be83SJeffrey Hugo usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 1744ff13be83SJeffrey Hugo if (!usr->qddev) { 1745ff13be83SJeffrey Hugo ret = -ENODEV; 1746ff13be83SJeffrey Hugo goto unlock_usr_srcu; 1747ff13be83SJeffrey Hugo } 1748ff13be83SJeffrey Hugo 1749ff13be83SJeffrey Hugo qdev = usr->qddev->qdev; 1750ff13be83SJeffrey Hugo qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 1751ff13be83SJeffrey Hugo if (qdev->in_reset) { 1752ff13be83SJeffrey Hugo ret = -ENODEV; 1753ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1754ff13be83SJeffrey Hugo } 1755ff13be83SJeffrey Hugo 1756ff13be83SJeffrey Hugo if (args->hdr.dbc_id >= qdev->num_dbc) { 1757ff13be83SJeffrey Hugo ret = -EINVAL; 1758ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1759ff13be83SJeffrey Hugo } 1760ff13be83SJeffrey Hugo 1761ff13be83SJeffrey Hugo ent = kcalloc(args->hdr.count, sizeof(*ent), GFP_KERNEL); 1762ff13be83SJeffrey Hugo if (!ent) { 1763ff13be83SJeffrey Hugo ret = -EINVAL; 1764ff13be83SJeffrey Hugo goto unlock_dev_srcu; 1765ff13be83SJeffrey Hugo } 1766ff13be83SJeffrey Hugo 1767ff13be83SJeffrey Hugo ret = copy_from_user(ent, u64_to_user_ptr(args->data), args->hdr.count * sizeof(*ent)); 1768ff13be83SJeffrey Hugo if (ret) { 1769ff13be83SJeffrey Hugo ret = -EFAULT; 1770ff13be83SJeffrey Hugo goto free_ent; 1771ff13be83SJeffrey Hugo } 1772ff13be83SJeffrey Hugo 1773ff13be83SJeffrey Hugo for (i = 0; i < args->hdr.count; i++) { 1774ff13be83SJeffrey Hugo obj = drm_gem_object_lookup(file_priv, ent[i].handle); 1775ff13be83SJeffrey Hugo if (!obj) { 1776ff13be83SJeffrey Hugo ret = -ENOENT; 1777ff13be83SJeffrey Hugo goto free_ent; 1778ff13be83SJeffrey Hugo } 1779ff13be83SJeffrey Hugo bo = to_qaic_bo(obj); 1780ff13be83SJeffrey Hugo /* 1781ff13be83SJeffrey Hugo * perf stats ioctl is called before wait ioctl is complete then 1782ff13be83SJeffrey Hugo * the latency information is invalid. 1783ff13be83SJeffrey Hugo */ 1784ff13be83SJeffrey Hugo if (bo->perf_stats.req_processed_ts < bo->perf_stats.req_submit_ts) { 1785ff13be83SJeffrey Hugo ent[i].device_latency_us = 0; 1786ff13be83SJeffrey Hugo } else { 1787ff13be83SJeffrey Hugo ent[i].device_latency_us = div_u64((bo->perf_stats.req_processed_ts - 1788ff13be83SJeffrey Hugo bo->perf_stats.req_submit_ts), 1000); 1789ff13be83SJeffrey Hugo } 1790ff13be83SJeffrey Hugo ent[i].submit_latency_us = div_u64((bo->perf_stats.req_submit_ts - 1791ff13be83SJeffrey Hugo bo->perf_stats.req_received_ts), 1000); 1792ff13be83SJeffrey Hugo ent[i].queue_level_before = bo->perf_stats.queue_level_before; 1793ff13be83SJeffrey Hugo ent[i].num_queue_element = bo->total_slice_nents; 1794ff13be83SJeffrey Hugo drm_gem_object_put(obj); 1795ff13be83SJeffrey Hugo } 1796ff13be83SJeffrey Hugo 1797ff13be83SJeffrey Hugo if (copy_to_user(u64_to_user_ptr(args->data), ent, args->hdr.count * sizeof(*ent))) 1798ff13be83SJeffrey Hugo ret = -EFAULT; 1799ff13be83SJeffrey Hugo 1800ff13be83SJeffrey Hugo free_ent: 1801ff13be83SJeffrey Hugo kfree(ent); 1802ff13be83SJeffrey Hugo unlock_dev_srcu: 1803ff13be83SJeffrey Hugo srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 1804ff13be83SJeffrey Hugo unlock_usr_srcu: 1805ff13be83SJeffrey Hugo srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 1806ff13be83SJeffrey Hugo return ret; 1807ff13be83SJeffrey Hugo } 1808ff13be83SJeffrey Hugo 1809ff13be83SJeffrey Hugo static void empty_xfer_list(struct qaic_device *qdev, struct dma_bridge_chan *dbc) 1810ff13be83SJeffrey Hugo { 1811ff13be83SJeffrey Hugo unsigned long flags; 1812ff13be83SJeffrey Hugo struct qaic_bo *bo; 1813ff13be83SJeffrey Hugo 1814ff13be83SJeffrey Hugo spin_lock_irqsave(&dbc->xfer_lock, flags); 1815ff13be83SJeffrey Hugo while (!list_empty(&dbc->xfer_list)) { 1816ff13be83SJeffrey Hugo bo = list_first_entry(&dbc->xfer_list, typeof(*bo), xfer_list); 1817ff13be83SJeffrey Hugo bo->queued = false; 1818ff13be83SJeffrey Hugo list_del(&bo->xfer_list); 1819ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1820ff13be83SJeffrey Hugo dma_sync_sgtable_for_cpu(&qdev->pdev->dev, bo->sgt, bo->dir); 1821ff13be83SJeffrey Hugo complete_all(&bo->xfer_done); 1822ff13be83SJeffrey Hugo drm_gem_object_put(&bo->base); 1823ff13be83SJeffrey Hugo spin_lock_irqsave(&dbc->xfer_lock, flags); 1824ff13be83SJeffrey Hugo } 1825ff13be83SJeffrey Hugo spin_unlock_irqrestore(&dbc->xfer_lock, flags); 1826ff13be83SJeffrey Hugo } 1827ff13be83SJeffrey Hugo 1828ff13be83SJeffrey Hugo int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr) 1829ff13be83SJeffrey Hugo { 1830ff13be83SJeffrey Hugo if (!qdev->dbc[dbc_id].usr || qdev->dbc[dbc_id].usr->handle != usr->handle) 1831ff13be83SJeffrey Hugo return -EPERM; 1832ff13be83SJeffrey Hugo 1833ff13be83SJeffrey Hugo qdev->dbc[dbc_id].usr = NULL; 1834ff13be83SJeffrey Hugo synchronize_srcu(&qdev->dbc[dbc_id].ch_lock); 1835ff13be83SJeffrey Hugo return 0; 1836ff13be83SJeffrey Hugo } 1837ff13be83SJeffrey Hugo 1838ff13be83SJeffrey Hugo /** 1839ff13be83SJeffrey Hugo * enable_dbc - Enable the DBC. DBCs are disabled by removing the context of 1840ff13be83SJeffrey Hugo * user. Add user context back to DBC to enable it. This function trusts the 1841ff13be83SJeffrey Hugo * DBC ID passed and expects the DBC to be disabled. 1842ff13be83SJeffrey Hugo * @qdev: Qranium device handle 1843ff13be83SJeffrey Hugo * @dbc_id: ID of the DBC 1844ff13be83SJeffrey Hugo * @usr: User context 1845ff13be83SJeffrey Hugo */ 1846ff13be83SJeffrey Hugo void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr) 1847ff13be83SJeffrey Hugo { 1848ff13be83SJeffrey Hugo qdev->dbc[dbc_id].usr = usr; 1849ff13be83SJeffrey Hugo } 1850ff13be83SJeffrey Hugo 1851ff13be83SJeffrey Hugo void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id) 1852ff13be83SJeffrey Hugo { 1853ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id]; 1854ff13be83SJeffrey Hugo 1855ff13be83SJeffrey Hugo dbc->usr = NULL; 1856ff13be83SJeffrey Hugo empty_xfer_list(qdev, dbc); 1857ff13be83SJeffrey Hugo synchronize_srcu(&dbc->ch_lock); 1858ff13be83SJeffrey Hugo } 1859ff13be83SJeffrey Hugo 1860ff13be83SJeffrey Hugo void release_dbc(struct qaic_device *qdev, u32 dbc_id) 1861ff13be83SJeffrey Hugo { 1862ff13be83SJeffrey Hugo struct bo_slice *slice, *slice_temp; 1863ff13be83SJeffrey Hugo struct qaic_bo *bo, *bo_temp; 1864ff13be83SJeffrey Hugo struct dma_bridge_chan *dbc; 1865ff13be83SJeffrey Hugo 1866ff13be83SJeffrey Hugo dbc = &qdev->dbc[dbc_id]; 1867ff13be83SJeffrey Hugo if (!dbc->in_use) 1868ff13be83SJeffrey Hugo return; 1869ff13be83SJeffrey Hugo 1870ff13be83SJeffrey Hugo wakeup_dbc(qdev, dbc_id); 1871ff13be83SJeffrey Hugo 1872ff13be83SJeffrey Hugo dma_free_coherent(&qdev->pdev->dev, dbc->total_size, dbc->req_q_base, dbc->dma_addr); 1873ff13be83SJeffrey Hugo dbc->total_size = 0; 1874ff13be83SJeffrey Hugo dbc->req_q_base = NULL; 1875ff13be83SJeffrey Hugo dbc->dma_addr = 0; 1876ff13be83SJeffrey Hugo dbc->nelem = 0; 1877ff13be83SJeffrey Hugo dbc->usr = NULL; 1878ff13be83SJeffrey Hugo 1879ff13be83SJeffrey Hugo list_for_each_entry_safe(bo, bo_temp, &dbc->bo_lists, bo_list) { 1880ff13be83SJeffrey Hugo list_for_each_entry_safe(slice, slice_temp, &bo->slices, slice) 1881ff13be83SJeffrey Hugo kref_put(&slice->ref_count, free_slice); 1882ff13be83SJeffrey Hugo bo->sliced = false; 1883ff13be83SJeffrey Hugo INIT_LIST_HEAD(&bo->slices); 1884ff13be83SJeffrey Hugo bo->total_slice_nents = 0; 1885ff13be83SJeffrey Hugo bo->dir = 0; 1886ff13be83SJeffrey Hugo bo->dbc = NULL; 1887ff13be83SJeffrey Hugo bo->nr_slice = 0; 1888ff13be83SJeffrey Hugo bo->nr_slice_xfer_done = 0; 1889ff13be83SJeffrey Hugo bo->queued = false; 1890ff13be83SJeffrey Hugo bo->req_id = 0; 1891ff13be83SJeffrey Hugo init_completion(&bo->xfer_done); 1892ff13be83SJeffrey Hugo complete_all(&bo->xfer_done); 1893ff13be83SJeffrey Hugo list_del(&bo->bo_list); 1894ff13be83SJeffrey Hugo bo->perf_stats.req_received_ts = 0; 1895ff13be83SJeffrey Hugo bo->perf_stats.req_submit_ts = 0; 1896ff13be83SJeffrey Hugo bo->perf_stats.req_processed_ts = 0; 1897ff13be83SJeffrey Hugo bo->perf_stats.queue_level_before = 0; 1898ff13be83SJeffrey Hugo } 1899ff13be83SJeffrey Hugo 1900ff13be83SJeffrey Hugo dbc->in_use = false; 1901ff13be83SJeffrey Hugo wake_up(&dbc->dbc_release); 1902ff13be83SJeffrey Hugo } 1903