1f4cd8765SMichael J. Ruhl #ifndef _HFI1_USER_SDMA_H 2f4cd8765SMichael J. Ruhl #define _HFI1_USER_SDMA_H 3f48ad614SDennis Dalessandro /* 45465f110SKaike Wan * Copyright(c) 2015 - 2018 Intel Corporation. 5f48ad614SDennis Dalessandro * 6f48ad614SDennis Dalessandro * This file is provided under a dual BSD/GPLv2 license. When using or 7f48ad614SDennis Dalessandro * redistributing this file, you may do so under either license. 8f48ad614SDennis Dalessandro * 9f48ad614SDennis Dalessandro * GPL LICENSE SUMMARY 10f48ad614SDennis Dalessandro * 11f48ad614SDennis Dalessandro * This program is free software; you can redistribute it and/or modify 12f48ad614SDennis Dalessandro * it under the terms of version 2 of the GNU General Public License as 13f48ad614SDennis Dalessandro * published by the Free Software Foundation. 14f48ad614SDennis Dalessandro * 15f48ad614SDennis Dalessandro * This program is distributed in the hope that it will be useful, but 16f48ad614SDennis Dalessandro * WITHOUT ANY WARRANTY; without even the implied warranty of 17f48ad614SDennis Dalessandro * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18f48ad614SDennis Dalessandro * General Public License for more details. 19f48ad614SDennis Dalessandro * 20f48ad614SDennis Dalessandro * BSD LICENSE 21f48ad614SDennis Dalessandro * 22f48ad614SDennis Dalessandro * Redistribution and use in source and binary forms, with or without 23f48ad614SDennis Dalessandro * modification, are permitted provided that the following conditions 24f48ad614SDennis Dalessandro * are met: 25f48ad614SDennis Dalessandro * 26f48ad614SDennis Dalessandro * - Redistributions of source code must retain the above copyright 27f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer. 28f48ad614SDennis Dalessandro * - Redistributions in binary form must reproduce the above copyright 29f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer in 30f48ad614SDennis Dalessandro * the documentation and/or other materials provided with the 31f48ad614SDennis Dalessandro * distribution. 32f48ad614SDennis Dalessandro * - Neither the name of Intel Corporation nor the names of its 33f48ad614SDennis Dalessandro * contributors may be used to endorse or promote products derived 34f48ad614SDennis Dalessandro * from this software without specific prior written permission. 35f48ad614SDennis Dalessandro * 36f48ad614SDennis Dalessandro * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37f48ad614SDennis Dalessandro * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38f48ad614SDennis Dalessandro * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 39f48ad614SDennis Dalessandro * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 40f48ad614SDennis Dalessandro * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 41f48ad614SDennis Dalessandro * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 42f48ad614SDennis Dalessandro * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 43f48ad614SDennis Dalessandro * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 44f48ad614SDennis Dalessandro * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 45f48ad614SDennis Dalessandro * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 46f48ad614SDennis Dalessandro * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 47f48ad614SDennis Dalessandro * 48f48ad614SDennis Dalessandro */ 49f48ad614SDennis Dalessandro #include <linux/device.h> 50f48ad614SDennis Dalessandro #include <linux/wait.h> 51f48ad614SDennis Dalessandro 52f48ad614SDennis Dalessandro #include "common.h" 53f48ad614SDennis Dalessandro #include "iowait.h" 54f48ad614SDennis Dalessandro #include "user_exp_rcv.h" 55f48ad614SDennis Dalessandro 5632500f27SHarish Chegondi /* The maximum number of Data io vectors per message/request */ 5732500f27SHarish Chegondi #define MAX_VECTORS_PER_REQ 8 5832500f27SHarish Chegondi /* 5932500f27SHarish Chegondi * Maximum number of packet to send from each message/request 6032500f27SHarish Chegondi * before moving to the next one. 6132500f27SHarish Chegondi */ 6232500f27SHarish Chegondi #define MAX_PKTS_PER_QUEUE 16 6332500f27SHarish Chegondi 6432500f27SHarish Chegondi #define num_pages(x) (1 + ((((x) - 1) & PAGE_MASK) >> PAGE_SHIFT)) 6532500f27SHarish Chegondi 6632500f27SHarish Chegondi #define req_opcode(x) \ 6732500f27SHarish Chegondi (((x) >> HFI1_SDMA_REQ_OPCODE_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK) 6832500f27SHarish Chegondi #define req_version(x) \ 6932500f27SHarish Chegondi (((x) >> HFI1_SDMA_REQ_VERSION_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK) 7032500f27SHarish Chegondi #define req_iovcnt(x) \ 7132500f27SHarish Chegondi (((x) >> HFI1_SDMA_REQ_IOVCNT_SHIFT) & HFI1_SDMA_REQ_IOVCNT_MASK) 7232500f27SHarish Chegondi 7332500f27SHarish Chegondi /* Number of BTH.PSN bits used for sequence number in expected rcvs */ 7432500f27SHarish Chegondi #define BTH_SEQ_MASK 0x7ffull 7532500f27SHarish Chegondi 7632500f27SHarish Chegondi #define AHG_KDETH_INTR_SHIFT 12 7732500f27SHarish Chegondi #define AHG_KDETH_SH_SHIFT 13 7832500f27SHarish Chegondi #define AHG_KDETH_ARRAY_SIZE 9 7932500f27SHarish Chegondi 8032500f27SHarish Chegondi #define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4) 8132500f27SHarish Chegondi #define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff) 8232500f27SHarish Chegondi 83d34ed562SHarish Chegondi /** 84d34ed562SHarish Chegondi * Build an SDMA AHG header update descriptor and save it to an array. 85d34ed562SHarish Chegondi * @arr - Array to save the descriptor to. 86d34ed562SHarish Chegondi * @idx - Index of the array at which the descriptor will be saved. 87d34ed562SHarish Chegondi * @array_size - Size of the array arr. 88d34ed562SHarish Chegondi * @dw - Update index into the header in DWs. 89d34ed562SHarish Chegondi * @bit - Start bit. 90d34ed562SHarish Chegondi * @width - Field width. 91d34ed562SHarish Chegondi * @value - 16 bits of immediate data to write into the field. 92d34ed562SHarish Chegondi * Returns -ERANGE if idx is invalid. If successful, returns the next index 93d34ed562SHarish Chegondi * (idx + 1) of the array to be used for the next descriptor. 94d34ed562SHarish Chegondi */ 95d34ed562SHarish Chegondi static inline int ahg_header_set(u32 *arr, int idx, size_t array_size, 96d34ed562SHarish Chegondi u8 dw, u8 bit, u8 width, u16 value) 97d34ed562SHarish Chegondi { 98d34ed562SHarish Chegondi if ((size_t)idx >= array_size) 99d34ed562SHarish Chegondi return -ERANGE; 100d34ed562SHarish Chegondi arr[idx++] = sdma_build_ahg_descriptor(value, dw, bit, width); 101d34ed562SHarish Chegondi return idx; 102d34ed562SHarish Chegondi } 10332500f27SHarish Chegondi 10432500f27SHarish Chegondi /* Tx request flag bits */ 10532500f27SHarish Chegondi #define TXREQ_FLAGS_REQ_ACK BIT(0) /* Set the ACK bit in the header */ 10632500f27SHarish Chegondi #define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1) /* Disable header suppression */ 10732500f27SHarish Chegondi 10828a9a9e8SMichael J. Ruhl enum pkt_q_sdma_state { 10928a9a9e8SMichael J. Ruhl SDMA_PKT_Q_ACTIVE, 11028a9a9e8SMichael J. Ruhl SDMA_PKT_Q_DEFERRED, 11128a9a9e8SMichael J. Ruhl }; 11232500f27SHarish Chegondi 11332500f27SHarish Chegondi /* 11432500f27SHarish Chegondi * Maximum retry attempts to submit a TX request 11532500f27SHarish Chegondi * before putting the process to sleep. 11632500f27SHarish Chegondi */ 11732500f27SHarish Chegondi #define MAX_DEFER_RETRY_COUNT 1 11832500f27SHarish Chegondi 11932500f27SHarish Chegondi #define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */ 12032500f27SHarish Chegondi 12132500f27SHarish Chegondi #define SDMA_DBG(req, fmt, ...) \ 12232500f27SHarish Chegondi hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \ 12332500f27SHarish Chegondi (req)->pq->ctxt, (req)->pq->subctxt, (req)->info.comp_idx, \ 12432500f27SHarish Chegondi ##__VA_ARGS__) 12532500f27SHarish Chegondi 126f48ad614SDennis Dalessandro struct hfi1_user_sdma_pkt_q { 127e6f7622dSMichael J. Ruhl u16 ctxt; 1288737ce95SMichael J. Ruhl u16 subctxt; 129f48ad614SDennis Dalessandro u16 n_max_reqs; 130f48ad614SDennis Dalessandro atomic_t n_reqs; 131f48ad614SDennis Dalessandro u16 reqidx; 132f48ad614SDennis Dalessandro struct hfi1_devdata *dd; 133f48ad614SDennis Dalessandro struct kmem_cache *txreq_cache; 134f48ad614SDennis Dalessandro struct user_sdma_request *reqs; 1357b3256e3SDean Luick unsigned long *req_in_use; 136f48ad614SDennis Dalessandro struct iowait busy; 13728a9a9e8SMichael J. Ruhl enum pkt_q_sdma_state state; 138f48ad614SDennis Dalessandro wait_queue_head_t wait; 139f48ad614SDennis Dalessandro unsigned long unpinned; 140e0b09ac5SDean Luick struct mmu_rb_handler *handler; 141b7df192fSDean Luick atomic_t n_locked; 1423faa3d9aSIra Weiny struct mm_struct *mm; 143f48ad614SDennis Dalessandro }; 144f48ad614SDennis Dalessandro 145f48ad614SDennis Dalessandro struct hfi1_user_sdma_comp_q { 146f48ad614SDennis Dalessandro u16 nentries; 147f48ad614SDennis Dalessandro struct hfi1_sdma_comp_entry *comps; 148f48ad614SDennis Dalessandro }; 149f48ad614SDennis Dalessandro 15032500f27SHarish Chegondi struct sdma_mmu_node { 15132500f27SHarish Chegondi struct mmu_rb_node rb; 15232500f27SHarish Chegondi struct hfi1_user_sdma_pkt_q *pq; 15332500f27SHarish Chegondi atomic_t refcount; 15432500f27SHarish Chegondi struct page **pages; 15532500f27SHarish Chegondi unsigned int npages; 15632500f27SHarish Chegondi }; 15732500f27SHarish Chegondi 15832500f27SHarish Chegondi struct user_sdma_iovec { 15932500f27SHarish Chegondi struct list_head list; 16032500f27SHarish Chegondi struct iovec iov; 16132500f27SHarish Chegondi /* number of pages in this vector */ 16232500f27SHarish Chegondi unsigned int npages; 16332500f27SHarish Chegondi /* array of pinned pages for this vector */ 16432500f27SHarish Chegondi struct page **pages; 16532500f27SHarish Chegondi /* 16632500f27SHarish Chegondi * offset into the virtual address space of the vector at 16732500f27SHarish Chegondi * which we last left off. 16832500f27SHarish Chegondi */ 16932500f27SHarish Chegondi u64 offset; 17032500f27SHarish Chegondi struct sdma_mmu_node *node; 17132500f27SHarish Chegondi }; 17232500f27SHarish Chegondi 17332500f27SHarish Chegondi /* evict operation argument */ 17432500f27SHarish Chegondi struct evict_data { 17532500f27SHarish Chegondi u32 cleared; /* count evicted so far */ 17632500f27SHarish Chegondi u32 target; /* target count to evict */ 17732500f27SHarish Chegondi }; 17832500f27SHarish Chegondi 17932500f27SHarish Chegondi struct user_sdma_request { 18032500f27SHarish Chegondi /* This is the original header from user space */ 18132500f27SHarish Chegondi struct hfi1_pkt_header hdr; 18232500f27SHarish Chegondi 18332500f27SHarish Chegondi /* Read mostly fields */ 18432500f27SHarish Chegondi struct hfi1_user_sdma_pkt_q *pq ____cacheline_aligned_in_smp; 18532500f27SHarish Chegondi struct hfi1_user_sdma_comp_q *cq; 18632500f27SHarish Chegondi /* 18732500f27SHarish Chegondi * Pointer to the SDMA engine for this request. 18832500f27SHarish Chegondi * Since different request could be on different VLs, 18932500f27SHarish Chegondi * each request will need it's own engine pointer. 19032500f27SHarish Chegondi */ 19132500f27SHarish Chegondi struct sdma_engine *sde; 19232500f27SHarish Chegondi struct sdma_req_info info; 19332500f27SHarish Chegondi /* TID array values copied from the tid_iov vector */ 19432500f27SHarish Chegondi u32 *tids; 19532500f27SHarish Chegondi /* total length of the data in the request */ 19632500f27SHarish Chegondi u32 data_len; 19732500f27SHarish Chegondi /* number of elements copied to the tids array */ 19832500f27SHarish Chegondi u16 n_tids; 19932500f27SHarish Chegondi /* 20032500f27SHarish Chegondi * We copy the iovs for this request (based on 20132500f27SHarish Chegondi * info.iovcnt). These are only the data vectors 20232500f27SHarish Chegondi */ 20332500f27SHarish Chegondi u8 data_iovs; 20432500f27SHarish Chegondi s8 ahg_idx; 20532500f27SHarish Chegondi 20632500f27SHarish Chegondi /* Writeable fields shared with interrupt */ 2073ca633f1SMichael J. Ruhl u16 seqcomp ____cacheline_aligned_in_smp; 2083ca633f1SMichael J. Ruhl u16 seqsubmitted; 20932500f27SHarish Chegondi 21032500f27SHarish Chegondi /* Send side fields */ 21132500f27SHarish Chegondi struct list_head txps ____cacheline_aligned_in_smp; 2123ca633f1SMichael J. Ruhl u16 seqnum; 21332500f27SHarish Chegondi /* 21432500f27SHarish Chegondi * KDETH.OFFSET (TID) field 21532500f27SHarish Chegondi * The offset can cover multiple packets, depending on the 21632500f27SHarish Chegondi * size of the TID entry. 21732500f27SHarish Chegondi */ 21832500f27SHarish Chegondi u32 tidoffset; 21932500f27SHarish Chegondi /* 22032500f27SHarish Chegondi * KDETH.Offset (Eager) field 22132500f27SHarish Chegondi * We need to remember the initial value so the headers 22232500f27SHarish Chegondi * can be updated properly. 22332500f27SHarish Chegondi */ 22432500f27SHarish Chegondi u32 koffset; 22532500f27SHarish Chegondi u32 sent; 22632500f27SHarish Chegondi /* TID index copied from the tid_iov vector */ 22732500f27SHarish Chegondi u16 tididx; 22832500f27SHarish Chegondi /* progress index moving along the iovs array */ 22932500f27SHarish Chegondi u8 iov_idx; 23032500f27SHarish Chegondi u8 has_error; 23132500f27SHarish Chegondi 23232500f27SHarish Chegondi struct user_sdma_iovec iovs[MAX_VECTORS_PER_REQ]; 23332500f27SHarish Chegondi } ____cacheline_aligned_in_smp; 23432500f27SHarish Chegondi 23532500f27SHarish Chegondi /* 23632500f27SHarish Chegondi * A single txreq could span up to 3 physical pages when the MTU 23732500f27SHarish Chegondi * is sufficiently large (> 4K). Each of the IOV pointers also 23832500f27SHarish Chegondi * needs it's own set of flags so the vector has been handled 23932500f27SHarish Chegondi * independently of each other. 24032500f27SHarish Chegondi */ 24132500f27SHarish Chegondi struct user_sdma_txreq { 24232500f27SHarish Chegondi /* Packet header for the txreq */ 24332500f27SHarish Chegondi struct hfi1_pkt_header hdr; 24432500f27SHarish Chegondi struct sdma_txreq txreq; 24532500f27SHarish Chegondi struct list_head list; 24632500f27SHarish Chegondi struct user_sdma_request *req; 24732500f27SHarish Chegondi u16 flags; 24832500f27SHarish Chegondi unsigned int busycount; 2493ca633f1SMichael J. Ruhl u16 seqnum; 25032500f27SHarish Chegondi }; 25132500f27SHarish Chegondi 2525042cddfSMichael J. Ruhl int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt, 2535042cddfSMichael J. Ruhl struct hfi1_filedata *fd); 254e87473bcSMichael J. Ruhl int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd, 255e87473bcSMichael J. Ruhl struct hfi1_ctxtdata *uctxt); 2565042cddfSMichael J. Ruhl int hfi1_user_sdma_process_request(struct hfi1_filedata *fd, 2575042cddfSMichael J. Ruhl struct iovec *iovec, unsigned long dim, 2585042cddfSMichael J. Ruhl unsigned long *count); 2595042cddfSMichael J. Ruhl 260f4cd8765SMichael J. Ruhl #endif /* _HFI1_USER_SDMA_H */ 261