1 /* 2 * Copyright(c) 2017 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 /* 49 * This file contains HFI1 support for VNIC SDMA functionality 50 */ 51 52 #include "sdma.h" 53 #include "vnic.h" 54 55 #define HFI1_VNIC_SDMA_Q_ACTIVE BIT(0) 56 #define HFI1_VNIC_SDMA_Q_DEFERRED BIT(1) 57 58 #define HFI1_VNIC_TXREQ_NAME_LEN 32 59 #define HFI1_VNIC_SDMA_DESC_WTRMRK 64 60 #define HFI1_VNIC_SDMA_RETRY_COUNT 1 61 62 /* 63 * struct vnic_txreq - VNIC transmit descriptor 64 * @txreq: sdma transmit request 65 * @sdma: vnic sdma pointer 66 * @skb: skb to send 67 * @pad: pad buffer 68 * @plen: pad length 69 * @pbc_val: pbc value 70 * @retry_count: tx retry count 71 */ 72 struct vnic_txreq { 73 struct sdma_txreq txreq; 74 struct hfi1_vnic_sdma *sdma; 75 76 struct sk_buff *skb; 77 unsigned char pad[HFI1_VNIC_MAX_PAD]; 78 u16 plen; 79 __le64 pbc_val; 80 81 u32 retry_count; 82 }; 83 84 static void vnic_sdma_complete(struct sdma_txreq *txreq, 85 int status) 86 { 87 struct vnic_txreq *tx = container_of(txreq, struct vnic_txreq, txreq); 88 struct hfi1_vnic_sdma *vnic_sdma = tx->sdma; 89 90 sdma_txclean(vnic_sdma->dd, txreq); 91 dev_kfree_skb_any(tx->skb); 92 kmem_cache_free(vnic_sdma->dd->vnic.txreq_cache, tx); 93 } 94 95 static noinline int build_vnic_ulp_payload(struct sdma_engine *sde, 96 struct vnic_txreq *tx) 97 { 98 int i, ret = 0; 99 100 ret = sdma_txadd_kvaddr( 101 sde->dd, 102 &tx->txreq, 103 tx->skb->data, 104 skb_headlen(tx->skb)); 105 if (unlikely(ret)) 106 goto bail_txadd; 107 108 for (i = 0; i < skb_shinfo(tx->skb)->nr_frags; i++) { 109 struct skb_frag_struct *frag = &skb_shinfo(tx->skb)->frags[i]; 110 111 /* combine physically continuous fragments later? */ 112 ret = sdma_txadd_page(sde->dd, 113 &tx->txreq, 114 skb_frag_page(frag), 115 frag->page_offset, 116 skb_frag_size(frag)); 117 if (unlikely(ret)) 118 goto bail_txadd; 119 } 120 121 if (tx->plen) 122 ret = sdma_txadd_kvaddr(sde->dd, &tx->txreq, 123 tx->pad + HFI1_VNIC_MAX_PAD - tx->plen, 124 tx->plen); 125 126 bail_txadd: 127 return ret; 128 } 129 130 static int build_vnic_tx_desc(struct sdma_engine *sde, 131 struct vnic_txreq *tx, 132 u64 pbc) 133 { 134 int ret = 0; 135 u16 hdrbytes = 2 << 2; /* PBC */ 136 137 ret = sdma_txinit_ahg( 138 &tx->txreq, 139 0, 140 hdrbytes + tx->skb->len + tx->plen, 141 0, 142 0, 143 NULL, 144 0, 145 vnic_sdma_complete); 146 if (unlikely(ret)) 147 goto bail_txadd; 148 149 /* add pbc */ 150 tx->pbc_val = cpu_to_le64(pbc); 151 ret = sdma_txadd_kvaddr( 152 sde->dd, 153 &tx->txreq, 154 &tx->pbc_val, 155 hdrbytes); 156 if (unlikely(ret)) 157 goto bail_txadd; 158 159 /* add the ulp payload */ 160 ret = build_vnic_ulp_payload(sde, tx); 161 bail_txadd: 162 return ret; 163 } 164 165 /* setup the last plen bypes of pad */ 166 static inline void hfi1_vnic_update_pad(unsigned char *pad, u8 plen) 167 { 168 pad[HFI1_VNIC_MAX_PAD - 1] = plen - OPA_VNIC_ICRC_TAIL_LEN; 169 } 170 171 int hfi1_vnic_send_dma(struct hfi1_devdata *dd, u8 q_idx, 172 struct hfi1_vnic_vport_info *vinfo, 173 struct sk_buff *skb, u64 pbc, u8 plen) 174 { 175 struct hfi1_vnic_sdma *vnic_sdma = &vinfo->sdma[q_idx]; 176 struct sdma_engine *sde = vnic_sdma->sde; 177 struct vnic_txreq *tx; 178 int ret = -ECOMM; 179 180 if (unlikely(READ_ONCE(vnic_sdma->state) != HFI1_VNIC_SDMA_Q_ACTIVE)) 181 goto tx_err; 182 183 if (unlikely(!sde || !sdma_running(sde))) 184 goto tx_err; 185 186 tx = kmem_cache_alloc(dd->vnic.txreq_cache, GFP_ATOMIC); 187 if (unlikely(!tx)) { 188 ret = -ENOMEM; 189 goto tx_err; 190 } 191 192 tx->sdma = vnic_sdma; 193 tx->skb = skb; 194 hfi1_vnic_update_pad(tx->pad, plen); 195 tx->plen = plen; 196 ret = build_vnic_tx_desc(sde, tx, pbc); 197 if (unlikely(ret)) 198 goto free_desc; 199 tx->retry_count = 0; 200 201 ret = sdma_send_txreq(sde, &vnic_sdma->wait, &tx->txreq, 202 vnic_sdma->pkts_sent); 203 /* When -ECOMM, sdma callback will be called with ABORT status */ 204 if (unlikely(ret && unlikely(ret != -ECOMM))) 205 goto free_desc; 206 207 if (!ret) { 208 vnic_sdma->pkts_sent = true; 209 iowait_starve_clear(vnic_sdma->pkts_sent, &vnic_sdma->wait); 210 } 211 return ret; 212 213 free_desc: 214 sdma_txclean(dd, &tx->txreq); 215 kmem_cache_free(dd->vnic.txreq_cache, tx); 216 tx_err: 217 if (ret != -EBUSY) 218 dev_kfree_skb_any(skb); 219 else 220 vnic_sdma->pkts_sent = false; 221 return ret; 222 } 223 224 /* 225 * hfi1_vnic_sdma_sleep - vnic sdma sleep function 226 * 227 * This function gets called from sdma_send_txreq() when there are not enough 228 * sdma descriptors available to send the packet. It adds Tx queue's wait 229 * structure to sdma engine's dmawait list to be woken up when descriptors 230 * become available. 231 */ 232 static int hfi1_vnic_sdma_sleep(struct sdma_engine *sde, 233 struct iowait *wait, 234 struct sdma_txreq *txreq, 235 uint seq, 236 bool pkts_sent) 237 { 238 struct hfi1_vnic_sdma *vnic_sdma = 239 container_of(wait, struct hfi1_vnic_sdma, wait); 240 struct hfi1_ibdev *dev = &vnic_sdma->dd->verbs_dev; 241 struct vnic_txreq *tx = container_of(txreq, struct vnic_txreq, txreq); 242 243 if (sdma_progress(sde, seq, txreq)) 244 if (tx->retry_count++ < HFI1_VNIC_SDMA_RETRY_COUNT) 245 return -EAGAIN; 246 247 vnic_sdma->state = HFI1_VNIC_SDMA_Q_DEFERRED; 248 write_seqlock(&dev->iowait_lock); 249 if (list_empty(&vnic_sdma->wait.list)) 250 iowait_queue(pkts_sent, wait, &sde->dmawait); 251 write_sequnlock(&dev->iowait_lock); 252 return -EBUSY; 253 } 254 255 /* 256 * hfi1_vnic_sdma_wakeup - vnic sdma wakeup function 257 * 258 * This function gets called when SDMA descriptors becomes available and Tx 259 * queue's wait structure was previously added to sdma engine's dmawait list. 260 * It notifies the upper driver about Tx queue wakeup. 261 */ 262 static void hfi1_vnic_sdma_wakeup(struct iowait *wait, int reason) 263 { 264 struct hfi1_vnic_sdma *vnic_sdma = 265 container_of(wait, struct hfi1_vnic_sdma, wait); 266 struct hfi1_vnic_vport_info *vinfo = vnic_sdma->vinfo; 267 268 vnic_sdma->state = HFI1_VNIC_SDMA_Q_ACTIVE; 269 if (__netif_subqueue_stopped(vinfo->netdev, vnic_sdma->q_idx)) 270 netif_wake_subqueue(vinfo->netdev, vnic_sdma->q_idx); 271 }; 272 273 inline bool hfi1_vnic_sdma_write_avail(struct hfi1_vnic_vport_info *vinfo, 274 u8 q_idx) 275 { 276 struct hfi1_vnic_sdma *vnic_sdma = &vinfo->sdma[q_idx]; 277 278 return (READ_ONCE(vnic_sdma->state) == HFI1_VNIC_SDMA_Q_ACTIVE); 279 } 280 281 void hfi1_vnic_sdma_init(struct hfi1_vnic_vport_info *vinfo) 282 { 283 int i; 284 285 for (i = 0; i < vinfo->num_tx_q; i++) { 286 struct hfi1_vnic_sdma *vnic_sdma = &vinfo->sdma[i]; 287 288 iowait_init(&vnic_sdma->wait, 0, NULL, hfi1_vnic_sdma_sleep, 289 hfi1_vnic_sdma_wakeup, NULL); 290 vnic_sdma->sde = &vinfo->dd->per_sdma[i]; 291 vnic_sdma->dd = vinfo->dd; 292 vnic_sdma->vinfo = vinfo; 293 vnic_sdma->q_idx = i; 294 vnic_sdma->state = HFI1_VNIC_SDMA_Q_ACTIVE; 295 296 /* Add a free descriptor watermark for wakeups */ 297 if (vnic_sdma->sde->descq_cnt > HFI1_VNIC_SDMA_DESC_WTRMRK) { 298 INIT_LIST_HEAD(&vnic_sdma->stx.list); 299 vnic_sdma->stx.num_desc = HFI1_VNIC_SDMA_DESC_WTRMRK; 300 list_add_tail(&vnic_sdma->stx.list, 301 &vnic_sdma->wait.tx_head); 302 } 303 } 304 } 305 306 int hfi1_vnic_txreq_init(struct hfi1_devdata *dd) 307 { 308 char buf[HFI1_VNIC_TXREQ_NAME_LEN]; 309 310 snprintf(buf, sizeof(buf), "hfi1_%u_vnic_txreq_cache", dd->unit); 311 dd->vnic.txreq_cache = kmem_cache_create(buf, 312 sizeof(struct vnic_txreq), 313 0, SLAB_HWCACHE_ALIGN, 314 NULL); 315 if (!dd->vnic.txreq_cache) 316 return -ENOMEM; 317 return 0; 318 } 319 320 void hfi1_vnic_txreq_deinit(struct hfi1_devdata *dd) 321 { 322 kmem_cache_destroy(dd->vnic.txreq_cache); 323 dd->vnic.txreq_cache = NULL; 324 } 325