1 /* 2 * aQuantia Corporation Network Driver 3 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 */ 9 10 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 11 12 #include "aq_ring.h" 13 #include "aq_nic.h" 14 #include "aq_hw.h" 15 16 #include <linux/netdevice.h> 17 #include <linux/etherdevice.h> 18 19 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 20 struct aq_nic_s *aq_nic) 21 { 22 int err = 0; 23 24 self->buff_ring = 25 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); 26 27 if (!self->buff_ring) { 28 err = -ENOMEM; 29 goto err_exit; 30 } 31 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 32 self->size * self->dx_size, 33 &self->dx_ring_pa, GFP_KERNEL); 34 if (!self->dx_ring) { 35 err = -ENOMEM; 36 goto err_exit; 37 } 38 39 err_exit: 40 if (err < 0) { 41 aq_ring_free(self); 42 self = NULL; 43 } 44 return self; 45 } 46 47 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, 48 struct aq_nic_s *aq_nic, 49 unsigned int idx, 50 struct aq_nic_cfg_s *aq_nic_cfg) 51 { 52 int err = 0; 53 54 self->aq_nic = aq_nic; 55 self->idx = idx; 56 self->size = aq_nic_cfg->txds; 57 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 58 59 self = aq_ring_alloc(self, aq_nic); 60 if (!self) { 61 err = -ENOMEM; 62 goto err_exit; 63 } 64 65 err_exit: 66 if (err < 0) { 67 aq_ring_free(self); 68 self = NULL; 69 } 70 return self; 71 } 72 73 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 74 struct aq_nic_s *aq_nic, 75 unsigned int idx, 76 struct aq_nic_cfg_s *aq_nic_cfg) 77 { 78 int err = 0; 79 80 self->aq_nic = aq_nic; 81 self->idx = idx; 82 self->size = aq_nic_cfg->rxds; 83 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 84 85 self = aq_ring_alloc(self, aq_nic); 86 if (!self) { 87 err = -ENOMEM; 88 goto err_exit; 89 } 90 91 err_exit: 92 if (err < 0) { 93 aq_ring_free(self); 94 self = NULL; 95 } 96 return self; 97 } 98 99 int aq_ring_init(struct aq_ring_s *self) 100 { 101 self->hw_head = 0; 102 self->sw_head = 0; 103 self->sw_tail = 0; 104 return 0; 105 } 106 107 void aq_ring_tx_clean(struct aq_ring_s *self) 108 { 109 struct device *dev = aq_nic_get_dev(self->aq_nic); 110 111 for (; self->sw_head != self->hw_head; 112 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 113 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 114 115 if (likely(buff->is_mapped)) { 116 if (unlikely(buff->is_sop)) 117 dma_unmap_single(dev, buff->pa, buff->len, 118 DMA_TO_DEVICE); 119 else 120 dma_unmap_page(dev, buff->pa, buff->len, 121 DMA_TO_DEVICE); 122 } 123 124 if (unlikely(buff->is_eop)) 125 dev_kfree_skb_any(buff->skb); 126 } 127 } 128 129 static inline unsigned int aq_ring_dx_in_range(unsigned int h, unsigned int i, 130 unsigned int t) 131 { 132 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 133 } 134 135 #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) 136 int aq_ring_rx_clean(struct aq_ring_s *self, int *work_done, int budget) 137 { 138 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 139 int err = 0; 140 bool is_rsc_completed = true; 141 142 for (; (self->sw_head != self->hw_head) && budget; 143 self->sw_head = aq_ring_next_dx(self, self->sw_head), 144 --budget, ++(*work_done)) { 145 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 146 struct sk_buff *skb = NULL; 147 unsigned int next_ = 0U; 148 unsigned int i = 0U; 149 struct aq_ring_buff_s *buff_ = NULL; 150 151 if (buff->is_error) { 152 __free_pages(buff->page, 0); 153 continue; 154 } 155 156 if (buff->is_cleaned) 157 continue; 158 159 if (!buff->is_eop) { 160 for (next_ = buff->next, 161 buff_ = &self->buff_ring[next_]; true; 162 next_ = buff_->next, 163 buff_ = &self->buff_ring[next_]) { 164 is_rsc_completed = 165 aq_ring_dx_in_range(self->sw_head, 166 next_, 167 self->hw_head); 168 169 if (unlikely(!is_rsc_completed)) { 170 is_rsc_completed = false; 171 break; 172 } 173 174 if (buff_->is_eop) 175 break; 176 } 177 178 if (!is_rsc_completed) { 179 err = 0; 180 goto err_exit; 181 } 182 } 183 184 /* for single fragment packets use build_skb() */ 185 if (buff->is_eop) { 186 skb = build_skb(page_address(buff->page), 187 buff->len + AQ_SKB_ALIGN); 188 if (unlikely(!skb)) { 189 err = -ENOMEM; 190 goto err_exit; 191 } 192 193 skb_put(skb, buff->len); 194 } else { 195 skb = netdev_alloc_skb(ndev, ETH_HLEN); 196 if (unlikely(!skb)) { 197 err = -ENOMEM; 198 goto err_exit; 199 } 200 skb_put(skb, ETH_HLEN); 201 memcpy(skb->data, page_address(buff->page), ETH_HLEN); 202 203 skb_add_rx_frag(skb, 0, buff->page, ETH_HLEN, 204 buff->len - ETH_HLEN, 205 SKB_TRUESIZE(buff->len - ETH_HLEN)); 206 207 for (i = 1U, next_ = buff->next, 208 buff_ = &self->buff_ring[next_]; true; 209 next_ = buff_->next, 210 buff_ = &self->buff_ring[next_], ++i) { 211 skb_add_rx_frag(skb, i, buff_->page, 0, 212 buff_->len, 213 SKB_TRUESIZE(buff->len - 214 ETH_HLEN)); 215 buff_->is_cleaned = 1; 216 217 if (buff_->is_eop) 218 break; 219 } 220 } 221 222 skb->protocol = eth_type_trans(skb, ndev); 223 if (unlikely(buff->is_cso_err)) { 224 ++self->stats.rx.errors; 225 __skb_mark_checksum_bad(skb); 226 } else { 227 if (buff->is_ip_cso) { 228 __skb_incr_checksum_unnecessary(skb); 229 if (buff->is_udp_cso || buff->is_tcp_cso) 230 __skb_incr_checksum_unnecessary(skb); 231 } else { 232 skb->ip_summed = CHECKSUM_NONE; 233 } 234 } 235 236 skb_set_hash(skb, buff->rss_hash, 237 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 238 PKT_HASH_TYPE_NONE); 239 240 skb_record_rx_queue(skb, self->idx); 241 242 netif_receive_skb(skb); 243 244 ++self->stats.rx.packets; 245 self->stats.rx.bytes += skb->len; 246 } 247 248 err_exit: 249 return err; 250 } 251 252 int aq_ring_rx_fill(struct aq_ring_s *self) 253 { 254 unsigned int pages_order = fls(AQ_CFG_RX_FRAME_MAX / PAGE_SIZE + 255 (AQ_CFG_RX_FRAME_MAX % PAGE_SIZE ? 1 : 0)) - 1; 256 struct aq_ring_buff_s *buff = NULL; 257 int err = 0; 258 int i = 0; 259 260 for (i = aq_ring_avail_dx(self); i--; 261 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 262 buff = &self->buff_ring[self->sw_tail]; 263 264 buff->flags = 0U; 265 buff->len = AQ_CFG_RX_FRAME_MAX; 266 267 buff->page = alloc_pages(GFP_ATOMIC | __GFP_COLD | 268 __GFP_COMP, pages_order); 269 if (!buff->page) { 270 err = -ENOMEM; 271 goto err_exit; 272 } 273 274 buff->pa = dma_map_page(aq_nic_get_dev(self->aq_nic), 275 buff->page, 0, 276 AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); 277 278 if (dma_mapping_error(aq_nic_get_dev(self->aq_nic), buff->pa)) { 279 err = -ENOMEM; 280 goto err_exit; 281 } 282 283 buff = NULL; 284 } 285 286 err_exit: 287 if (err < 0) { 288 if (buff && buff->page) 289 __free_pages(buff->page, 0); 290 } 291 292 return err; 293 } 294 295 void aq_ring_rx_deinit(struct aq_ring_s *self) 296 { 297 if (!self) 298 goto err_exit; 299 300 for (; self->sw_head != self->sw_tail; 301 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 302 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 303 304 dma_unmap_page(aq_nic_get_dev(self->aq_nic), buff->pa, 305 AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); 306 307 __free_pages(buff->page, 0); 308 } 309 310 err_exit:; 311 } 312 313 void aq_ring_free(struct aq_ring_s *self) 314 { 315 if (!self) 316 goto err_exit; 317 318 kfree(self->buff_ring); 319 320 if (self->dx_ring) 321 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 322 self->size * self->dx_size, self->dx_ring, 323 self->dx_ring_pa); 324 325 err_exit:; 326 } 327