1 /* 2 * aQuantia Corporation Network Driver 3 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 */ 9 10 /* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings. 11 * Definition of functions for Rx and Tx rings. Friendly module for aq_nic. 12 */ 13 14 #include "aq_vec.h" 15 #include "aq_nic.h" 16 #include "aq_ring.h" 17 #include "aq_hw.h" 18 19 #include <linux/netdevice.h> 20 21 struct aq_vec_s { 22 const struct aq_hw_ops *aq_hw_ops; 23 struct aq_hw_s *aq_hw; 24 struct aq_nic_s *aq_nic; 25 unsigned int tx_rings; 26 unsigned int rx_rings; 27 struct aq_ring_param_s aq_ring_param; 28 struct napi_struct napi; 29 struct aq_ring_s ring[AQ_CFG_TCS_MAX][2]; 30 }; 31 32 #define AQ_VEC_TX_ID 0 33 #define AQ_VEC_RX_ID 1 34 35 static int aq_vec_poll(struct napi_struct *napi, int budget) 36 { 37 struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi); 38 struct aq_ring_s *ring = NULL; 39 int work_done = 0; 40 int err = 0; 41 unsigned int i = 0U; 42 unsigned int sw_tail_old = 0U; 43 bool was_tx_cleaned = false; 44 45 if (!self) { 46 err = -EINVAL; 47 } else { 48 for (i = 0U, ring = self->ring[0]; 49 self->tx_rings > i; ++i, ring = self->ring[i]) { 50 if (self->aq_hw_ops->hw_ring_tx_head_update) { 51 err = self->aq_hw_ops->hw_ring_tx_head_update( 52 self->aq_hw, 53 &ring[AQ_VEC_TX_ID]); 54 if (err < 0) 55 goto err_exit; 56 } 57 58 if (ring[AQ_VEC_TX_ID].sw_head != 59 ring[AQ_VEC_TX_ID].hw_head) { 60 aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]); 61 aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]); 62 was_tx_cleaned = true; 63 } 64 65 err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw, 66 &ring[AQ_VEC_RX_ID]); 67 if (err < 0) 68 goto err_exit; 69 70 if (ring[AQ_VEC_RX_ID].sw_head != 71 ring[AQ_VEC_RX_ID].hw_head) { 72 err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID], 73 napi, 74 &work_done, 75 budget - work_done); 76 if (err < 0) 77 goto err_exit; 78 79 sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail; 80 81 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]); 82 if (err < 0) 83 goto err_exit; 84 85 err = self->aq_hw_ops->hw_ring_rx_fill( 86 self->aq_hw, 87 &ring[AQ_VEC_RX_ID], sw_tail_old); 88 if (err < 0) 89 goto err_exit; 90 } 91 } 92 93 if (was_tx_cleaned) 94 work_done = budget; 95 96 if (work_done < budget) { 97 napi_complete_done(napi, work_done); 98 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 99 1U << self->aq_ring_param.vec_idx); 100 } 101 } 102 err_exit: 103 return work_done; 104 } 105 106 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx, 107 struct aq_nic_cfg_s *aq_nic_cfg) 108 { 109 struct aq_vec_s *self = NULL; 110 struct aq_ring_s *ring = NULL; 111 unsigned int i = 0U; 112 int err = 0; 113 114 self = kzalloc(sizeof(*self), GFP_KERNEL); 115 if (!self) { 116 err = -ENOMEM; 117 goto err_exit; 118 } 119 120 self->aq_nic = aq_nic; 121 self->aq_ring_param.vec_idx = idx; 122 self->aq_ring_param.cpu = 123 idx + aq_nic_cfg->aq_rss.base_cpu_number; 124 125 cpumask_set_cpu(self->aq_ring_param.cpu, 126 &self->aq_ring_param.affinity_mask); 127 128 self->tx_rings = 0; 129 self->rx_rings = 0; 130 131 netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, 132 aq_vec_poll, AQ_CFG_NAPI_WEIGHT); 133 134 for (i = 0; i < aq_nic_cfg->tcs; ++i) { 135 unsigned int idx_ring = AQ_NIC_TCVEC2RING(self->nic, 136 self->tx_rings, 137 self->aq_ring_param.vec_idx); 138 139 ring = aq_ring_tx_alloc(&self->ring[i][AQ_VEC_TX_ID], aq_nic, 140 idx_ring, aq_nic_cfg); 141 if (!ring) { 142 err = -ENOMEM; 143 goto err_exit; 144 } 145 146 ++self->tx_rings; 147 148 aq_nic_set_tx_ring(aq_nic, idx_ring, ring); 149 150 ring = aq_ring_rx_alloc(&self->ring[i][AQ_VEC_RX_ID], aq_nic, 151 idx_ring, aq_nic_cfg); 152 if (!ring) { 153 err = -ENOMEM; 154 goto err_exit; 155 } 156 157 ++self->rx_rings; 158 } 159 160 err_exit: 161 if (err < 0) { 162 aq_vec_free(self); 163 self = NULL; 164 } 165 return self; 166 } 167 168 int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops, 169 struct aq_hw_s *aq_hw) 170 { 171 struct aq_ring_s *ring = NULL; 172 unsigned int i = 0U; 173 int err = 0; 174 175 self->aq_hw_ops = aq_hw_ops; 176 self->aq_hw = aq_hw; 177 178 for (i = 0U, ring = self->ring[0]; 179 self->tx_rings > i; ++i, ring = self->ring[i]) { 180 err = aq_ring_init(&ring[AQ_VEC_TX_ID]); 181 if (err < 0) 182 goto err_exit; 183 184 err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw, 185 &ring[AQ_VEC_TX_ID], 186 &self->aq_ring_param); 187 if (err < 0) 188 goto err_exit; 189 190 err = aq_ring_init(&ring[AQ_VEC_RX_ID]); 191 if (err < 0) 192 goto err_exit; 193 194 err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw, 195 &ring[AQ_VEC_RX_ID], 196 &self->aq_ring_param); 197 if (err < 0) 198 goto err_exit; 199 200 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]); 201 if (err < 0) 202 goto err_exit; 203 204 err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw, 205 &ring[AQ_VEC_RX_ID], 0U); 206 if (err < 0) 207 goto err_exit; 208 } 209 210 err_exit: 211 return err; 212 } 213 214 int aq_vec_start(struct aq_vec_s *self) 215 { 216 struct aq_ring_s *ring = NULL; 217 unsigned int i = 0U; 218 int err = 0; 219 220 for (i = 0U, ring = self->ring[0]; 221 self->tx_rings > i; ++i, ring = self->ring[i]) { 222 err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw, 223 &ring[AQ_VEC_TX_ID]); 224 if (err < 0) 225 goto err_exit; 226 227 err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw, 228 &ring[AQ_VEC_RX_ID]); 229 if (err < 0) 230 goto err_exit; 231 } 232 233 napi_enable(&self->napi); 234 235 err_exit: 236 return err; 237 } 238 239 void aq_vec_stop(struct aq_vec_s *self) 240 { 241 struct aq_ring_s *ring = NULL; 242 unsigned int i = 0U; 243 244 for (i = 0U, ring = self->ring[0]; 245 self->tx_rings > i; ++i, ring = self->ring[i]) { 246 self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw, 247 &ring[AQ_VEC_TX_ID]); 248 249 self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw, 250 &ring[AQ_VEC_RX_ID]); 251 } 252 253 napi_disable(&self->napi); 254 } 255 256 void aq_vec_deinit(struct aq_vec_s *self) 257 { 258 struct aq_ring_s *ring = NULL; 259 unsigned int i = 0U; 260 261 if (!self) 262 goto err_exit; 263 264 for (i = 0U, ring = self->ring[0]; 265 self->tx_rings > i; ++i, ring = self->ring[i]) { 266 aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]); 267 aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]); 268 } 269 err_exit:; 270 } 271 272 void aq_vec_free(struct aq_vec_s *self) 273 { 274 struct aq_ring_s *ring = NULL; 275 unsigned int i = 0U; 276 277 if (!self) 278 goto err_exit; 279 280 for (i = 0U, ring = self->ring[0]; 281 self->tx_rings > i; ++i, ring = self->ring[i]) { 282 aq_ring_free(&ring[AQ_VEC_TX_ID]); 283 aq_ring_free(&ring[AQ_VEC_RX_ID]); 284 } 285 286 netif_napi_del(&self->napi); 287 288 kfree(self); 289 290 err_exit:; 291 } 292 293 irqreturn_t aq_vec_isr(int irq, void *private) 294 { 295 struct aq_vec_s *self = private; 296 int err = 0; 297 298 if (!self) { 299 err = -EINVAL; 300 goto err_exit; 301 } 302 napi_schedule(&self->napi); 303 304 err_exit: 305 return err >= 0 ? IRQ_HANDLED : IRQ_NONE; 306 } 307 308 irqreturn_t aq_vec_isr_legacy(int irq, void *private) 309 { 310 struct aq_vec_s *self = private; 311 u64 irq_mask = 0U; 312 irqreturn_t err = 0; 313 314 if (!self) { 315 err = -EINVAL; 316 goto err_exit; 317 } 318 err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask); 319 if (err < 0) 320 goto err_exit; 321 322 if (irq_mask) { 323 self->aq_hw_ops->hw_irq_disable(self->aq_hw, 324 1U << self->aq_ring_param.vec_idx); 325 napi_schedule(&self->napi); 326 } else { 327 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U); 328 err = IRQ_NONE; 329 } 330 331 err_exit: 332 return err >= 0 ? IRQ_HANDLED : IRQ_NONE; 333 } 334 335 cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self) 336 { 337 return &self->aq_ring_param.affinity_mask; 338 } 339 340 void aq_vec_add_stats(struct aq_vec_s *self, 341 struct aq_ring_stats_rx_s *stats_rx, 342 struct aq_ring_stats_tx_s *stats_tx) 343 { 344 struct aq_ring_s *ring = NULL; 345 unsigned int r = 0U; 346 347 for (r = 0U, ring = self->ring[0]; 348 self->tx_rings > r; ++r, ring = self->ring[r]) { 349 struct aq_ring_stats_tx_s *tx = &ring[AQ_VEC_TX_ID].stats.tx; 350 struct aq_ring_stats_rx_s *rx = &ring[AQ_VEC_RX_ID].stats.rx; 351 352 stats_rx->packets += rx->packets; 353 stats_rx->bytes += rx->bytes; 354 stats_rx->errors += rx->errors; 355 stats_rx->jumbo_packets += rx->jumbo_packets; 356 stats_rx->lro_packets += rx->lro_packets; 357 358 stats_tx->packets += tx->packets; 359 stats_tx->bytes += tx->bytes; 360 stats_tx->errors += tx->errors; 361 stats_tx->queue_restarts += tx->queue_restarts; 362 } 363 } 364 365 int aq_vec_get_sw_stats(struct aq_vec_s *self, u64 *data, unsigned int *p_count) 366 { 367 unsigned int count = 0U; 368 struct aq_ring_stats_rx_s stats_rx; 369 struct aq_ring_stats_tx_s stats_tx; 370 371 memset(&stats_rx, 0U, sizeof(struct aq_ring_stats_rx_s)); 372 memset(&stats_tx, 0U, sizeof(struct aq_ring_stats_tx_s)); 373 aq_vec_add_stats(self, &stats_rx, &stats_tx); 374 375 /* This data should mimic aq_ethtool_queue_stat_names structure 376 */ 377 data[count] += stats_rx.packets; 378 data[++count] += stats_tx.packets; 379 data[++count] += stats_tx.queue_restarts; 380 data[++count] += stats_rx.jumbo_packets; 381 data[++count] += stats_rx.lro_packets; 382 data[++count] += stats_rx.errors; 383 384 if (p_count) 385 *p_count = ++count; 386 387 return 0; 388 } 389