1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings. 9 * Definition of functions for Rx and Tx rings. Friendly module for aq_nic. 10 */ 11 12 #include "aq_vec.h" 13 14 struct aq_vec_s { 15 const struct aq_hw_ops *aq_hw_ops; 16 struct aq_hw_s *aq_hw; 17 struct aq_nic_s *aq_nic; 18 unsigned int tx_rings; 19 unsigned int rx_rings; 20 struct aq_ring_param_s aq_ring_param; 21 struct napi_struct napi; 22 struct aq_ring_s ring[AQ_CFG_TCS_MAX][2]; 23 }; 24 25 #define AQ_VEC_TX_ID 0 26 #define AQ_VEC_RX_ID 1 27 28 static int aq_vec_poll(struct napi_struct *napi, int budget) 29 { 30 struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi); 31 unsigned int sw_tail_old = 0U; 32 struct aq_ring_s *ring = NULL; 33 bool was_tx_cleaned = true; 34 unsigned int i = 0U; 35 int work_done = 0; 36 int err = 0; 37 38 if (!self) { 39 err = -EINVAL; 40 } else { 41 for (i = 0U; self->tx_rings > i; ++i) { 42 ring = self->ring[i]; 43 u64_stats_update_begin(&ring[AQ_VEC_RX_ID].stats.rx.syncp); 44 ring[AQ_VEC_RX_ID].stats.rx.polls++; 45 u64_stats_update_end(&ring[AQ_VEC_RX_ID].stats.rx.syncp); 46 if (self->aq_hw_ops->hw_ring_tx_head_update) { 47 err = self->aq_hw_ops->hw_ring_tx_head_update( 48 self->aq_hw, 49 &ring[AQ_VEC_TX_ID]); 50 if (err < 0) 51 goto err_exit; 52 } 53 54 if (ring[AQ_VEC_TX_ID].sw_head != 55 ring[AQ_VEC_TX_ID].hw_head) { 56 was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]); 57 aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]); 58 } 59 60 err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw, 61 &ring[AQ_VEC_RX_ID]); 62 if (err < 0) 63 goto err_exit; 64 65 if (ring[AQ_VEC_RX_ID].sw_head != 66 ring[AQ_VEC_RX_ID].hw_head) { 67 err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID], 68 napi, 69 &work_done, 70 budget - work_done); 71 if (err < 0) 72 goto err_exit; 73 74 sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail; 75 76 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]); 77 if (err < 0) 78 goto err_exit; 79 80 err = self->aq_hw_ops->hw_ring_rx_fill( 81 self->aq_hw, 82 &ring[AQ_VEC_RX_ID], sw_tail_old); 83 if (err < 0) 84 goto err_exit; 85 } 86 } 87 88 err_exit: 89 if (!was_tx_cleaned) 90 work_done = budget; 91 92 if (work_done < budget) { 93 napi_complete_done(napi, work_done); 94 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 95 1U << self->aq_ring_param.vec_idx); 96 } 97 } 98 99 return work_done; 100 } 101 102 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx, 103 struct aq_nic_cfg_s *aq_nic_cfg) 104 { 105 struct aq_vec_s *self = NULL; 106 107 self = kzalloc(sizeof(*self), GFP_KERNEL); 108 if (!self) 109 goto err_exit; 110 111 self->aq_nic = aq_nic; 112 self->aq_ring_param.vec_idx = idx; 113 self->aq_ring_param.cpu = 114 idx + aq_nic_cfg->aq_rss.base_cpu_number; 115 116 cpumask_set_cpu(self->aq_ring_param.cpu, 117 &self->aq_ring_param.affinity_mask); 118 119 self->tx_rings = 0; 120 self->rx_rings = 0; 121 122 netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, aq_vec_poll); 123 124 err_exit: 125 return self; 126 } 127 128 int aq_vec_ring_alloc(struct aq_vec_s *self, struct aq_nic_s *aq_nic, 129 unsigned int idx, struct aq_nic_cfg_s *aq_nic_cfg) 130 { 131 struct aq_ring_s *ring = NULL; 132 unsigned int i = 0U; 133 int err = 0; 134 135 for (i = 0; i < aq_nic_cfg->tcs; ++i) { 136 const unsigned int idx_ring = AQ_NIC_CFG_TCVEC2RING(aq_nic_cfg, 137 i, idx); 138 139 ring = aq_ring_tx_alloc(&self->ring[i][AQ_VEC_TX_ID], aq_nic, 140 idx_ring, aq_nic_cfg); 141 if (!ring) { 142 err = -ENOMEM; 143 goto err_exit; 144 } 145 146 ++self->tx_rings; 147 148 aq_nic_set_tx_ring(aq_nic, idx_ring, ring); 149 150 if (xdp_rxq_info_reg(&self->ring[i][AQ_VEC_RX_ID].xdp_rxq, 151 aq_nic->ndev, idx, 152 self->napi.napi_id) < 0) { 153 err = -ENOMEM; 154 goto err_exit; 155 } 156 if (xdp_rxq_info_reg_mem_model(&self->ring[i][AQ_VEC_RX_ID].xdp_rxq, 157 MEM_TYPE_PAGE_SHARED, NULL) < 0) { 158 xdp_rxq_info_unreg(&self->ring[i][AQ_VEC_RX_ID].xdp_rxq); 159 err = -ENOMEM; 160 goto err_exit; 161 } 162 163 ring = aq_ring_rx_alloc(&self->ring[i][AQ_VEC_RX_ID], aq_nic, 164 idx_ring, aq_nic_cfg); 165 if (!ring) { 166 xdp_rxq_info_unreg(&self->ring[i][AQ_VEC_RX_ID].xdp_rxq); 167 err = -ENOMEM; 168 goto err_exit; 169 } 170 171 ++self->rx_rings; 172 } 173 174 err_exit: 175 if (err < 0) { 176 aq_vec_ring_free(self); 177 self = NULL; 178 } 179 180 return err; 181 } 182 183 int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops, 184 struct aq_hw_s *aq_hw) 185 { 186 struct aq_ring_s *ring = NULL; 187 unsigned int i = 0U; 188 int err = 0; 189 190 self->aq_hw_ops = aq_hw_ops; 191 self->aq_hw = aq_hw; 192 193 for (i = 0U; self->tx_rings > i; ++i) { 194 ring = self->ring[i]; 195 err = aq_ring_init(&ring[AQ_VEC_TX_ID], ATL_RING_TX); 196 if (err < 0) 197 goto err_exit; 198 199 err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw, 200 &ring[AQ_VEC_TX_ID], 201 &self->aq_ring_param); 202 if (err < 0) 203 goto err_exit; 204 205 err = aq_ring_init(&ring[AQ_VEC_RX_ID], ATL_RING_RX); 206 if (err < 0) 207 goto err_exit; 208 209 err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw, 210 &ring[AQ_VEC_RX_ID], 211 &self->aq_ring_param); 212 if (err < 0) 213 goto err_exit; 214 215 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]); 216 if (err < 0) 217 goto err_exit; 218 219 err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw, 220 &ring[AQ_VEC_RX_ID], 0U); 221 if (err < 0) 222 goto err_exit; 223 } 224 225 err_exit: 226 return err; 227 } 228 229 int aq_vec_start(struct aq_vec_s *self) 230 { 231 struct aq_ring_s *ring = NULL; 232 unsigned int i = 0U; 233 int err = 0; 234 235 for (i = 0U; self->tx_rings > i; ++i) { 236 ring = self->ring[i]; 237 err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw, 238 &ring[AQ_VEC_TX_ID]); 239 if (err < 0) 240 goto err_exit; 241 242 err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw, 243 &ring[AQ_VEC_RX_ID]); 244 if (err < 0) 245 goto err_exit; 246 } 247 248 napi_enable(&self->napi); 249 250 err_exit: 251 return err; 252 } 253 254 void aq_vec_stop(struct aq_vec_s *self) 255 { 256 struct aq_ring_s *ring = NULL; 257 unsigned int i = 0U; 258 259 for (i = 0U; self->tx_rings > i; ++i) { 260 ring = self->ring[i]; 261 self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw, 262 &ring[AQ_VEC_TX_ID]); 263 264 self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw, 265 &ring[AQ_VEC_RX_ID]); 266 } 267 268 napi_disable(&self->napi); 269 } 270 271 void aq_vec_deinit(struct aq_vec_s *self) 272 { 273 struct aq_ring_s *ring = NULL; 274 unsigned int i = 0U; 275 276 if (!self) 277 goto err_exit; 278 279 for (i = 0U; self->tx_rings > i; ++i) { 280 ring = self->ring[i]; 281 aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]); 282 aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]); 283 } 284 285 err_exit:; 286 } 287 288 void aq_vec_free(struct aq_vec_s *self) 289 { 290 if (!self) 291 goto err_exit; 292 293 netif_napi_del(&self->napi); 294 295 kfree(self); 296 297 err_exit:; 298 } 299 300 void aq_vec_ring_free(struct aq_vec_s *self) 301 { 302 struct aq_ring_s *ring = NULL; 303 unsigned int i = 0U; 304 305 if (!self) 306 goto err_exit; 307 308 for (i = 0U; self->tx_rings > i; ++i) { 309 ring = self->ring[i]; 310 aq_ring_free(&ring[AQ_VEC_TX_ID]); 311 if (i < self->rx_rings) { 312 xdp_rxq_info_unreg(&ring[AQ_VEC_RX_ID].xdp_rxq); 313 aq_ring_free(&ring[AQ_VEC_RX_ID]); 314 } 315 } 316 317 self->tx_rings = 0; 318 self->rx_rings = 0; 319 err_exit:; 320 } 321 322 irqreturn_t aq_vec_isr(int irq, void *private) 323 { 324 struct aq_vec_s *self = private; 325 int err = 0; 326 327 if (!self) { 328 err = -EINVAL; 329 goto err_exit; 330 } 331 napi_schedule(&self->napi); 332 333 err_exit: 334 return err >= 0 ? IRQ_HANDLED : IRQ_NONE; 335 } 336 337 irqreturn_t aq_vec_isr_legacy(int irq, void *private) 338 { 339 struct aq_vec_s *self = private; 340 u64 irq_mask = 0U; 341 int err; 342 343 if (!self) 344 return IRQ_NONE; 345 err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask); 346 if (err < 0) 347 return IRQ_NONE; 348 349 if (irq_mask) { 350 self->aq_hw_ops->hw_irq_disable(self->aq_hw, 351 1U << self->aq_ring_param.vec_idx); 352 napi_schedule(&self->napi); 353 } else { 354 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U); 355 return IRQ_NONE; 356 } 357 358 return IRQ_HANDLED; 359 } 360 361 cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self) 362 { 363 return &self->aq_ring_param.affinity_mask; 364 } 365 366 bool aq_vec_is_valid_tc(struct aq_vec_s *self, const unsigned int tc) 367 { 368 return tc < self->rx_rings && tc < self->tx_rings; 369 } 370 371 unsigned int aq_vec_get_sw_stats(struct aq_vec_s *self, const unsigned int tc, u64 *data) 372 { 373 unsigned int count; 374 375 if (!aq_vec_is_valid_tc(self, tc)) 376 return 0; 377 378 count = aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_RX_ID], data); 379 count += aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_TX_ID], data + count); 380 381 return count; 382 } 383