1 #ifndef _NET_GRO_CELLS_H 2 #define _NET_GRO_CELLS_H 3 4 #include <linux/skbuff.h> 5 #include <linux/slab.h> 6 #include <linux/netdevice.h> 7 8 struct gro_cell { 9 struct sk_buff_head napi_skbs; 10 struct napi_struct napi; 11 } ____cacheline_aligned_in_smp; 12 13 struct gro_cells { 14 unsigned int gro_cells_mask; 15 struct gro_cell *cells; 16 }; 17 18 static inline void gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) 19 { 20 struct gro_cell *cell = gcells->cells; 21 struct net_device *dev = skb->dev; 22 23 if (!cell || skb_cloned(skb) || !(dev->features & NETIF_F_GRO)) { 24 netif_rx(skb); 25 return; 26 } 27 28 if (skb_rx_queue_recorded(skb)) 29 cell += skb_get_rx_queue(skb) & gcells->gro_cells_mask; 30 31 if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) { 32 atomic_long_inc(&dev->rx_dropped); 33 kfree_skb(skb); 34 return; 35 } 36 37 /* We run in BH context */ 38 spin_lock(&cell->napi_skbs.lock); 39 40 __skb_queue_tail(&cell->napi_skbs, skb); 41 if (skb_queue_len(&cell->napi_skbs) == 1) 42 napi_schedule(&cell->napi); 43 44 spin_unlock(&cell->napi_skbs.lock); 45 } 46 47 /* called unser BH context */ 48 static inline int gro_cell_poll(struct napi_struct *napi, int budget) 49 { 50 struct gro_cell *cell = container_of(napi, struct gro_cell, napi); 51 struct sk_buff *skb; 52 int work_done = 0; 53 54 spin_lock(&cell->napi_skbs.lock); 55 while (work_done < budget) { 56 skb = __skb_dequeue(&cell->napi_skbs); 57 if (!skb) 58 break; 59 spin_unlock(&cell->napi_skbs.lock); 60 napi_gro_receive(napi, skb); 61 work_done++; 62 spin_lock(&cell->napi_skbs.lock); 63 } 64 65 if (work_done < budget) 66 napi_complete(napi); 67 spin_unlock(&cell->napi_skbs.lock); 68 return work_done; 69 } 70 71 static inline int gro_cells_init(struct gro_cells *gcells, struct net_device *dev) 72 { 73 int i; 74 75 gcells->gro_cells_mask = roundup_pow_of_two(netif_get_num_default_rss_queues()) - 1; 76 gcells->cells = kcalloc(gcells->gro_cells_mask + 1, 77 sizeof(struct gro_cell), 78 GFP_KERNEL); 79 if (!gcells->cells) 80 return -ENOMEM; 81 82 for (i = 0; i <= gcells->gro_cells_mask; i++) { 83 struct gro_cell *cell = gcells->cells + i; 84 85 skb_queue_head_init(&cell->napi_skbs); 86 netif_napi_add(dev, &cell->napi, gro_cell_poll, 64); 87 napi_enable(&cell->napi); 88 } 89 return 0; 90 } 91 92 static inline void gro_cells_destroy(struct gro_cells *gcells) 93 { 94 struct gro_cell *cell = gcells->cells; 95 int i; 96 97 if (!cell) 98 return; 99 for (i = 0; i <= gcells->gro_cells_mask; i++,cell++) { 100 netif_napi_del(&cell->napi); 101 skb_queue_purge(&cell->napi_skbs); 102 } 103 kfree(gcells->cells); 104 gcells->cells = NULL; 105 } 106 107 #endif 108