1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2021, Microsoft Corporation. */
3 
4 #include <linux/inetdevice.h>
5 #include <linux/etherdevice.h>
6 #include <linux/mm.h>
7 #include <linux/bpf.h>
8 #include <linux/bpf_trace.h>
9 #include <net/xdp.h>
10 
11 #include <net/mana/mana.h>
12 
13 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev)
14 {
15 	u16 txq_idx = skb_get_queue_mapping(skb);
16 	struct netdev_queue *ndevtxq;
17 	int rc;
18 
19 	__skb_push(skb, ETH_HLEN);
20 
21 	ndevtxq = netdev_get_tx_queue(ndev, txq_idx);
22 	__netif_tx_lock(ndevtxq, smp_processor_id());
23 
24 	rc = mana_start_xmit(skb, ndev);
25 
26 	__netif_tx_unlock(ndevtxq);
27 
28 	if (dev_xmit_complete(rc))
29 		return;
30 
31 	dev_kfree_skb_any(skb);
32 	ndev->stats.tx_dropped++;
33 }
34 
35 static int mana_xdp_xmit_fm(struct net_device *ndev, struct xdp_frame *frame,
36 			    u16 q_idx)
37 {
38 	struct sk_buff *skb;
39 
40 	skb = xdp_build_skb_from_frame(frame, ndev);
41 	if (unlikely(!skb))
42 		return -ENOMEM;
43 
44 	skb_set_queue_mapping(skb, q_idx);
45 
46 	mana_xdp_tx(skb, ndev);
47 
48 	return 0;
49 }
50 
51 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,
52 		  u32 flags)
53 {
54 	struct mana_port_context *apc = netdev_priv(ndev);
55 	struct mana_stats_tx *tx_stats;
56 	int i, count = 0;
57 	u16 q_idx;
58 
59 	if (unlikely(!apc->port_is_up))
60 		return 0;
61 
62 	q_idx = smp_processor_id() % ndev->real_num_tx_queues;
63 
64 	for (i = 0; i < n; i++) {
65 		if (mana_xdp_xmit_fm(ndev, frames[i], q_idx))
66 			break;
67 
68 		count++;
69 	}
70 
71 	tx_stats = &apc->tx_qp[q_idx].txq.stats;
72 
73 	u64_stats_update_begin(&tx_stats->syncp);
74 	tx_stats->xdp_xmit += count;
75 	u64_stats_update_end(&tx_stats->syncp);
76 
77 	return count;
78 }
79 
80 u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,
81 		 struct xdp_buff *xdp, void *buf_va, uint pkt_len)
82 {
83 	struct mana_stats_rx *rx_stats;
84 	struct bpf_prog *prog;
85 	u32 act = XDP_PASS;
86 
87 	rcu_read_lock();
88 	prog = rcu_dereference(rxq->bpf_prog);
89 
90 	if (!prog)
91 		goto out;
92 
93 	xdp_init_buff(xdp, PAGE_SIZE, &rxq->xdp_rxq);
94 	xdp_prepare_buff(xdp, buf_va, XDP_PACKET_HEADROOM, pkt_len, false);
95 
96 	act = bpf_prog_run_xdp(prog, xdp);
97 
98 	rx_stats = &rxq->stats;
99 
100 	switch (act) {
101 	case XDP_PASS:
102 	case XDP_TX:
103 	case XDP_DROP:
104 		break;
105 
106 	case XDP_REDIRECT:
107 		rxq->xdp_rc = xdp_do_redirect(ndev, xdp, prog);
108 		if (!rxq->xdp_rc) {
109 			rxq->xdp_flush = true;
110 
111 			u64_stats_update_begin(&rx_stats->syncp);
112 			rx_stats->packets++;
113 			rx_stats->bytes += pkt_len;
114 			rx_stats->xdp_redirect++;
115 			u64_stats_update_end(&rx_stats->syncp);
116 
117 			break;
118 		}
119 
120 		fallthrough;
121 
122 	case XDP_ABORTED:
123 		trace_xdp_exception(ndev, prog, act);
124 		break;
125 
126 	default:
127 		bpf_warn_invalid_xdp_action(ndev, prog, act);
128 	}
129 
130 out:
131 	rcu_read_unlock();
132 
133 	return act;
134 }
135 
136 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc)
137 {
138 	ASSERT_RTNL();
139 
140 	return apc->bpf_prog;
141 }
142 
143 static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc)
144 {
145 	return rtnl_dereference(apc->rxqs[0]->bpf_prog);
146 }
147 
148 /* Set xdp program on channels */
149 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog)
150 {
151 	struct bpf_prog *old_prog = mana_chn_xdp_get(apc);
152 	unsigned int num_queues = apc->num_queues;
153 	int i;
154 
155 	ASSERT_RTNL();
156 
157 	if (old_prog == prog)
158 		return;
159 
160 	if (prog)
161 		bpf_prog_add(prog, num_queues);
162 
163 	for (i = 0; i < num_queues; i++)
164 		rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog);
165 
166 	if (old_prog)
167 		for (i = 0; i < num_queues; i++)
168 			bpf_prog_put(old_prog);
169 }
170 
171 static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
172 			struct netlink_ext_ack *extack)
173 {
174 	struct mana_port_context *apc = netdev_priv(ndev);
175 	struct bpf_prog *old_prog;
176 	struct gdma_context *gc;
177 
178 	gc = apc->ac->gdma_dev->gdma_context;
179 
180 	old_prog = mana_xdp_get(apc);
181 
182 	if (!old_prog && !prog)
183 		return 0;
184 
185 	if (prog && ndev->mtu > MANA_XDP_MTU_MAX) {
186 		netdev_err(ndev, "XDP: mtu:%u too large, mtu_max:%lu\n",
187 			   ndev->mtu, MANA_XDP_MTU_MAX);
188 		NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large");
189 
190 		return -EOPNOTSUPP;
191 	}
192 
193 	/* One refcnt of the prog is hold by the caller already, so
194 	 * don't increase refcnt for this one.
195 	 */
196 	apc->bpf_prog = prog;
197 
198 	if (old_prog)
199 		bpf_prog_put(old_prog);
200 
201 	if (apc->port_is_up)
202 		mana_chn_setxdp(apc, prog);
203 
204 	if (prog)
205 		ndev->max_mtu = MANA_XDP_MTU_MAX;
206 	else
207 		ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
208 
209 	return 0;
210 }
211 
212 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
213 {
214 	struct netlink_ext_ack *extack = bpf->extack;
215 	int ret;
216 
217 	switch (bpf->command) {
218 	case XDP_SETUP_PROG:
219 		return mana_xdp_set(ndev, bpf->prog, extack);
220 
221 	default:
222 		return -EOPNOTSUPP;
223 	}
224 
225 	return ret;
226 }
227