virtio_net.c (18117a842ab029df139f776bf31eebff6d0e0730) virtio_net.c (fab89bafa95b6f333b0e2dca0ae07a0b3600e954)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* A network driver using virtio.
3 *
4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5 */
6//#define DEBUG
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>

--- 1076 unchanged lines hidden (view full) ---

1085 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
1086 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
1087 struct page *page = virt_to_head_page(buf);
1088 int offset = buf - page_address(page);
1089 struct sk_buff *head_skb, *curr_skb;
1090 struct bpf_prog *xdp_prog;
1091 unsigned int truesize = mergeable_ctx_to_truesize(ctx);
1092 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* A network driver using virtio.
3 *
4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
5 */
6//#define DEBUG
7#include <linux/netdevice.h>
8#include <linux/etherdevice.h>

--- 1076 unchanged lines hidden (view full) ---

1085 struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
1086 u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
1087 struct page *page = virt_to_head_page(buf);
1088 int offset = buf - page_address(page);
1089 struct sk_buff *head_skb, *curr_skb;
1090 struct bpf_prog *xdp_prog;
1091 unsigned int truesize = mergeable_ctx_to_truesize(ctx);
1092 unsigned int headroom = mergeable_ctx_to_headroom(ctx);
1093 unsigned int metasize = 0;
1094 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1095 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1096 unsigned int frame_sz, xdp_room;
1097 int err;
1098
1099 head_skb = NULL;
1100 stats->bytes += len - vi->hdr_len;
1101

--- 79 unchanged lines hidden (view full) ---

1181 if (unlikely(err))
1182 goto err_xdp_frags;
1183
1184 act = bpf_prog_run_xdp(xdp_prog, &xdp);
1185 stats->xdp_packets++;
1186
1187 switch (act) {
1188 case XDP_PASS:
1093 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
1094 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
1095 unsigned int frame_sz, xdp_room;
1096 int err;
1097
1098 head_skb = NULL;
1099 stats->bytes += len - vi->hdr_len;
1100

--- 79 unchanged lines hidden (view full) ---

1180 if (unlikely(err))
1181 goto err_xdp_frags;
1182
1183 act = bpf_prog_run_xdp(xdp_prog, &xdp);
1184 stats->xdp_packets++;
1185
1186 switch (act) {
1187 case XDP_PASS:
1189 metasize = xdp.data - xdp.data_meta;
1190
1191 /* recalculate offset to account for any header
1192 * adjustments and minus the metasize to copy the
1193 * metadata in page_to_skb(). Note other cases do not
1194 * build an skb and avoid using offset
1195 */
1196 offset = xdp.data - page_address(xdp_page) -
1197 vi->hdr_len - metasize;
1198
1199 /* recalculate len if xdp.data, xdp.data_end or
1200 * xdp.data_meta were adjusted
1201 */
1202 len = xdp.data_end - xdp.data + vi->hdr_len + metasize;
1203
1204 /* recalculate headroom if xdp.data or xdp_data_meta
1205 * were adjusted, note that offset should always point
1206 * to the start of the reserved bytes for virtio_net
1207 * header which are followed by xdp.data, that means
1208 * that offset is equal to the headroom (when buf is
1209 * starting at the beginning of the page, otherwise
1210 * there is a base offset inside the page) but it's used
1211 * with a different starting point (buf start) than
1212 * xdp.data (buf start + vnet hdr size). If xdp.data or
1213 * data_meta were adjusted by the xdp prog then the
1214 * headroom size has changed and so has the offset, we
1215 * can use data_hard_start, which points at buf start +
1216 * vnet hdr size, to calculate the new headroom and use
1217 * it later to compute buf start in page_to_skb()
1218 */
1219 headroom = xdp.data - xdp.data_hard_start - metasize;
1220
1221 /* We can only create skb based on xdp_page. */
1222 if (unlikely(xdp_page != page)) {
1223 rcu_read_unlock();
1188 if (unlikely(xdp_page != page))
1224 put_page(page);
1189 put_page(page);
1225 head_skb = page_to_skb(vi, rq, xdp_page, offset,
1226 len, PAGE_SIZE);
1227 return head_skb;
1228 }
1229 break;
1190 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
1191 rcu_read_unlock();
1192 return head_skb;
1230 case XDP_TX:
1231 stats->xdp_tx++;
1232 xdpf = xdp_convert_buff_to_frame(&xdp);
1233 if (unlikely(!xdpf)) {
1193 case XDP_TX:
1194 stats->xdp_tx++;
1195 xdpf = xdp_convert_buff_to_frame(&xdp);
1196 if (unlikely(!xdpf)) {
1234 if (unlikely(xdp_page != page))
1235 put_page(xdp_page);
1236 goto err_xdp;
1197 netdev_dbg(dev, "convert buff to frame failed for xdp\n");
1198 goto err_xdp_frags;
1237 }
1238 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
1239 if (unlikely(!err)) {
1240 xdp_return_frame_rx_napi(xdpf);
1241 } else if (unlikely(err < 0)) {
1242 trace_xdp_exception(vi->dev, xdp_prog, act);
1199 }
1200 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
1201 if (unlikely(!err)) {
1202 xdp_return_frame_rx_napi(xdpf);
1203 } else if (unlikely(err < 0)) {
1204 trace_xdp_exception(vi->dev, xdp_prog, act);
1243 if (unlikely(xdp_page != page))
1244 put_page(xdp_page);
1245 goto err_xdp;
1205 goto err_xdp_frags;
1246 }
1247 *xdp_xmit |= VIRTIO_XDP_TX;
1248 if (unlikely(xdp_page != page))
1249 put_page(page);
1250 rcu_read_unlock();
1251 goto xdp_xmit;
1252 case XDP_REDIRECT:
1253 stats->xdp_redirects++;
1254 err = xdp_do_redirect(dev, &xdp, xdp_prog);
1206 }
1207 *xdp_xmit |= VIRTIO_XDP_TX;
1208 if (unlikely(xdp_page != page))
1209 put_page(page);
1210 rcu_read_unlock();
1211 goto xdp_xmit;
1212 case XDP_REDIRECT:
1213 stats->xdp_redirects++;
1214 err = xdp_do_redirect(dev, &xdp, xdp_prog);
1255 if (err) {
1256 if (unlikely(xdp_page != page))
1257 put_page(xdp_page);
1258 goto err_xdp;
1259 }
1215 if (err)
1216 goto err_xdp_frags;
1260 *xdp_xmit |= VIRTIO_XDP_REDIR;
1261 if (unlikely(xdp_page != page))
1262 put_page(page);
1263 rcu_read_unlock();
1264 goto xdp_xmit;
1265 default:
1266 bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
1267 fallthrough;
1268 case XDP_ABORTED:
1269 trace_xdp_exception(vi->dev, xdp_prog, act);
1270 fallthrough;
1271 case XDP_DROP:
1217 *xdp_xmit |= VIRTIO_XDP_REDIR;
1218 if (unlikely(xdp_page != page))
1219 put_page(page);
1220 rcu_read_unlock();
1221 goto xdp_xmit;
1222 default:
1223 bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
1224 fallthrough;
1225 case XDP_ABORTED:
1226 trace_xdp_exception(vi->dev, xdp_prog, act);
1227 fallthrough;
1228 case XDP_DROP:
1272 if (unlikely(xdp_page != page))
1273 __free_pages(xdp_page, 0);
1274 goto err_xdp;
1229 goto err_xdp_frags;
1275 }
1276err_xdp_frags:
1277 if (unlikely(xdp_page != page))
1278 __free_pages(xdp_page, 0);
1279
1280 if (xdp_buff_has_frags(&xdp)) {
1281 shinfo = xdp_get_shared_info_from_buff(&xdp);
1282 for (i = 0; i < shinfo->nr_frags; i++) {

--- 3015 unchanged lines hidden --- 		1230 }
1231err_xdp_frags:
1232 if (unlikely(xdp_page != page))
1233 __free_pages(xdp_page, 0);
1234
1235 if (xdp_buff_has_frags(&xdp)) {
1236 shinfo = xdp_get_shared_info_from_buff(&xdp);
1237 for (i = 0; i < shinfo->nr_frags; i++) {

--- 3015 unchanged lines hidden ---