xref: /openbmc/linux/net/openvswitch/datapath.c (revision 6be50c89)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2007-2014 Nicira, Inc.
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/init.h>
9 #include <linux/module.h>
10 #include <linux/if_arp.h>
11 #include <linux/if_vlan.h>
12 #include <linux/in.h>
13 #include <linux/ip.h>
14 #include <linux/jhash.h>
15 #include <linux/delay.h>
16 #include <linux/time.h>
17 #include <linux/etherdevice.h>
18 #include <linux/genetlink.h>
19 #include <linux/kernel.h>
20 #include <linux/kthread.h>
21 #include <linux/mutex.h>
22 #include <linux/percpu.h>
23 #include <linux/rcupdate.h>
24 #include <linux/tcp.h>
25 #include <linux/udp.h>
26 #include <linux/ethtool.h>
27 #include <linux/wait.h>
28 #include <asm/div64.h>
29 #include <linux/highmem.h>
30 #include <linux/netfilter_bridge.h>
31 #include <linux/netfilter_ipv4.h>
32 #include <linux/inetdevice.h>
33 #include <linux/list.h>
34 #include <linux/openvswitch.h>
35 #include <linux/rculist.h>
36 #include <linux/dmi.h>
37 #include <net/genetlink.h>
38 #include <net/gso.h>
39 #include <net/net_namespace.h>
40 #include <net/netns/generic.h>
41 #include <net/pkt_cls.h>
42 
43 #include "datapath.h"
44 #include "drop.h"
45 #include "flow.h"
46 #include "flow_table.h"
47 #include "flow_netlink.h"
48 #include "meter.h"
49 #include "openvswitch_trace.h"
50 #include "vport-internal_dev.h"
51 #include "vport-netdev.h"
52 
53 unsigned int ovs_net_id __read_mostly;
54 
55 static struct genl_family dp_packet_genl_family;
56 static struct genl_family dp_flow_genl_family;
57 static struct genl_family dp_datapath_genl_family;
58 
59 static const struct nla_policy flow_policy[];
60 
61 static const struct genl_multicast_group ovs_dp_flow_multicast_group = {
62 	.name = OVS_FLOW_MCGROUP,
63 };
64 
65 static const struct genl_multicast_group ovs_dp_datapath_multicast_group = {
66 	.name = OVS_DATAPATH_MCGROUP,
67 };
68 
69 static const struct genl_multicast_group ovs_dp_vport_multicast_group = {
70 	.name = OVS_VPORT_MCGROUP,
71 };
72 
73 /* Check if need to build a reply message.
74  * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
ovs_must_notify(struct genl_family * family,struct genl_info * info,unsigned int group)75 static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
76 			    unsigned int group)
77 {
78 	return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
79 	       genl_has_listeners(family, genl_info_net(info), group);
80 }
81 
ovs_notify(struct genl_family * family,struct sk_buff * skb,struct genl_info * info)82 static void ovs_notify(struct genl_family *family,
83 		       struct sk_buff *skb, struct genl_info *info)
84 {
85 	genl_notify(family, skb, info, 0, GFP_KERNEL);
86 }
87 
88 /**
89  * DOC: Locking:
90  *
91  * All writes e.g. Writes to device state (add/remove datapath, port, set
92  * operations on vports, etc.), Writes to other state (flow table
93  * modifications, set miscellaneous datapath parameters, etc.) are protected
94  * by ovs_lock.
95  *
96  * Reads are protected by RCU.
97  *
98  * There are a few special cases (mostly stats) that have their own
99  * synchronization but they nest under all of above and don't interact with
100  * each other.
101  *
102  * The RTNL lock nests inside ovs_mutex.
103  */
104 
105 static DEFINE_MUTEX(ovs_mutex);
106 
ovs_lock(void)107 void ovs_lock(void)
108 {
109 	mutex_lock(&ovs_mutex);
110 }
111 
ovs_unlock(void)112 void ovs_unlock(void)
113 {
114 	mutex_unlock(&ovs_mutex);
115 }
116 
117 #ifdef CONFIG_LOCKDEP
lockdep_ovsl_is_held(void)118 int lockdep_ovsl_is_held(void)
119 {
120 	if (debug_locks)
121 		return lockdep_is_held(&ovs_mutex);
122 	else
123 		return 1;
124 }
125 #endif
126 
127 static struct vport *new_vport(const struct vport_parms *);
128 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
129 			     const struct sw_flow_key *,
130 			     const struct dp_upcall_info *,
131 			     uint32_t cutlen);
132 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
133 				  const struct sw_flow_key *,
134 				  const struct dp_upcall_info *,
135 				  uint32_t cutlen);
136 
137 static void ovs_dp_masks_rebalance(struct work_struct *work);
138 
139 static int ovs_dp_set_upcall_portids(struct datapath *, const struct nlattr *);
140 
141 /* Must be called with rcu_read_lock or ovs_mutex. */
ovs_dp_name(const struct datapath * dp)142 const char *ovs_dp_name(const struct datapath *dp)
143 {
144 	struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
145 	return ovs_vport_name(vport);
146 }
147 
get_dpifindex(const struct datapath * dp)148 static int get_dpifindex(const struct datapath *dp)
149 {
150 	struct vport *local;
151 	int ifindex;
152 
153 	rcu_read_lock();
154 
155 	local = ovs_vport_rcu(dp, OVSP_LOCAL);
156 	if (local)
157 		ifindex = local->dev->ifindex;
158 	else
159 		ifindex = 0;
160 
161 	rcu_read_unlock();
162 
163 	return ifindex;
164 }
165 
destroy_dp_rcu(struct rcu_head * rcu)166 static void destroy_dp_rcu(struct rcu_head *rcu)
167 {
168 	struct datapath *dp = container_of(rcu, struct datapath, rcu);
169 
170 	ovs_flow_tbl_destroy(&dp->table);
171 	free_percpu(dp->stats_percpu);
172 	kfree(dp->ports);
173 	ovs_meters_exit(dp);
174 	kfree(rcu_dereference_raw(dp->upcall_portids));
175 	kfree(dp);
176 }
177 
vport_hash_bucket(const struct datapath * dp,u16 port_no)178 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
179 					    u16 port_no)
180 {
181 	return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
182 }
183 
184 /* Called with ovs_mutex or RCU read lock. */
ovs_lookup_vport(const struct datapath * dp,u16 port_no)185 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
186 {
187 	struct vport *vport;
188 	struct hlist_head *head;
189 
190 	head = vport_hash_bucket(dp, port_no);
191 	hlist_for_each_entry_rcu(vport, head, dp_hash_node,
192 				 lockdep_ovsl_is_held()) {
193 		if (vport->port_no == port_no)
194 			return vport;
195 	}
196 	return NULL;
197 }
198 
199 /* Called with ovs_mutex. */
new_vport(const struct vport_parms * parms)200 static struct vport *new_vport(const struct vport_parms *parms)
201 {
202 	struct vport *vport;
203 
204 	vport = ovs_vport_add(parms);
205 	if (!IS_ERR(vport)) {
206 		struct datapath *dp = parms->dp;
207 		struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
208 
209 		hlist_add_head_rcu(&vport->dp_hash_node, head);
210 	}
211 	return vport;
212 }
213 
ovs_vport_update_upcall_stats(struct sk_buff * skb,const struct dp_upcall_info * upcall_info,bool upcall_result)214 static void ovs_vport_update_upcall_stats(struct sk_buff *skb,
215 					  const struct dp_upcall_info *upcall_info,
216 					  bool upcall_result)
217 {
218 	struct vport *p = OVS_CB(skb)->input_vport;
219 	struct vport_upcall_stats_percpu *stats;
220 
221 	if (upcall_info->cmd != OVS_PACKET_CMD_MISS &&
222 	    upcall_info->cmd != OVS_PACKET_CMD_ACTION)
223 		return;
224 
225 	stats = this_cpu_ptr(p->upcall_stats);
226 	u64_stats_update_begin(&stats->syncp);
227 	if (upcall_result)
228 		u64_stats_inc(&stats->n_success);
229 	else
230 		u64_stats_inc(&stats->n_fail);
231 	u64_stats_update_end(&stats->syncp);
232 }
233 
ovs_dp_detach_port(struct vport * p)234 void ovs_dp_detach_port(struct vport *p)
235 {
236 	ASSERT_OVSL();
237 
238 	/* First drop references to device. */
239 	hlist_del_rcu(&p->dp_hash_node);
240 
241 	/* Then destroy it. */
242 	ovs_vport_del(p);
243 }
244 
245 /* Must be called with rcu_read_lock. */
ovs_dp_process_packet(struct sk_buff * skb,struct sw_flow_key * key)246 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
247 {
248 	const struct vport *p = OVS_CB(skb)->input_vport;
249 	struct datapath *dp = p->dp;
250 	struct sw_flow *flow;
251 	struct sw_flow_actions *sf_acts;
252 	struct dp_stats_percpu *stats;
253 	u64 *stats_counter;
254 	u32 n_mask_hit;
255 	u32 n_cache_hit;
256 	int error;
257 
258 	stats = this_cpu_ptr(dp->stats_percpu);
259 
260 	/* Look up flow. */
261 	flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
262 					 &n_mask_hit, &n_cache_hit);
263 	if (unlikely(!flow)) {
264 		struct dp_upcall_info upcall;
265 
266 		memset(&upcall, 0, sizeof(upcall));
267 		upcall.cmd = OVS_PACKET_CMD_MISS;
268 
269 		if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU)
270 			upcall.portid =
271 			    ovs_dp_get_upcall_portid(dp, smp_processor_id());
272 		else
273 			upcall.portid = ovs_vport_find_upcall_portid(p, skb);
274 
275 		upcall.mru = OVS_CB(skb)->mru;
276 		error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
277 		switch (error) {
278 		case 0:
279 		case -EAGAIN:
280 		case -ERESTARTSYS:
281 		case -EINTR:
282 			consume_skb(skb);
283 			break;
284 		default:
285 			kfree_skb(skb);
286 			break;
287 		}
288 		stats_counter = &stats->n_missed;
289 		goto out;
290 	}
291 
292 	ovs_flow_stats_update(flow, key->tp.flags, skb);
293 	sf_acts = rcu_dereference(flow->sf_acts);
294 	error = ovs_execute_actions(dp, skb, sf_acts, key);
295 	if (unlikely(error))
296 		net_dbg_ratelimited("ovs: action execution error on datapath %s: %d\n",
297 				    ovs_dp_name(dp), error);
298 
299 	stats_counter = &stats->n_hit;
300 
301 out:
302 	/* Update datapath statistics. */
303 	u64_stats_update_begin(&stats->syncp);
304 	(*stats_counter)++;
305 	stats->n_mask_hit += n_mask_hit;
306 	stats->n_cache_hit += n_cache_hit;
307 	u64_stats_update_end(&stats->syncp);
308 }
309 
ovs_dp_upcall(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)310 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
311 		  const struct sw_flow_key *key,
312 		  const struct dp_upcall_info *upcall_info,
313 		  uint32_t cutlen)
314 {
315 	struct dp_stats_percpu *stats;
316 	int err;
317 
318 	if (trace_ovs_dp_upcall_enabled())
319 		trace_ovs_dp_upcall(dp, skb, key, upcall_info);
320 
321 	if (upcall_info->portid == 0) {
322 		err = -ENOTCONN;
323 		goto err;
324 	}
325 
326 	if (!skb_is_gso(skb))
327 		err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
328 	else
329 		err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
330 
331 	ovs_vport_update_upcall_stats(skb, upcall_info, !err);
332 	if (err)
333 		goto err;
334 
335 	return 0;
336 
337 err:
338 	stats = this_cpu_ptr(dp->stats_percpu);
339 
340 	u64_stats_update_begin(&stats->syncp);
341 	stats->n_lost++;
342 	u64_stats_update_end(&stats->syncp);
343 
344 	return err;
345 }
346 
queue_gso_packets(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)347 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
348 			     const struct sw_flow_key *key,
349 			     const struct dp_upcall_info *upcall_info,
350 			     uint32_t cutlen)
351 {
352 	unsigned int gso_type = skb_shinfo(skb)->gso_type;
353 	struct sw_flow_key later_key;
354 	struct sk_buff *segs, *nskb;
355 	int err;
356 
357 	BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET);
358 	segs = __skb_gso_segment(skb, NETIF_F_SG, false);
359 	if (IS_ERR(segs))
360 		return PTR_ERR(segs);
361 	if (segs == NULL)
362 		return -EINVAL;
363 
364 	if (gso_type & SKB_GSO_UDP) {
365 		/* The initial flow key extracted by ovs_flow_key_extract()
366 		 * in this case is for a first fragment, so we need to
367 		 * properly mark later fragments.
368 		 */
369 		later_key = *key;
370 		later_key.ip.frag = OVS_FRAG_TYPE_LATER;
371 	}
372 
373 	/* Queue all of the segments. */
374 	skb_list_walk_safe(segs, skb, nskb) {
375 		if (gso_type & SKB_GSO_UDP && skb != segs)
376 			key = &later_key;
377 
378 		err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
379 		if (err)
380 			break;
381 
382 	}
383 
384 	/* Free all of the segments. */
385 	skb_list_walk_safe(segs, skb, nskb) {
386 		if (err)
387 			kfree_skb(skb);
388 		else
389 			consume_skb(skb);
390 	}
391 	return err;
392 }
393 
upcall_msg_size(const struct dp_upcall_info * upcall_info,unsigned int hdrlen,int actions_attrlen)394 static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
395 			      unsigned int hdrlen, int actions_attrlen)
396 {
397 	size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
398 		+ nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
399 		+ nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
400 		+ nla_total_size(sizeof(unsigned int)) /* OVS_PACKET_ATTR_LEN */
401 		+ nla_total_size(sizeof(u64)); /* OVS_PACKET_ATTR_HASH */
402 
403 	/* OVS_PACKET_ATTR_USERDATA */
404 	if (upcall_info->userdata)
405 		size += NLA_ALIGN(upcall_info->userdata->nla_len);
406 
407 	/* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
408 	if (upcall_info->egress_tun_info)
409 		size += nla_total_size(ovs_tun_key_attr_size());
410 
411 	/* OVS_PACKET_ATTR_ACTIONS */
412 	if (upcall_info->actions_len)
413 		size += nla_total_size(actions_attrlen);
414 
415 	/* OVS_PACKET_ATTR_MRU */
416 	if (upcall_info->mru)
417 		size += nla_total_size(sizeof(upcall_info->mru));
418 
419 	return size;
420 }
421 
pad_packet(struct datapath * dp,struct sk_buff * skb)422 static void pad_packet(struct datapath *dp, struct sk_buff *skb)
423 {
424 	if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
425 		size_t plen = NLA_ALIGN(skb->len) - skb->len;
426 
427 		if (plen > 0)
428 			skb_put_zero(skb, plen);
429 	}
430 }
431 
queue_userspace_packet(struct datapath * dp,struct sk_buff * skb,const struct sw_flow_key * key,const struct dp_upcall_info * upcall_info,uint32_t cutlen)432 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
433 				  const struct sw_flow_key *key,
434 				  const struct dp_upcall_info *upcall_info,
435 				  uint32_t cutlen)
436 {
437 	struct ovs_header *upcall;
438 	struct sk_buff *nskb = NULL;
439 	struct sk_buff *user_skb = NULL; /* to be queued to userspace */
440 	struct nlattr *nla;
441 	size_t len;
442 	unsigned int hlen;
443 	int err, dp_ifindex;
444 	u64 hash;
445 
446 	dp_ifindex = get_dpifindex(dp);
447 	if (!dp_ifindex)
448 		return -ENODEV;
449 
450 	if (skb_vlan_tag_present(skb)) {
451 		nskb = skb_clone(skb, GFP_ATOMIC);
452 		if (!nskb)
453 			return -ENOMEM;
454 
455 		nskb = __vlan_hwaccel_push_inside(nskb);
456 		if (!nskb)
457 			return -ENOMEM;
458 
459 		skb = nskb;
460 	}
461 
462 	if (nla_attr_size(skb->len) > USHRT_MAX) {
463 		err = -EFBIG;
464 		goto out;
465 	}
466 
467 	/* Complete checksum if needed */
468 	if (skb->ip_summed == CHECKSUM_PARTIAL &&
469 	    (err = skb_csum_hwoffload_help(skb, 0)))
470 		goto out;
471 
472 	/* Older versions of OVS user space enforce alignment of the last
473 	 * Netlink attribute to NLA_ALIGNTO which would require extensive
474 	 * padding logic. Only perform zerocopy if padding is not required.
475 	 */
476 	if (dp->user_features & OVS_DP_F_UNALIGNED)
477 		hlen = skb_zerocopy_headlen(skb);
478 	else
479 		hlen = skb->len;
480 
481 	len = upcall_msg_size(upcall_info, hlen - cutlen,
482 			      OVS_CB(skb)->acts_origlen);
483 	user_skb = genlmsg_new(len, GFP_ATOMIC);
484 	if (!user_skb) {
485 		err = -ENOMEM;
486 		goto out;
487 	}
488 
489 	upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
490 			     0, upcall_info->cmd);
491 	if (!upcall) {
492 		err = -EINVAL;
493 		goto out;
494 	}
495 	upcall->dp_ifindex = dp_ifindex;
496 
497 	err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
498 	if (err)
499 		goto out;
500 
501 	if (upcall_info->userdata)
502 		__nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
503 			  nla_len(upcall_info->userdata),
504 			  nla_data(upcall_info->userdata));
505 
506 	if (upcall_info->egress_tun_info) {
507 		nla = nla_nest_start_noflag(user_skb,
508 					    OVS_PACKET_ATTR_EGRESS_TUN_KEY);
509 		if (!nla) {
510 			err = -EMSGSIZE;
511 			goto out;
512 		}
513 		err = ovs_nla_put_tunnel_info(user_skb,
514 					      upcall_info->egress_tun_info);
515 		if (err)
516 			goto out;
517 
518 		nla_nest_end(user_skb, nla);
519 	}
520 
521 	if (upcall_info->actions_len) {
522 		nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS);
523 		if (!nla) {
524 			err = -EMSGSIZE;
525 			goto out;
526 		}
527 		err = ovs_nla_put_actions(upcall_info->actions,
528 					  upcall_info->actions_len,
529 					  user_skb);
530 		if (!err)
531 			nla_nest_end(user_skb, nla);
532 		else
533 			nla_nest_cancel(user_skb, nla);
534 	}
535 
536 	/* Add OVS_PACKET_ATTR_MRU */
537 	if (upcall_info->mru &&
538 	    nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU, upcall_info->mru)) {
539 		err = -ENOBUFS;
540 		goto out;
541 	}
542 
543 	/* Add OVS_PACKET_ATTR_LEN when packet is truncated */
544 	if (cutlen > 0 &&
545 	    nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) {
546 		err = -ENOBUFS;
547 		goto out;
548 	}
549 
550 	/* Add OVS_PACKET_ATTR_HASH */
551 	hash = skb_get_hash_raw(skb);
552 	if (skb->sw_hash)
553 		hash |= OVS_PACKET_HASH_SW_BIT;
554 
555 	if (skb->l4_hash)
556 		hash |= OVS_PACKET_HASH_L4_BIT;
557 
558 	if (nla_put(user_skb, OVS_PACKET_ATTR_HASH, sizeof (u64), &hash)) {
559 		err = -ENOBUFS;
560 		goto out;
561 	}
562 
563 	/* Only reserve room for attribute header, packet data is added
564 	 * in skb_zerocopy() */
565 	if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
566 		err = -ENOBUFS;
567 		goto out;
568 	}
569 	nla->nla_len = nla_attr_size(skb->len - cutlen);
570 
571 	err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen);
572 	if (err)
573 		goto out;
574 
575 	/* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
576 	pad_packet(dp, user_skb);
577 
578 	((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
579 
580 	err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
581 	user_skb = NULL;
582 out:
583 	if (err)
584 		skb_tx_error(skb);
585 	consume_skb(user_skb);
586 	consume_skb(nskb);
587 
588 	return err;
589 }
590 
ovs_packet_cmd_execute(struct sk_buff * skb,struct genl_info * info)591 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
592 {
593 	struct ovs_header *ovs_header = genl_info_userhdr(info);
594 	struct net *net = sock_net(skb->sk);
595 	struct nlattr **a = info->attrs;
596 	struct sw_flow_actions *acts;
597 	struct sk_buff *packet;
598 	struct sw_flow *flow;
599 	struct sw_flow_actions *sf_acts;
600 	struct datapath *dp;
601 	struct vport *input_vport;
602 	u16 mru = 0;
603 	u64 hash;
604 	int len;
605 	int err;
606 	bool log = !a[OVS_PACKET_ATTR_PROBE];
607 
608 	err = -EINVAL;
609 	if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
610 	    !a[OVS_PACKET_ATTR_ACTIONS])
611 		goto err;
612 
613 	len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
614 	packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
615 	err = -ENOMEM;
616 	if (!packet)
617 		goto err;
618 	skb_reserve(packet, NET_IP_ALIGN);
619 
620 	nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
621 
622 	/* Set packet's mru */
623 	if (a[OVS_PACKET_ATTR_MRU]) {
624 		mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
625 		packet->ignore_df = 1;
626 	}
627 	OVS_CB(packet)->mru = mru;
628 
629 	if (a[OVS_PACKET_ATTR_HASH]) {
630 		hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]);
631 
632 		__skb_set_hash(packet, hash & 0xFFFFFFFFULL,
633 			       !!(hash & OVS_PACKET_HASH_SW_BIT),
634 			       !!(hash & OVS_PACKET_HASH_L4_BIT));
635 	}
636 
637 	/* Build an sw_flow for sending this packet. */
638 	flow = ovs_flow_alloc();
639 	err = PTR_ERR(flow);
640 	if (IS_ERR(flow))
641 		goto err_kfree_skb;
642 
643 	err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
644 					     packet, &flow->key, log);
645 	if (err)
646 		goto err_flow_free;
647 
648 	err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
649 				   &flow->key, &acts, log);
650 	if (err)
651 		goto err_flow_free;
652 
653 	rcu_assign_pointer(flow->sf_acts, acts);
654 	packet->priority = flow->key.phy.priority;
655 	packet->mark = flow->key.phy.skb_mark;
656 
657 	rcu_read_lock();
658 	dp = get_dp_rcu(net, ovs_header->dp_ifindex);
659 	err = -ENODEV;
660 	if (!dp)
661 		goto err_unlock;
662 
663 	input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
664 	if (!input_vport)
665 		input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
666 
667 	if (!input_vport)
668 		goto err_unlock;
669 
670 	packet->dev = input_vport->dev;
671 	OVS_CB(packet)->input_vport = input_vport;
672 	sf_acts = rcu_dereference(flow->sf_acts);
673 
674 	local_bh_disable();
675 	err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
676 	local_bh_enable();
677 	rcu_read_unlock();
678 
679 	ovs_flow_free(flow, false);
680 	return err;
681 
682 err_unlock:
683 	rcu_read_unlock();
684 err_flow_free:
685 	ovs_flow_free(flow, false);
686 err_kfree_skb:
687 	kfree_skb(packet);
688 err:
689 	return err;
690 }
691 
692 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
693 	[OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
694 	[OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
695 	[OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
696 	[OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
697 	[OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
698 	[OVS_PACKET_ATTR_HASH] = { .type = NLA_U64 },
699 };
700 
701 static const struct genl_small_ops dp_packet_genl_ops[] = {
702 	{ .cmd = OVS_PACKET_CMD_EXECUTE,
703 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
704 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
705 	  .doit = ovs_packet_cmd_execute
706 	}
707 };
708 
709 static struct genl_family dp_packet_genl_family __ro_after_init = {
710 	.hdrsize = sizeof(struct ovs_header),
711 	.name = OVS_PACKET_FAMILY,
712 	.version = OVS_PACKET_VERSION,
713 	.maxattr = OVS_PACKET_ATTR_MAX,
714 	.policy = packet_policy,
715 	.netnsok = true,
716 	.parallel_ops = true,
717 	.small_ops = dp_packet_genl_ops,
718 	.n_small_ops = ARRAY_SIZE(dp_packet_genl_ops),
719 	.resv_start_op = OVS_PACKET_CMD_EXECUTE + 1,
720 	.module = THIS_MODULE,
721 };
722 
get_dp_stats(const struct datapath * dp,struct ovs_dp_stats * stats,struct ovs_dp_megaflow_stats * mega_stats)723 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
724 			 struct ovs_dp_megaflow_stats *mega_stats)
725 {
726 	int i;
727 
728 	memset(mega_stats, 0, sizeof(*mega_stats));
729 
730 	stats->n_flows = ovs_flow_tbl_count(&dp->table);
731 	mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
732 
733 	stats->n_hit = stats->n_missed = stats->n_lost = 0;
734 
735 	for_each_possible_cpu(i) {
736 		const struct dp_stats_percpu *percpu_stats;
737 		struct dp_stats_percpu local_stats;
738 		unsigned int start;
739 
740 		percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
741 
742 		do {
743 			start = u64_stats_fetch_begin(&percpu_stats->syncp);
744 			local_stats = *percpu_stats;
745 		} while (u64_stats_fetch_retry(&percpu_stats->syncp, start));
746 
747 		stats->n_hit += local_stats.n_hit;
748 		stats->n_missed += local_stats.n_missed;
749 		stats->n_lost += local_stats.n_lost;
750 		mega_stats->n_mask_hit += local_stats.n_mask_hit;
751 		mega_stats->n_cache_hit += local_stats.n_cache_hit;
752 	}
753 }
754 
should_fill_key(const struct sw_flow_id * sfid,uint32_t ufid_flags)755 static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
756 {
757 	return ovs_identifier_is_ufid(sfid) &&
758 	       !(ufid_flags & OVS_UFID_F_OMIT_KEY);
759 }
760 
should_fill_mask(uint32_t ufid_flags)761 static bool should_fill_mask(uint32_t ufid_flags)
762 {
763 	return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
764 }
765 
should_fill_actions(uint32_t ufid_flags)766 static bool should_fill_actions(uint32_t ufid_flags)
767 {
768 	return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
769 }
770 
ovs_flow_cmd_msg_size(const struct sw_flow_actions * acts,const struct sw_flow_id * sfid,uint32_t ufid_flags)771 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
772 				    const struct sw_flow_id *sfid,
773 				    uint32_t ufid_flags)
774 {
775 	size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
776 
777 	/* OVS_FLOW_ATTR_UFID, or unmasked flow key as fallback
778 	 * see ovs_nla_put_identifier()
779 	 */
780 	if (sfid && ovs_identifier_is_ufid(sfid))
781 		len += nla_total_size(sfid->ufid_len);
782 	else
783 		len += nla_total_size(ovs_key_attr_size());
784 
785 	/* OVS_FLOW_ATTR_KEY */
786 	if (!sfid || should_fill_key(sfid, ufid_flags))
787 		len += nla_total_size(ovs_key_attr_size());
788 
789 	/* OVS_FLOW_ATTR_MASK */
790 	if (should_fill_mask(ufid_flags))
791 		len += nla_total_size(ovs_key_attr_size());
792 
793 	/* OVS_FLOW_ATTR_ACTIONS */
794 	if (should_fill_actions(ufid_flags))
795 		len += nla_total_size(acts->orig_len);
796 
797 	return len
798 		+ nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
799 		+ nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
800 		+ nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
801 }
802 
803 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_stats(const struct sw_flow * flow,struct sk_buff * skb)804 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
805 				   struct sk_buff *skb)
806 {
807 	struct ovs_flow_stats stats;
808 	__be16 tcp_flags;
809 	unsigned long used;
810 
811 	ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
812 
813 	if (used &&
814 	    nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
815 			      OVS_FLOW_ATTR_PAD))
816 		return -EMSGSIZE;
817 
818 	if (stats.n_packets &&
819 	    nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
820 			  sizeof(struct ovs_flow_stats), &stats,
821 			  OVS_FLOW_ATTR_PAD))
822 		return -EMSGSIZE;
823 
824 	if ((u8)ntohs(tcp_flags) &&
825 	     nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
826 		return -EMSGSIZE;
827 
828 	return 0;
829 }
830 
831 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_actions(const struct sw_flow * flow,struct sk_buff * skb,int skb_orig_len)832 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
833 				     struct sk_buff *skb, int skb_orig_len)
834 {
835 	struct nlattr *start;
836 	int err;
837 
838 	/* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
839 	 * this is the first flow to be dumped into 'skb'.  This is unusual for
840 	 * Netlink but individual action lists can be longer than
841 	 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
842 	 * The userspace caller can always fetch the actions separately if it
843 	 * really wants them.  (Most userspace callers in fact don't care.)
844 	 *
845 	 * This can only fail for dump operations because the skb is always
846 	 * properly sized for single flows.
847 	 */
848 	start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS);
849 	if (start) {
850 		const struct sw_flow_actions *sf_acts;
851 
852 		sf_acts = rcu_dereference_ovsl(flow->sf_acts);
853 		err = ovs_nla_put_actions(sf_acts->actions,
854 					  sf_acts->actions_len, skb);
855 
856 		if (!err)
857 			nla_nest_end(skb, start);
858 		else {
859 			if (skb_orig_len)
860 				return err;
861 
862 			nla_nest_cancel(skb, start);
863 		}
864 	} else if (skb_orig_len) {
865 		return -EMSGSIZE;
866 	}
867 
868 	return 0;
869 }
870 
871 /* Called with ovs_mutex or RCU read lock. */
ovs_flow_cmd_fill_info(const struct sw_flow * flow,int dp_ifindex,struct sk_buff * skb,u32 portid,u32 seq,u32 flags,u8 cmd,u32 ufid_flags)872 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
873 				  struct sk_buff *skb, u32 portid,
874 				  u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
875 {
876 	const int skb_orig_len = skb->len;
877 	struct ovs_header *ovs_header;
878 	int err;
879 
880 	ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
881 				 flags, cmd);
882 	if (!ovs_header)
883 		return -EMSGSIZE;
884 
885 	ovs_header->dp_ifindex = dp_ifindex;
886 
887 	err = ovs_nla_put_identifier(flow, skb);
888 	if (err)
889 		goto error;
890 
891 	if (should_fill_key(&flow->id, ufid_flags)) {
892 		err = ovs_nla_put_masked_key(flow, skb);
893 		if (err)
894 			goto error;
895 	}
896 
897 	if (should_fill_mask(ufid_flags)) {
898 		err = ovs_nla_put_mask(flow, skb);
899 		if (err)
900 			goto error;
901 	}
902 
903 	err = ovs_flow_cmd_fill_stats(flow, skb);
904 	if (err)
905 		goto error;
906 
907 	if (should_fill_actions(ufid_flags)) {
908 		err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
909 		if (err)
910 			goto error;
911 	}
912 
913 	genlmsg_end(skb, ovs_header);
914 	return 0;
915 
916 error:
917 	genlmsg_cancel(skb, ovs_header);
918 	return err;
919 }
920 
921 /* May not be called with RCU read lock. */
ovs_flow_cmd_alloc_info(const struct sw_flow_actions * acts,const struct sw_flow_id * sfid,struct genl_info * info,bool always,uint32_t ufid_flags)922 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
923 					       const struct sw_flow_id *sfid,
924 					       struct genl_info *info,
925 					       bool always,
926 					       uint32_t ufid_flags)
927 {
928 	struct sk_buff *skb;
929 	size_t len;
930 
931 	if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0))
932 		return NULL;
933 
934 	len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
935 	skb = genlmsg_new(len, GFP_KERNEL);
936 	if (!skb)
937 		return ERR_PTR(-ENOMEM);
938 
939 	return skb;
940 }
941 
942 /* Called with ovs_mutex. */
ovs_flow_cmd_build_info(const struct sw_flow * flow,int dp_ifindex,struct genl_info * info,u8 cmd,bool always,u32 ufid_flags)943 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
944 					       int dp_ifindex,
945 					       struct genl_info *info, u8 cmd,
946 					       bool always, u32 ufid_flags)
947 {
948 	struct sk_buff *skb;
949 	int retval;
950 
951 	skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
952 				      &flow->id, info, always, ufid_flags);
953 	if (IS_ERR_OR_NULL(skb))
954 		return skb;
955 
956 	retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
957 					info->snd_portid, info->snd_seq, 0,
958 					cmd, ufid_flags);
959 	if (WARN_ON_ONCE(retval < 0)) {
960 		kfree_skb(skb);
961 		skb = ERR_PTR(retval);
962 	}
963 	return skb;
964 }
965 
ovs_flow_cmd_new(struct sk_buff * skb,struct genl_info * info)966 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
967 {
968 	struct net *net = sock_net(skb->sk);
969 	struct nlattr **a = info->attrs;
970 	struct ovs_header *ovs_header = genl_info_userhdr(info);
971 	struct sw_flow *flow = NULL, *new_flow;
972 	struct sw_flow_mask mask;
973 	struct sk_buff *reply;
974 	struct datapath *dp;
975 	struct sw_flow_key *key;
976 	struct sw_flow_actions *acts;
977 	struct sw_flow_match match;
978 	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
979 	int error;
980 	bool log = !a[OVS_FLOW_ATTR_PROBE];
981 
982 	/* Must have key and actions. */
983 	error = -EINVAL;
984 	if (!a[OVS_FLOW_ATTR_KEY]) {
985 		OVS_NLERR(log, "Flow key attr not present in new flow.");
986 		goto error;
987 	}
988 	if (!a[OVS_FLOW_ATTR_ACTIONS]) {
989 		OVS_NLERR(log, "Flow actions attr not present in new flow.");
990 		goto error;
991 	}
992 
993 	/* Most of the time we need to allocate a new flow, do it before
994 	 * locking.
995 	 */
996 	new_flow = ovs_flow_alloc();
997 	if (IS_ERR(new_flow)) {
998 		error = PTR_ERR(new_flow);
999 		goto error;
1000 	}
1001 
1002 	/* Extract key. */
1003 	key = kzalloc(sizeof(*key), GFP_KERNEL);
1004 	if (!key) {
1005 		error = -ENOMEM;
1006 		goto err_kfree_flow;
1007 	}
1008 
1009 	ovs_match_init(&match, key, false, &mask);
1010 	error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1011 				  a[OVS_FLOW_ATTR_MASK], log);
1012 	if (error)
1013 		goto err_kfree_key;
1014 
1015 	ovs_flow_mask_key(&new_flow->key, key, true, &mask);
1016 
1017 	/* Extract flow identifier. */
1018 	error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
1019 				       key, log);
1020 	if (error)
1021 		goto err_kfree_key;
1022 
1023 	/* Validate actions. */
1024 	error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
1025 				     &new_flow->key, &acts, log);
1026 	if (error) {
1027 		OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
1028 		goto err_kfree_key;
1029 	}
1030 
1031 	reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
1032 					ufid_flags);
1033 	if (IS_ERR(reply)) {
1034 		error = PTR_ERR(reply);
1035 		goto err_kfree_acts;
1036 	}
1037 
1038 	ovs_lock();
1039 	dp = get_dp(net, ovs_header->dp_ifindex);
1040 	if (unlikely(!dp)) {
1041 		error = -ENODEV;
1042 		goto err_unlock_ovs;
1043 	}
1044 
1045 	/* Check if this is a duplicate flow */
1046 	if (ovs_identifier_is_ufid(&new_flow->id))
1047 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
1048 	if (!flow)
1049 		flow = ovs_flow_tbl_lookup(&dp->table, key);
1050 	if (likely(!flow)) {
1051 		rcu_assign_pointer(new_flow->sf_acts, acts);
1052 
1053 		/* Put flow in bucket. */
1054 		error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
1055 		if (unlikely(error)) {
1056 			acts = NULL;
1057 			goto err_unlock_ovs;
1058 		}
1059 
1060 		if (unlikely(reply)) {
1061 			error = ovs_flow_cmd_fill_info(new_flow,
1062 						       ovs_header->dp_ifindex,
1063 						       reply, info->snd_portid,
1064 						       info->snd_seq, 0,
1065 						       OVS_FLOW_CMD_NEW,
1066 						       ufid_flags);
1067 			BUG_ON(error < 0);
1068 		}
1069 		ovs_unlock();
1070 	} else {
1071 		struct sw_flow_actions *old_acts;
1072 
1073 		/* Bail out if we're not allowed to modify an existing flow.
1074 		 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1075 		 * because Generic Netlink treats the latter as a dump
1076 		 * request.  We also accept NLM_F_EXCL in case that bug ever
1077 		 * gets fixed.
1078 		 */
1079 		if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
1080 							 | NLM_F_EXCL))) {
1081 			error = -EEXIST;
1082 			goto err_unlock_ovs;
1083 		}
1084 		/* The flow identifier has to be the same for flow updates.
1085 		 * Look for any overlapping flow.
1086 		 */
1087 		if (unlikely(!ovs_flow_cmp(flow, &match))) {
1088 			if (ovs_identifier_is_key(&flow->id))
1089 				flow = ovs_flow_tbl_lookup_exact(&dp->table,
1090 								 &match);
1091 			else /* UFID matches but key is different */
1092 				flow = NULL;
1093 			if (!flow) {
1094 				error = -ENOENT;
1095 				goto err_unlock_ovs;
1096 			}
1097 		}
1098 		/* Update actions. */
1099 		old_acts = ovsl_dereference(flow->sf_acts);
1100 		rcu_assign_pointer(flow->sf_acts, acts);
1101 
1102 		if (unlikely(reply)) {
1103 			error = ovs_flow_cmd_fill_info(flow,
1104 						       ovs_header->dp_ifindex,
1105 						       reply, info->snd_portid,
1106 						       info->snd_seq, 0,
1107 						       OVS_FLOW_CMD_NEW,
1108 						       ufid_flags);
1109 			BUG_ON(error < 0);
1110 		}
1111 		ovs_unlock();
1112 
1113 		ovs_nla_free_flow_actions_rcu(old_acts);
1114 		ovs_flow_free(new_flow, false);
1115 	}
1116 
1117 	if (reply)
1118 		ovs_notify(&dp_flow_genl_family, reply, info);
1119 
1120 	kfree(key);
1121 	return 0;
1122 
1123 err_unlock_ovs:
1124 	ovs_unlock();
1125 	kfree_skb(reply);
1126 err_kfree_acts:
1127 	ovs_nla_free_flow_actions(acts);
1128 err_kfree_key:
1129 	kfree(key);
1130 err_kfree_flow:
1131 	ovs_flow_free(new_flow, false);
1132 error:
1133 	return error;
1134 }
1135 
1136 /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1137 static noinline_for_stack
get_flow_actions(struct net * net,const struct nlattr * a,const struct sw_flow_key * key,const struct sw_flow_mask * mask,bool log)1138 struct sw_flow_actions *get_flow_actions(struct net *net,
1139 					 const struct nlattr *a,
1140 					 const struct sw_flow_key *key,
1141 					 const struct sw_flow_mask *mask,
1142 					 bool log)
1143 {
1144 	struct sw_flow_actions *acts;
1145 	struct sw_flow_key masked_key;
1146 	int error;
1147 
1148 	ovs_flow_mask_key(&masked_key, key, true, mask);
1149 	error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
1150 	if (error) {
1151 		OVS_NLERR(log,
1152 			  "Actions may not be safe on all matching packets");
1153 		return ERR_PTR(error);
1154 	}
1155 
1156 	return acts;
1157 }
1158 
1159 /* Factor out match-init and action-copy to avoid
1160  * "Wframe-larger-than=1024" warning. Because mask is only
1161  * used to get actions, we new a function to save some
1162  * stack space.
1163  *
1164  * If there are not key and action attrs, we return 0
1165  * directly. In the case, the caller will also not use the
1166  * match as before. If there is action attr, we try to get
1167  * actions and save them to *acts. Before returning from
1168  * the function, we reset the match->mask pointer. Because
1169  * we should not to return match object with dangling reference
1170  * to mask.
1171  * */
1172 static noinline_for_stack int
ovs_nla_init_match_and_action(struct net * net,struct sw_flow_match * match,struct sw_flow_key * key,struct nlattr ** a,struct sw_flow_actions ** acts,bool log)1173 ovs_nla_init_match_and_action(struct net *net,
1174 			      struct sw_flow_match *match,
1175 			      struct sw_flow_key *key,
1176 			      struct nlattr **a,
1177 			      struct sw_flow_actions **acts,
1178 			      bool log)
1179 {
1180 	struct sw_flow_mask mask;
1181 	int error = 0;
1182 
1183 	if (a[OVS_FLOW_ATTR_KEY]) {
1184 		ovs_match_init(match, key, true, &mask);
1185 		error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
1186 					  a[OVS_FLOW_ATTR_MASK], log);
1187 		if (error)
1188 			goto error;
1189 	}
1190 
1191 	if (a[OVS_FLOW_ATTR_ACTIONS]) {
1192 		if (!a[OVS_FLOW_ATTR_KEY]) {
1193 			OVS_NLERR(log,
1194 				  "Flow key attribute not present in set flow.");
1195 			error = -EINVAL;
1196 			goto error;
1197 		}
1198 
1199 		*acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
1200 					 &mask, log);
1201 		if (IS_ERR(*acts)) {
1202 			error = PTR_ERR(*acts);
1203 			goto error;
1204 		}
1205 	}
1206 
1207 	/* On success, error is 0. */
1208 error:
1209 	match->mask = NULL;
1210 	return error;
1211 }
1212 
ovs_flow_cmd_set(struct sk_buff * skb,struct genl_info * info)1213 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1214 {
1215 	struct net *net = sock_net(skb->sk);
1216 	struct nlattr **a = info->attrs;
1217 	struct ovs_header *ovs_header = genl_info_userhdr(info);
1218 	struct sw_flow_key key;
1219 	struct sw_flow *flow;
1220 	struct sk_buff *reply = NULL;
1221 	struct datapath *dp;
1222 	struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1223 	struct sw_flow_match match;
1224 	struct sw_flow_id sfid;
1225 	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1226 	int error = 0;
1227 	bool log = !a[OVS_FLOW_ATTR_PROBE];
1228 	bool ufid_present;
1229 
1230 	ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1231 	if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
1232 		OVS_NLERR(log,
1233 			  "Flow set message rejected, Key attribute missing.");
1234 		return -EINVAL;
1235 	}
1236 
1237 	error = ovs_nla_init_match_and_action(net, &match, &key, a,
1238 					      &acts, log);
1239 	if (error)
1240 		goto error;
1241 
1242 	if (acts) {
1243 		/* Can allocate before locking if have acts. */
1244 		reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1245 						ufid_flags);
1246 		if (IS_ERR(reply)) {
1247 			error = PTR_ERR(reply);
1248 			goto err_kfree_acts;
1249 		}
1250 	}
1251 
1252 	ovs_lock();
1253 	dp = get_dp(net, ovs_header->dp_ifindex);
1254 	if (unlikely(!dp)) {
1255 		error = -ENODEV;
1256 		goto err_unlock_ovs;
1257 	}
1258 	/* Check that the flow exists. */
1259 	if (ufid_present)
1260 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1261 	else
1262 		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1263 	if (unlikely(!flow)) {
1264 		error = -ENOENT;
1265 		goto err_unlock_ovs;
1266 	}
1267 
1268 	/* Update actions, if present. */
1269 	if (likely(acts)) {
1270 		old_acts = ovsl_dereference(flow->sf_acts);
1271 		rcu_assign_pointer(flow->sf_acts, acts);
1272 
1273 		if (unlikely(reply)) {
1274 			error = ovs_flow_cmd_fill_info(flow,
1275 						       ovs_header->dp_ifindex,
1276 						       reply, info->snd_portid,
1277 						       info->snd_seq, 0,
1278 						       OVS_FLOW_CMD_SET,
1279 						       ufid_flags);
1280 			BUG_ON(error < 0);
1281 		}
1282 	} else {
1283 		/* Could not alloc without acts before locking. */
1284 		reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1285 						info, OVS_FLOW_CMD_SET, false,
1286 						ufid_flags);
1287 
1288 		if (IS_ERR(reply)) {
1289 			error = PTR_ERR(reply);
1290 			goto err_unlock_ovs;
1291 		}
1292 	}
1293 
1294 	/* Clear stats. */
1295 	if (a[OVS_FLOW_ATTR_CLEAR])
1296 		ovs_flow_stats_clear(flow);
1297 	ovs_unlock();
1298 
1299 	if (reply)
1300 		ovs_notify(&dp_flow_genl_family, reply, info);
1301 	if (old_acts)
1302 		ovs_nla_free_flow_actions_rcu(old_acts);
1303 
1304 	return 0;
1305 
1306 err_unlock_ovs:
1307 	ovs_unlock();
1308 	kfree_skb(reply);
1309 err_kfree_acts:
1310 	ovs_nla_free_flow_actions(acts);
1311 error:
1312 	return error;
1313 }
1314 
ovs_flow_cmd_get(struct sk_buff * skb,struct genl_info * info)1315 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
1316 {
1317 	struct nlattr **a = info->attrs;
1318 	struct ovs_header *ovs_header = genl_info_userhdr(info);
1319 	struct net *net = sock_net(skb->sk);
1320 	struct sw_flow_key key;
1321 	struct sk_buff *reply;
1322 	struct sw_flow *flow;
1323 	struct datapath *dp;
1324 	struct sw_flow_match match;
1325 	struct sw_flow_id ufid;
1326 	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1327 	int err = 0;
1328 	bool log = !a[OVS_FLOW_ATTR_PROBE];
1329 	bool ufid_present;
1330 
1331 	ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1332 	if (a[OVS_FLOW_ATTR_KEY]) {
1333 		ovs_match_init(&match, &key, true, NULL);
1334 		err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
1335 					log);
1336 	} else if (!ufid_present) {
1337 		OVS_NLERR(log,
1338 			  "Flow get message rejected, Key attribute missing.");
1339 		err = -EINVAL;
1340 	}
1341 	if (err)
1342 		return err;
1343 
1344 	ovs_lock();
1345 	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1346 	if (!dp) {
1347 		err = -ENODEV;
1348 		goto unlock;
1349 	}
1350 
1351 	if (ufid_present)
1352 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1353 	else
1354 		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1355 	if (!flow) {
1356 		err = -ENOENT;
1357 		goto unlock;
1358 	}
1359 
1360 	reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1361 					OVS_FLOW_CMD_GET, true, ufid_flags);
1362 	if (IS_ERR(reply)) {
1363 		err = PTR_ERR(reply);
1364 		goto unlock;
1365 	}
1366 
1367 	ovs_unlock();
1368 	return genlmsg_reply(reply, info);
1369 unlock:
1370 	ovs_unlock();
1371 	return err;
1372 }
1373 
ovs_flow_cmd_del(struct sk_buff * skb,struct genl_info * info)1374 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
1375 {
1376 	struct nlattr **a = info->attrs;
1377 	struct ovs_header *ovs_header = genl_info_userhdr(info);
1378 	struct net *net = sock_net(skb->sk);
1379 	struct sw_flow_key key;
1380 	struct sk_buff *reply;
1381 	struct sw_flow *flow = NULL;
1382 	struct datapath *dp;
1383 	struct sw_flow_match match;
1384 	struct sw_flow_id ufid;
1385 	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1386 	int err;
1387 	bool log = !a[OVS_FLOW_ATTR_PROBE];
1388 	bool ufid_present;
1389 
1390 	ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1391 	if (a[OVS_FLOW_ATTR_KEY]) {
1392 		ovs_match_init(&match, &key, true, NULL);
1393 		err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1394 					NULL, log);
1395 		if (unlikely(err))
1396 			return err;
1397 	}
1398 
1399 	ovs_lock();
1400 	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1401 	if (unlikely(!dp)) {
1402 		err = -ENODEV;
1403 		goto unlock;
1404 	}
1405 
1406 	if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1407 		err = ovs_flow_tbl_flush(&dp->table);
1408 		goto unlock;
1409 	}
1410 
1411 	if (ufid_present)
1412 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1413 	else
1414 		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1415 	if (unlikely(!flow)) {
1416 		err = -ENOENT;
1417 		goto unlock;
1418 	}
1419 
1420 	ovs_flow_tbl_remove(&dp->table, flow);
1421 	ovs_unlock();
1422 
1423 	reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts,
1424 					&flow->id, info, false, ufid_flags);
1425 	if (likely(reply)) {
1426 		if (!IS_ERR(reply)) {
1427 			rcu_read_lock();	/*To keep RCU checker happy. */
1428 			err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
1429 						     reply, info->snd_portid,
1430 						     info->snd_seq, 0,
1431 						     OVS_FLOW_CMD_DEL,
1432 						     ufid_flags);
1433 			rcu_read_unlock();
1434 			if (WARN_ON_ONCE(err < 0)) {
1435 				kfree_skb(reply);
1436 				goto out_free;
1437 			}
1438 
1439 			ovs_notify(&dp_flow_genl_family, reply, info);
1440 		} else {
1441 			netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0,
1442 					PTR_ERR(reply));
1443 		}
1444 	}
1445 
1446 out_free:
1447 	ovs_flow_free(flow, true);
1448 	return 0;
1449 unlock:
1450 	ovs_unlock();
1451 	return err;
1452 }
1453 
ovs_flow_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)1454 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1455 {
1456 	struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1457 	struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1458 	struct table_instance *ti;
1459 	struct datapath *dp;
1460 	u32 ufid_flags;
1461 	int err;
1462 
1463 	err = genlmsg_parse_deprecated(cb->nlh, &dp_flow_genl_family, a,
1464 				       OVS_FLOW_ATTR_MAX, flow_policy, NULL);
1465 	if (err)
1466 		return err;
1467 	ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1468 
1469 	rcu_read_lock();
1470 	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1471 	if (!dp) {
1472 		rcu_read_unlock();
1473 		return -ENODEV;
1474 	}
1475 
1476 	ti = rcu_dereference(dp->table.ti);
1477 	for (;;) {
1478 		struct sw_flow *flow;
1479 		u32 bucket, obj;
1480 
1481 		bucket = cb->args[0];
1482 		obj = cb->args[1];
1483 		flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1484 		if (!flow)
1485 			break;
1486 
1487 		if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1488 					   NETLINK_CB(cb->skb).portid,
1489 					   cb->nlh->nlmsg_seq, NLM_F_MULTI,
1490 					   OVS_FLOW_CMD_GET, ufid_flags) < 0)
1491 			break;
1492 
1493 		cb->args[0] = bucket;
1494 		cb->args[1] = obj;
1495 	}
1496 	rcu_read_unlock();
1497 	return skb->len;
1498 }
1499 
1500 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1501 	[OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1502 	[OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1503 	[OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1504 	[OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1505 	[OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1506 	[OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1507 	[OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1508 };
1509 
1510 static const struct genl_small_ops dp_flow_genl_ops[] = {
1511 	{ .cmd = OVS_FLOW_CMD_NEW,
1512 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1513 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1514 	  .doit = ovs_flow_cmd_new
1515 	},
1516 	{ .cmd = OVS_FLOW_CMD_DEL,
1517 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1518 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1519 	  .doit = ovs_flow_cmd_del
1520 	},
1521 	{ .cmd = OVS_FLOW_CMD_GET,
1522 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1523 	  .flags = 0,		    /* OK for unprivileged users. */
1524 	  .doit = ovs_flow_cmd_get,
1525 	  .dumpit = ovs_flow_cmd_dump
1526 	},
1527 	{ .cmd = OVS_FLOW_CMD_SET,
1528 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
1529 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1530 	  .doit = ovs_flow_cmd_set,
1531 	},
1532 };
1533 
1534 static struct genl_family dp_flow_genl_family __ro_after_init = {
1535 	.hdrsize = sizeof(struct ovs_header),
1536 	.name = OVS_FLOW_FAMILY,
1537 	.version = OVS_FLOW_VERSION,
1538 	.maxattr = OVS_FLOW_ATTR_MAX,
1539 	.policy = flow_policy,
1540 	.netnsok = true,
1541 	.parallel_ops = true,
1542 	.small_ops = dp_flow_genl_ops,
1543 	.n_small_ops = ARRAY_SIZE(dp_flow_genl_ops),
1544 	.resv_start_op = OVS_FLOW_CMD_SET + 1,
1545 	.mcgrps = &ovs_dp_flow_multicast_group,
1546 	.n_mcgrps = 1,
1547 	.module = THIS_MODULE,
1548 };
1549 
ovs_dp_cmd_msg_size(void)1550 static size_t ovs_dp_cmd_msg_size(void)
1551 {
1552 	size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1553 
1554 	msgsize += nla_total_size(IFNAMSIZ);
1555 	msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
1556 	msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
1557 	msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1558 	msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
1559 	msgsize += nla_total_size(sizeof(u32) * nr_cpu_ids); /* OVS_DP_ATTR_PER_CPU_PIDS */
1560 
1561 	return msgsize;
1562 }
1563 
1564 /* Called with ovs_mutex. */
ovs_dp_cmd_fill_info(struct datapath * dp,struct sk_buff * skb,u32 portid,u32 seq,u32 flags,u8 cmd)1565 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1566 				u32 portid, u32 seq, u32 flags, u8 cmd)
1567 {
1568 	struct ovs_header *ovs_header;
1569 	struct ovs_dp_stats dp_stats;
1570 	struct ovs_dp_megaflow_stats dp_megaflow_stats;
1571 	struct dp_nlsk_pids *pids = ovsl_dereference(dp->upcall_portids);
1572 	int err, pids_len;
1573 
1574 	ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1575 				 flags, cmd);
1576 	if (!ovs_header)
1577 		goto error;
1578 
1579 	ovs_header->dp_ifindex = get_dpifindex(dp);
1580 
1581 	err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1582 	if (err)
1583 		goto nla_put_failure;
1584 
1585 	get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1586 	if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1587 			  &dp_stats, OVS_DP_ATTR_PAD))
1588 		goto nla_put_failure;
1589 
1590 	if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1591 			  sizeof(struct ovs_dp_megaflow_stats),
1592 			  &dp_megaflow_stats, OVS_DP_ATTR_PAD))
1593 		goto nla_put_failure;
1594 
1595 	if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1596 		goto nla_put_failure;
1597 
1598 	if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE,
1599 			ovs_flow_tbl_masks_cache_size(&dp->table)))
1600 		goto nla_put_failure;
1601 
1602 	if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) {
1603 		pids_len = min(pids->n_pids, nr_cpu_ids) * sizeof(u32);
1604 		if (nla_put(skb, OVS_DP_ATTR_PER_CPU_PIDS, pids_len, &pids->pids))
1605 			goto nla_put_failure;
1606 	}
1607 
1608 	genlmsg_end(skb, ovs_header);
1609 	return 0;
1610 
1611 nla_put_failure:
1612 	genlmsg_cancel(skb, ovs_header);
1613 error:
1614 	return -EMSGSIZE;
1615 }
1616 
ovs_dp_cmd_alloc_info(void)1617 static struct sk_buff *ovs_dp_cmd_alloc_info(void)
1618 {
1619 	return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
1620 }
1621 
1622 /* Called with rcu_read_lock or ovs_mutex. */
lookup_datapath(struct net * net,const struct ovs_header * ovs_header,struct nlattr * a[OVS_DP_ATTR_MAX+1])1623 static struct datapath *lookup_datapath(struct net *net,
1624 					const struct ovs_header *ovs_header,
1625 					struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1626 {
1627 	struct datapath *dp;
1628 
1629 	if (!a[OVS_DP_ATTR_NAME])
1630 		dp = get_dp(net, ovs_header->dp_ifindex);
1631 	else {
1632 		struct vport *vport;
1633 
1634 		vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1635 		dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1636 	}
1637 	return dp ? dp : ERR_PTR(-ENODEV);
1638 }
1639 
ovs_dp_reset_user_features(struct sk_buff * skb,struct genl_info * info)1640 static void ovs_dp_reset_user_features(struct sk_buff *skb,
1641 				       struct genl_info *info)
1642 {
1643 	struct datapath *dp;
1644 
1645 	dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
1646 			     info->attrs);
1647 	if (IS_ERR(dp))
1648 		return;
1649 
1650 	pr_warn("%s: Dropping previously announced user features\n",
1651 		ovs_dp_name(dp));
1652 	dp->user_features = 0;
1653 }
1654 
ovs_dp_set_upcall_portids(struct datapath * dp,const struct nlattr * ids)1655 static int ovs_dp_set_upcall_portids(struct datapath *dp,
1656 			      const struct nlattr *ids)
1657 {
1658 	struct dp_nlsk_pids *old, *dp_nlsk_pids;
1659 
1660 	if (!nla_len(ids) || nla_len(ids) % sizeof(u32))
1661 		return -EINVAL;
1662 
1663 	old = ovsl_dereference(dp->upcall_portids);
1664 
1665 	dp_nlsk_pids = kmalloc(sizeof(*dp_nlsk_pids) + nla_len(ids),
1666 			       GFP_KERNEL);
1667 	if (!dp_nlsk_pids)
1668 		return -ENOMEM;
1669 
1670 	dp_nlsk_pids->n_pids = nla_len(ids) / sizeof(u32);
1671 	nla_memcpy(dp_nlsk_pids->pids, ids, nla_len(ids));
1672 
1673 	rcu_assign_pointer(dp->upcall_portids, dp_nlsk_pids);
1674 
1675 	kfree_rcu(old, rcu);
1676 
1677 	return 0;
1678 }
1679 
ovs_dp_get_upcall_portid(const struct datapath * dp,uint32_t cpu_id)1680 u32 ovs_dp_get_upcall_portid(const struct datapath *dp, uint32_t cpu_id)
1681 {
1682 	struct dp_nlsk_pids *dp_nlsk_pids;
1683 
1684 	dp_nlsk_pids = rcu_dereference(dp->upcall_portids);
1685 
1686 	if (dp_nlsk_pids) {
1687 		if (cpu_id < dp_nlsk_pids->n_pids) {
1688 			return dp_nlsk_pids->pids[cpu_id];
1689 		} else if (dp_nlsk_pids->n_pids > 0 &&
1690 			   cpu_id >= dp_nlsk_pids->n_pids) {
1691 			/* If the number of netlink PIDs is mismatched with
1692 			 * the number of CPUs as seen by the kernel, log this
1693 			 * and send the upcall to an arbitrary socket (0) in
1694 			 * order to not drop packets
1695 			 */
1696 			pr_info_ratelimited("cpu_id mismatch with handler threads");
1697 			return dp_nlsk_pids->pids[cpu_id %
1698 						  dp_nlsk_pids->n_pids];
1699 		} else {
1700 			return 0;
1701 		}
1702 	} else {
1703 		return 0;
1704 	}
1705 }
1706 
ovs_dp_change(struct datapath * dp,struct nlattr * a[])1707 static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1708 {
1709 	u32 user_features = 0, old_features = dp->user_features;
1710 	int err;
1711 
1712 	if (a[OVS_DP_ATTR_USER_FEATURES]) {
1713 		user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1714 
1715 		if (user_features & ~(OVS_DP_F_VPORT_PIDS |
1716 				      OVS_DP_F_UNALIGNED |
1717 				      OVS_DP_F_TC_RECIRC_SHARING |
1718 				      OVS_DP_F_DISPATCH_UPCALL_PER_CPU))
1719 			return -EOPNOTSUPP;
1720 
1721 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1722 		if (user_features & OVS_DP_F_TC_RECIRC_SHARING)
1723 			return -EOPNOTSUPP;
1724 #endif
1725 	}
1726 
1727 	if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) {
1728 		int err;
1729 		u32 cache_size;
1730 
1731 		cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]);
1732 		err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size);
1733 		if (err)
1734 			return err;
1735 	}
1736 
1737 	dp->user_features = user_features;
1738 
1739 	if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU &&
1740 	    a[OVS_DP_ATTR_PER_CPU_PIDS]) {
1741 		/* Upcall Netlink Port IDs have been updated */
1742 		err = ovs_dp_set_upcall_portids(dp,
1743 						a[OVS_DP_ATTR_PER_CPU_PIDS]);
1744 		if (err)
1745 			return err;
1746 	}
1747 
1748 	if ((dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) &&
1749 	    !(old_features & OVS_DP_F_TC_RECIRC_SHARING))
1750 		tc_skb_ext_tc_enable();
1751 	else if (!(dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) &&
1752 		 (old_features & OVS_DP_F_TC_RECIRC_SHARING))
1753 		tc_skb_ext_tc_disable();
1754 
1755 	return 0;
1756 }
1757 
ovs_dp_stats_init(struct datapath * dp)1758 static int ovs_dp_stats_init(struct datapath *dp)
1759 {
1760 	dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1761 	if (!dp->stats_percpu)
1762 		return -ENOMEM;
1763 
1764 	return 0;
1765 }
1766 
ovs_dp_vport_init(struct datapath * dp)1767 static int ovs_dp_vport_init(struct datapath *dp)
1768 {
1769 	int i;
1770 
1771 	dp->ports = kmalloc_array(DP_VPORT_HASH_BUCKETS,
1772 				  sizeof(struct hlist_head),
1773 				  GFP_KERNEL);
1774 	if (!dp->ports)
1775 		return -ENOMEM;
1776 
1777 	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1778 		INIT_HLIST_HEAD(&dp->ports[i]);
1779 
1780 	return 0;
1781 }
1782 
ovs_dp_cmd_new(struct sk_buff * skb,struct genl_info * info)1783 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1784 {
1785 	struct nlattr **a = info->attrs;
1786 	struct vport_parms parms;
1787 	struct sk_buff *reply;
1788 	struct datapath *dp;
1789 	struct vport *vport;
1790 	struct ovs_net *ovs_net;
1791 	int err;
1792 
1793 	err = -EINVAL;
1794 	if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1795 		goto err;
1796 
1797 	reply = ovs_dp_cmd_alloc_info();
1798 	if (!reply)
1799 		return -ENOMEM;
1800 
1801 	err = -ENOMEM;
1802 	dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1803 	if (dp == NULL)
1804 		goto err_destroy_reply;
1805 
1806 	ovs_dp_set_net(dp, sock_net(skb->sk));
1807 
1808 	/* Allocate table. */
1809 	err = ovs_flow_tbl_init(&dp->table);
1810 	if (err)
1811 		goto err_destroy_dp;
1812 
1813 	err = ovs_dp_stats_init(dp);
1814 	if (err)
1815 		goto err_destroy_table;
1816 
1817 	err = ovs_dp_vport_init(dp);
1818 	if (err)
1819 		goto err_destroy_stats;
1820 
1821 	err = ovs_meters_init(dp);
1822 	if (err)
1823 		goto err_destroy_ports;
1824 
1825 	/* Set up our datapath device. */
1826 	parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1827 	parms.type = OVS_VPORT_TYPE_INTERNAL;
1828 	parms.options = NULL;
1829 	parms.dp = dp;
1830 	parms.port_no = OVSP_LOCAL;
1831 	parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1832 	parms.desired_ifindex = a[OVS_DP_ATTR_IFINDEX]
1833 		? nla_get_s32(a[OVS_DP_ATTR_IFINDEX]) : 0;
1834 
1835 	/* So far only local changes have been made, now need the lock. */
1836 	ovs_lock();
1837 
1838 	err = ovs_dp_change(dp, a);
1839 	if (err)
1840 		goto err_unlock_and_destroy_meters;
1841 
1842 	vport = new_vport(&parms);
1843 	if (IS_ERR(vport)) {
1844 		err = PTR_ERR(vport);
1845 		if (err == -EBUSY)
1846 			err = -EEXIST;
1847 
1848 		if (err == -EEXIST) {
1849 			/* An outdated user space instance that does not understand
1850 			 * the concept of user_features has attempted to create a new
1851 			 * datapath and is likely to reuse it. Drop all user features.
1852 			 */
1853 			if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1854 				ovs_dp_reset_user_features(skb, info);
1855 		}
1856 
1857 		goto err_destroy_portids;
1858 	}
1859 
1860 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1861 				   info->snd_seq, 0, OVS_DP_CMD_NEW);
1862 	BUG_ON(err < 0);
1863 
1864 	ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1865 	list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1866 
1867 	ovs_unlock();
1868 
1869 	ovs_notify(&dp_datapath_genl_family, reply, info);
1870 	return 0;
1871 
1872 err_destroy_portids:
1873 	kfree(rcu_dereference_raw(dp->upcall_portids));
1874 err_unlock_and_destroy_meters:
1875 	ovs_unlock();
1876 	ovs_meters_exit(dp);
1877 err_destroy_ports:
1878 	kfree(dp->ports);
1879 err_destroy_stats:
1880 	free_percpu(dp->stats_percpu);
1881 err_destroy_table:
1882 	ovs_flow_tbl_destroy(&dp->table);
1883 err_destroy_dp:
1884 	kfree(dp);
1885 err_destroy_reply:
1886 	kfree_skb(reply);
1887 err:
1888 	return err;
1889 }
1890 
1891 /* Called with ovs_mutex. */
__dp_destroy(struct datapath * dp)1892 static void __dp_destroy(struct datapath *dp)
1893 {
1894 	struct flow_table *table = &dp->table;
1895 	int i;
1896 
1897 	if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING)
1898 		tc_skb_ext_tc_disable();
1899 
1900 	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1901 		struct vport *vport;
1902 		struct hlist_node *n;
1903 
1904 		hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1905 			if (vport->port_no != OVSP_LOCAL)
1906 				ovs_dp_detach_port(vport);
1907 	}
1908 
1909 	list_del_rcu(&dp->list_node);
1910 
1911 	/* OVSP_LOCAL is datapath internal port. We need to make sure that
1912 	 * all ports in datapath are destroyed first before freeing datapath.
1913 	 */
1914 	ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1915 
1916 	/* Flush sw_flow in the tables. RCU cb only releases resource
1917 	 * such as dp, ports and tables. That may avoid some issues
1918 	 * such as RCU usage warning.
1919 	 */
1920 	table_instance_flow_flush(table, ovsl_dereference(table->ti),
1921 				  ovsl_dereference(table->ufid_ti));
1922 
1923 	/* RCU destroy the ports, meters and flow tables. */
1924 	call_rcu(&dp->rcu, destroy_dp_rcu);
1925 }
1926 
ovs_dp_cmd_del(struct sk_buff * skb,struct genl_info * info)1927 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1928 {
1929 	struct sk_buff *reply;
1930 	struct datapath *dp;
1931 	int err;
1932 
1933 	reply = ovs_dp_cmd_alloc_info();
1934 	if (!reply)
1935 		return -ENOMEM;
1936 
1937 	ovs_lock();
1938 	dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
1939 			     info->attrs);
1940 	err = PTR_ERR(dp);
1941 	if (IS_ERR(dp))
1942 		goto err_unlock_free;
1943 
1944 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1945 				   info->snd_seq, 0, OVS_DP_CMD_DEL);
1946 	BUG_ON(err < 0);
1947 
1948 	__dp_destroy(dp);
1949 	ovs_unlock();
1950 
1951 	ovs_notify(&dp_datapath_genl_family, reply, info);
1952 
1953 	return 0;
1954 
1955 err_unlock_free:
1956 	ovs_unlock();
1957 	kfree_skb(reply);
1958 	return err;
1959 }
1960 
ovs_dp_cmd_set(struct sk_buff * skb,struct genl_info * info)1961 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1962 {
1963 	struct sk_buff *reply;
1964 	struct datapath *dp;
1965 	int err;
1966 
1967 	reply = ovs_dp_cmd_alloc_info();
1968 	if (!reply)
1969 		return -ENOMEM;
1970 
1971 	ovs_lock();
1972 	dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
1973 			     info->attrs);
1974 	err = PTR_ERR(dp);
1975 	if (IS_ERR(dp))
1976 		goto err_unlock_free;
1977 
1978 	err = ovs_dp_change(dp, info->attrs);
1979 	if (err)
1980 		goto err_unlock_free;
1981 
1982 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1983 				   info->snd_seq, 0, OVS_DP_CMD_SET);
1984 	BUG_ON(err < 0);
1985 
1986 	ovs_unlock();
1987 	ovs_notify(&dp_datapath_genl_family, reply, info);
1988 
1989 	return 0;
1990 
1991 err_unlock_free:
1992 	ovs_unlock();
1993 	kfree_skb(reply);
1994 	return err;
1995 }
1996 
ovs_dp_cmd_get(struct sk_buff * skb,struct genl_info * info)1997 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1998 {
1999 	struct sk_buff *reply;
2000 	struct datapath *dp;
2001 	int err;
2002 
2003 	reply = ovs_dp_cmd_alloc_info();
2004 	if (!reply)
2005 		return -ENOMEM;
2006 
2007 	ovs_lock();
2008 	dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info),
2009 			     info->attrs);
2010 	if (IS_ERR(dp)) {
2011 		err = PTR_ERR(dp);
2012 		goto err_unlock_free;
2013 	}
2014 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
2015 				   info->snd_seq, 0, OVS_DP_CMD_GET);
2016 	BUG_ON(err < 0);
2017 	ovs_unlock();
2018 
2019 	return genlmsg_reply(reply, info);
2020 
2021 err_unlock_free:
2022 	ovs_unlock();
2023 	kfree_skb(reply);
2024 	return err;
2025 }
2026 
ovs_dp_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)2027 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2028 {
2029 	struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
2030 	struct datapath *dp;
2031 	int skip = cb->args[0];
2032 	int i = 0;
2033 
2034 	ovs_lock();
2035 	list_for_each_entry(dp, &ovs_net->dps, list_node) {
2036 		if (i >= skip &&
2037 		    ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
2038 					 cb->nlh->nlmsg_seq, NLM_F_MULTI,
2039 					 OVS_DP_CMD_GET) < 0)
2040 			break;
2041 		i++;
2042 	}
2043 	ovs_unlock();
2044 
2045 	cb->args[0] = i;
2046 
2047 	return skb->len;
2048 }
2049 
2050 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
2051 	[OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2052 	[OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2053 	[OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
2054 	[OVS_DP_ATTR_MASKS_CACHE_SIZE] =  NLA_POLICY_RANGE(NLA_U32, 0,
2055 		PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)),
2056 	[OVS_DP_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0),
2057 };
2058 
2059 static const struct genl_small_ops dp_datapath_genl_ops[] = {
2060 	{ .cmd = OVS_DP_CMD_NEW,
2061 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2062 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2063 	  .doit = ovs_dp_cmd_new
2064 	},
2065 	{ .cmd = OVS_DP_CMD_DEL,
2066 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2067 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2068 	  .doit = ovs_dp_cmd_del
2069 	},
2070 	{ .cmd = OVS_DP_CMD_GET,
2071 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2072 	  .flags = 0,		    /* OK for unprivileged users. */
2073 	  .doit = ovs_dp_cmd_get,
2074 	  .dumpit = ovs_dp_cmd_dump
2075 	},
2076 	{ .cmd = OVS_DP_CMD_SET,
2077 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2078 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2079 	  .doit = ovs_dp_cmd_set,
2080 	},
2081 };
2082 
2083 static struct genl_family dp_datapath_genl_family __ro_after_init = {
2084 	.hdrsize = sizeof(struct ovs_header),
2085 	.name = OVS_DATAPATH_FAMILY,
2086 	.version = OVS_DATAPATH_VERSION,
2087 	.maxattr = OVS_DP_ATTR_MAX,
2088 	.policy = datapath_policy,
2089 	.netnsok = true,
2090 	.parallel_ops = true,
2091 	.small_ops = dp_datapath_genl_ops,
2092 	.n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops),
2093 	.resv_start_op = OVS_DP_CMD_SET + 1,
2094 	.mcgrps = &ovs_dp_datapath_multicast_group,
2095 	.n_mcgrps = 1,
2096 	.module = THIS_MODULE,
2097 };
2098 
2099 /* Called with ovs_mutex or RCU read lock. */
ovs_vport_cmd_fill_info(struct vport * vport,struct sk_buff * skb,struct net * net,u32 portid,u32 seq,u32 flags,u8 cmd,gfp_t gfp)2100 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
2101 				   struct net *net, u32 portid, u32 seq,
2102 				   u32 flags, u8 cmd, gfp_t gfp)
2103 {
2104 	struct ovs_header *ovs_header;
2105 	struct ovs_vport_stats vport_stats;
2106 	int err;
2107 
2108 	ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
2109 				 flags, cmd);
2110 	if (!ovs_header)
2111 		return -EMSGSIZE;
2112 
2113 	ovs_header->dp_ifindex = get_dpifindex(vport->dp);
2114 
2115 	if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
2116 	    nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
2117 	    nla_put_string(skb, OVS_VPORT_ATTR_NAME,
2118 			   ovs_vport_name(vport)) ||
2119 	    nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
2120 		goto nla_put_failure;
2121 
2122 	if (!net_eq(net, dev_net(vport->dev))) {
2123 		int id = peernet2id_alloc(net, dev_net(vport->dev), gfp);
2124 
2125 		if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
2126 			goto nla_put_failure;
2127 	}
2128 
2129 	ovs_vport_get_stats(vport, &vport_stats);
2130 	if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
2131 			  sizeof(struct ovs_vport_stats), &vport_stats,
2132 			  OVS_VPORT_ATTR_PAD))
2133 		goto nla_put_failure;
2134 
2135 	if (ovs_vport_get_upcall_stats(vport, skb))
2136 		goto nla_put_failure;
2137 
2138 	if (ovs_vport_get_upcall_portids(vport, skb))
2139 		goto nla_put_failure;
2140 
2141 	err = ovs_vport_get_options(vport, skb);
2142 	if (err == -EMSGSIZE)
2143 		goto error;
2144 
2145 	genlmsg_end(skb, ovs_header);
2146 	return 0;
2147 
2148 nla_put_failure:
2149 	err = -EMSGSIZE;
2150 error:
2151 	genlmsg_cancel(skb, ovs_header);
2152 	return err;
2153 }
2154 
ovs_vport_cmd_alloc_info(void)2155 static struct sk_buff *ovs_vport_cmd_alloc_info(void)
2156 {
2157 	return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2158 }
2159 
2160 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
ovs_vport_cmd_build_info(struct vport * vport,struct net * net,u32 portid,u32 seq,u8 cmd)2161 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
2162 					 u32 portid, u32 seq, u8 cmd)
2163 {
2164 	struct sk_buff *skb;
2165 	int retval;
2166 
2167 	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2168 	if (!skb)
2169 		return ERR_PTR(-ENOMEM);
2170 
2171 	retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd,
2172 					 GFP_KERNEL);
2173 	BUG_ON(retval < 0);
2174 
2175 	return skb;
2176 }
2177 
2178 /* Called with ovs_mutex or RCU read lock. */
lookup_vport(struct net * net,const struct ovs_header * ovs_header,struct nlattr * a[OVS_VPORT_ATTR_MAX+1])2179 static struct vport *lookup_vport(struct net *net,
2180 				  const struct ovs_header *ovs_header,
2181 				  struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
2182 {
2183 	struct datapath *dp;
2184 	struct vport *vport;
2185 
2186 	if (a[OVS_VPORT_ATTR_IFINDEX])
2187 		return ERR_PTR(-EOPNOTSUPP);
2188 	if (a[OVS_VPORT_ATTR_NAME]) {
2189 		vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
2190 		if (!vport)
2191 			return ERR_PTR(-ENODEV);
2192 		if (ovs_header->dp_ifindex &&
2193 		    ovs_header->dp_ifindex != get_dpifindex(vport->dp))
2194 			return ERR_PTR(-ENODEV);
2195 		return vport;
2196 	} else if (a[OVS_VPORT_ATTR_PORT_NO]) {
2197 		u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
2198 
2199 		if (port_no >= DP_MAX_PORTS)
2200 			return ERR_PTR(-EFBIG);
2201 
2202 		dp = get_dp(net, ovs_header->dp_ifindex);
2203 		if (!dp)
2204 			return ERR_PTR(-ENODEV);
2205 
2206 		vport = ovs_vport_ovsl_rcu(dp, port_no);
2207 		if (!vport)
2208 			return ERR_PTR(-ENODEV);
2209 		return vport;
2210 	} else
2211 		return ERR_PTR(-EINVAL);
2212 
2213 }
2214 
ovs_get_max_headroom(struct datapath * dp)2215 static unsigned int ovs_get_max_headroom(struct datapath *dp)
2216 {
2217 	unsigned int dev_headroom, max_headroom = 0;
2218 	struct net_device *dev;
2219 	struct vport *vport;
2220 	int i;
2221 
2222 	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2223 		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2224 					 lockdep_ovsl_is_held()) {
2225 			dev = vport->dev;
2226 			dev_headroom = netdev_get_fwd_headroom(dev);
2227 			if (dev_headroom > max_headroom)
2228 				max_headroom = dev_headroom;
2229 		}
2230 	}
2231 
2232 	return max_headroom;
2233 }
2234 
2235 /* Called with ovs_mutex */
ovs_update_headroom(struct datapath * dp,unsigned int new_headroom)2236 static void ovs_update_headroom(struct datapath *dp, unsigned int new_headroom)
2237 {
2238 	struct vport *vport;
2239 	int i;
2240 
2241 	dp->max_headroom = new_headroom;
2242 	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2243 		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
2244 					 lockdep_ovsl_is_held())
2245 			netdev_set_rx_headroom(vport->dev, new_headroom);
2246 	}
2247 }
2248 
ovs_vport_cmd_new(struct sk_buff * skb,struct genl_info * info)2249 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
2250 {
2251 	struct nlattr **a = info->attrs;
2252 	struct ovs_header *ovs_header = genl_info_userhdr(info);
2253 	struct vport_parms parms;
2254 	struct sk_buff *reply;
2255 	struct vport *vport;
2256 	struct datapath *dp;
2257 	unsigned int new_headroom;
2258 	u32 port_no;
2259 	int err;
2260 
2261 	if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
2262 	    !a[OVS_VPORT_ATTR_UPCALL_PID])
2263 		return -EINVAL;
2264 
2265 	parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
2266 
2267 	if (a[OVS_VPORT_ATTR_IFINDEX] && parms.type != OVS_VPORT_TYPE_INTERNAL)
2268 		return -EOPNOTSUPP;
2269 
2270 	port_no = a[OVS_VPORT_ATTR_PORT_NO]
2271 		? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
2272 	if (port_no >= DP_MAX_PORTS)
2273 		return -EFBIG;
2274 
2275 	reply = ovs_vport_cmd_alloc_info();
2276 	if (!reply)
2277 		return -ENOMEM;
2278 
2279 	ovs_lock();
2280 restart:
2281 	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
2282 	err = -ENODEV;
2283 	if (!dp)
2284 		goto exit_unlock_free;
2285 
2286 	if (port_no) {
2287 		vport = ovs_vport_ovsl(dp, port_no);
2288 		err = -EBUSY;
2289 		if (vport)
2290 			goto exit_unlock_free;
2291 	} else {
2292 		for (port_no = 1; ; port_no++) {
2293 			if (port_no >= DP_MAX_PORTS) {
2294 				err = -EFBIG;
2295 				goto exit_unlock_free;
2296 			}
2297 			vport = ovs_vport_ovsl(dp, port_no);
2298 			if (!vport)
2299 				break;
2300 		}
2301 	}
2302 
2303 	parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
2304 	parms.options = a[OVS_VPORT_ATTR_OPTIONS];
2305 	parms.dp = dp;
2306 	parms.port_no = port_no;
2307 	parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
2308 	parms.desired_ifindex = a[OVS_VPORT_ATTR_IFINDEX]
2309 		? nla_get_s32(a[OVS_VPORT_ATTR_IFINDEX]) : 0;
2310 
2311 	vport = new_vport(&parms);
2312 	err = PTR_ERR(vport);
2313 	if (IS_ERR(vport)) {
2314 		if (err == -EAGAIN)
2315 			goto restart;
2316 		goto exit_unlock_free;
2317 	}
2318 
2319 	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2320 				      info->snd_portid, info->snd_seq, 0,
2321 				      OVS_VPORT_CMD_NEW, GFP_KERNEL);
2322 
2323 	new_headroom = netdev_get_fwd_headroom(vport->dev);
2324 
2325 	if (new_headroom > dp->max_headroom)
2326 		ovs_update_headroom(dp, new_headroom);
2327 	else
2328 		netdev_set_rx_headroom(vport->dev, dp->max_headroom);
2329 
2330 	BUG_ON(err < 0);
2331 	ovs_unlock();
2332 
2333 	ovs_notify(&dp_vport_genl_family, reply, info);
2334 	return 0;
2335 
2336 exit_unlock_free:
2337 	ovs_unlock();
2338 	kfree_skb(reply);
2339 	return err;
2340 }
2341 
ovs_vport_cmd_set(struct sk_buff * skb,struct genl_info * info)2342 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
2343 {
2344 	struct nlattr **a = info->attrs;
2345 	struct sk_buff *reply;
2346 	struct vport *vport;
2347 	int err;
2348 
2349 	reply = ovs_vport_cmd_alloc_info();
2350 	if (!reply)
2351 		return -ENOMEM;
2352 
2353 	ovs_lock();
2354 	vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a);
2355 	err = PTR_ERR(vport);
2356 	if (IS_ERR(vport))
2357 		goto exit_unlock_free;
2358 
2359 	if (a[OVS_VPORT_ATTR_TYPE] &&
2360 	    nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
2361 		err = -EINVAL;
2362 		goto exit_unlock_free;
2363 	}
2364 
2365 	if (a[OVS_VPORT_ATTR_OPTIONS]) {
2366 		err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
2367 		if (err)
2368 			goto exit_unlock_free;
2369 	}
2370 
2371 
2372 	if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
2373 		struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2374 
2375 		err = ovs_vport_set_upcall_portids(vport, ids);
2376 		if (err)
2377 			goto exit_unlock_free;
2378 	}
2379 
2380 	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2381 				      info->snd_portid, info->snd_seq, 0,
2382 				      OVS_VPORT_CMD_SET, GFP_KERNEL);
2383 	BUG_ON(err < 0);
2384 
2385 	ovs_unlock();
2386 	ovs_notify(&dp_vport_genl_family, reply, info);
2387 	return 0;
2388 
2389 exit_unlock_free:
2390 	ovs_unlock();
2391 	kfree_skb(reply);
2392 	return err;
2393 }
2394 
ovs_vport_cmd_del(struct sk_buff * skb,struct genl_info * info)2395 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
2396 {
2397 	bool update_headroom = false;
2398 	struct nlattr **a = info->attrs;
2399 	struct sk_buff *reply;
2400 	struct datapath *dp;
2401 	struct vport *vport;
2402 	unsigned int new_headroom;
2403 	int err;
2404 
2405 	reply = ovs_vport_cmd_alloc_info();
2406 	if (!reply)
2407 		return -ENOMEM;
2408 
2409 	ovs_lock();
2410 	vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a);
2411 	err = PTR_ERR(vport);
2412 	if (IS_ERR(vport))
2413 		goto exit_unlock_free;
2414 
2415 	if (vport->port_no == OVSP_LOCAL) {
2416 		err = -EINVAL;
2417 		goto exit_unlock_free;
2418 	}
2419 
2420 	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2421 				      info->snd_portid, info->snd_seq, 0,
2422 				      OVS_VPORT_CMD_DEL, GFP_KERNEL);
2423 	BUG_ON(err < 0);
2424 
2425 	/* the vport deletion may trigger dp headroom update */
2426 	dp = vport->dp;
2427 	if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
2428 		update_headroom = true;
2429 
2430 	netdev_reset_rx_headroom(vport->dev);
2431 	ovs_dp_detach_port(vport);
2432 
2433 	if (update_headroom) {
2434 		new_headroom = ovs_get_max_headroom(dp);
2435 
2436 		if (new_headroom < dp->max_headroom)
2437 			ovs_update_headroom(dp, new_headroom);
2438 	}
2439 	ovs_unlock();
2440 
2441 	ovs_notify(&dp_vport_genl_family, reply, info);
2442 	return 0;
2443 
2444 exit_unlock_free:
2445 	ovs_unlock();
2446 	kfree_skb(reply);
2447 	return err;
2448 }
2449 
ovs_vport_cmd_get(struct sk_buff * skb,struct genl_info * info)2450 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
2451 {
2452 	struct nlattr **a = info->attrs;
2453 	struct ovs_header *ovs_header = genl_info_userhdr(info);
2454 	struct sk_buff *reply;
2455 	struct vport *vport;
2456 	int err;
2457 
2458 	reply = ovs_vport_cmd_alloc_info();
2459 	if (!reply)
2460 		return -ENOMEM;
2461 
2462 	rcu_read_lock();
2463 	vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2464 	err = PTR_ERR(vport);
2465 	if (IS_ERR(vport))
2466 		goto exit_unlock_free;
2467 	err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2468 				      info->snd_portid, info->snd_seq, 0,
2469 				      OVS_VPORT_CMD_GET, GFP_ATOMIC);
2470 	BUG_ON(err < 0);
2471 	rcu_read_unlock();
2472 
2473 	return genlmsg_reply(reply, info);
2474 
2475 exit_unlock_free:
2476 	rcu_read_unlock();
2477 	kfree_skb(reply);
2478 	return err;
2479 }
2480 
ovs_vport_cmd_dump(struct sk_buff * skb,struct netlink_callback * cb)2481 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
2482 {
2483 	struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
2484 	struct datapath *dp;
2485 	int bucket = cb->args[0], skip = cb->args[1];
2486 	int i, j = 0;
2487 
2488 	rcu_read_lock();
2489 	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
2490 	if (!dp) {
2491 		rcu_read_unlock();
2492 		return -ENODEV;
2493 	}
2494 	for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2495 		struct vport *vport;
2496 
2497 		j = 0;
2498 		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2499 			if (j >= skip &&
2500 			    ovs_vport_cmd_fill_info(vport, skb,
2501 						    sock_net(skb->sk),
2502 						    NETLINK_CB(cb->skb).portid,
2503 						    cb->nlh->nlmsg_seq,
2504 						    NLM_F_MULTI,
2505 						    OVS_VPORT_CMD_GET,
2506 						    GFP_ATOMIC) < 0)
2507 				goto out;
2508 
2509 			j++;
2510 		}
2511 		skip = 0;
2512 	}
2513 out:
2514 	rcu_read_unlock();
2515 
2516 	cb->args[0] = i;
2517 	cb->args[1] = j;
2518 
2519 	return skb->len;
2520 }
2521 
ovs_dp_masks_rebalance(struct work_struct * work)2522 static void ovs_dp_masks_rebalance(struct work_struct *work)
2523 {
2524 	struct ovs_net *ovs_net = container_of(work, struct ovs_net,
2525 					       masks_rebalance.work);
2526 	struct datapath *dp;
2527 
2528 	ovs_lock();
2529 
2530 	list_for_each_entry(dp, &ovs_net->dps, list_node)
2531 		ovs_flow_masks_rebalance(&dp->table);
2532 
2533 	ovs_unlock();
2534 
2535 	schedule_delayed_work(&ovs_net->masks_rebalance,
2536 			      msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2537 }
2538 
2539 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2540 	[OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2541 	[OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2542 	[OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2543 	[OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2544 	[OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC },
2545 	[OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
2546 	[OVS_VPORT_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0),
2547 	[OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
2548 	[OVS_VPORT_ATTR_UPCALL_STATS] = { .type = NLA_NESTED },
2549 };
2550 
2551 static const struct genl_small_ops dp_vport_genl_ops[] = {
2552 	{ .cmd = OVS_VPORT_CMD_NEW,
2553 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2554 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2555 	  .doit = ovs_vport_cmd_new
2556 	},
2557 	{ .cmd = OVS_VPORT_CMD_DEL,
2558 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2559 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2560 	  .doit = ovs_vport_cmd_del
2561 	},
2562 	{ .cmd = OVS_VPORT_CMD_GET,
2563 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2564 	  .flags = 0,		    /* OK for unprivileged users. */
2565 	  .doit = ovs_vport_cmd_get,
2566 	  .dumpit = ovs_vport_cmd_dump
2567 	},
2568 	{ .cmd = OVS_VPORT_CMD_SET,
2569 	  .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
2570 	  .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
2571 	  .doit = ovs_vport_cmd_set,
2572 	},
2573 };
2574 
2575 struct genl_family dp_vport_genl_family __ro_after_init = {
2576 	.hdrsize = sizeof(struct ovs_header),
2577 	.name = OVS_VPORT_FAMILY,
2578 	.version = OVS_VPORT_VERSION,
2579 	.maxattr = OVS_VPORT_ATTR_MAX,
2580 	.policy = vport_policy,
2581 	.netnsok = true,
2582 	.parallel_ops = true,
2583 	.small_ops = dp_vport_genl_ops,
2584 	.n_small_ops = ARRAY_SIZE(dp_vport_genl_ops),
2585 	.resv_start_op = OVS_VPORT_CMD_SET + 1,
2586 	.mcgrps = &ovs_dp_vport_multicast_group,
2587 	.n_mcgrps = 1,
2588 	.module = THIS_MODULE,
2589 };
2590 
2591 static struct genl_family * const dp_genl_families[] = {
2592 	&dp_datapath_genl_family,
2593 	&dp_vport_genl_family,
2594 	&dp_flow_genl_family,
2595 	&dp_packet_genl_family,
2596 	&dp_meter_genl_family,
2597 #if	IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
2598 	&dp_ct_limit_genl_family,
2599 #endif
2600 };
2601 
dp_unregister_genl(int n_families)2602 static void dp_unregister_genl(int n_families)
2603 {
2604 	int i;
2605 
2606 	for (i = 0; i < n_families; i++)
2607 		genl_unregister_family(dp_genl_families[i]);
2608 }
2609 
dp_register_genl(void)2610 static int __init dp_register_genl(void)
2611 {
2612 	int err;
2613 	int i;
2614 
2615 	for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
2616 
2617 		err = genl_register_family(dp_genl_families[i]);
2618 		if (err)
2619 			goto error;
2620 	}
2621 
2622 	return 0;
2623 
2624 error:
2625 	dp_unregister_genl(i);
2626 	return err;
2627 }
2628 
ovs_init_net(struct net * net)2629 static int __net_init ovs_init_net(struct net *net)
2630 {
2631 	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2632 	int err;
2633 
2634 	INIT_LIST_HEAD(&ovs_net->dps);
2635 	INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2636 	INIT_DELAYED_WORK(&ovs_net->masks_rebalance, ovs_dp_masks_rebalance);
2637 
2638 	err = ovs_ct_init(net);
2639 	if (err)
2640 		return err;
2641 
2642 	schedule_delayed_work(&ovs_net->masks_rebalance,
2643 			      msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
2644 	return 0;
2645 }
2646 
list_vports_from_net(struct net * net,struct net * dnet,struct list_head * head)2647 static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2648 					    struct list_head *head)
2649 {
2650 	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2651 	struct datapath *dp;
2652 
2653 	list_for_each_entry(dp, &ovs_net->dps, list_node) {
2654 		int i;
2655 
2656 		for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2657 			struct vport *vport;
2658 
2659 			hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
2660 				if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2661 					continue;
2662 
2663 				if (dev_net(vport->dev) == dnet)
2664 					list_add(&vport->detach_list, head);
2665 			}
2666 		}
2667 	}
2668 }
2669 
ovs_exit_net(struct net * dnet)2670 static void __net_exit ovs_exit_net(struct net *dnet)
2671 {
2672 	struct datapath *dp, *dp_next;
2673 	struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2674 	struct vport *vport, *vport_next;
2675 	struct net *net;
2676 	LIST_HEAD(head);
2677 
2678 	ovs_lock();
2679 
2680 	ovs_ct_exit(dnet);
2681 
2682 	list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2683 		__dp_destroy(dp);
2684 
2685 	down_read(&net_rwsem);
2686 	for_each_net(net)
2687 		list_vports_from_net(net, dnet, &head);
2688 	up_read(&net_rwsem);
2689 
2690 	/* Detach all vports from given namespace. */
2691 	list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2692 		list_del(&vport->detach_list);
2693 		ovs_dp_detach_port(vport);
2694 	}
2695 
2696 	ovs_unlock();
2697 
2698 	cancel_delayed_work_sync(&ovs_net->masks_rebalance);
2699 	cancel_work_sync(&ovs_net->dp_notify_work);
2700 }
2701 
2702 static struct pernet_operations ovs_net_ops = {
2703 	.init = ovs_init_net,
2704 	.exit = ovs_exit_net,
2705 	.id   = &ovs_net_id,
2706 	.size = sizeof(struct ovs_net),
2707 };
2708 
2709 static const char * const ovs_drop_reasons[] = {
2710 #define S(x) [(x) & ~SKB_DROP_REASON_SUBSYS_MASK] = (#x),
2711 	OVS_DROP_REASONS(S)
2712 #undef S
2713 };
2714 
2715 static struct drop_reason_list drop_reason_list_ovs = {
2716 	.reasons = ovs_drop_reasons,
2717 	.n_reasons = ARRAY_SIZE(ovs_drop_reasons),
2718 };
2719 
dp_init(void)2720 static int __init dp_init(void)
2721 {
2722 	int err;
2723 
2724 	BUILD_BUG_ON(sizeof(struct ovs_skb_cb) >
2725 		     sizeof_field(struct sk_buff, cb));
2726 
2727 	pr_info("Open vSwitch switching datapath\n");
2728 
2729 	err = action_fifos_init();
2730 	if (err)
2731 		goto error;
2732 
2733 	err = ovs_internal_dev_rtnl_link_register();
2734 	if (err)
2735 		goto error_action_fifos_exit;
2736 
2737 	err = ovs_flow_init();
2738 	if (err)
2739 		goto error_unreg_rtnl_link;
2740 
2741 	err = ovs_vport_init();
2742 	if (err)
2743 		goto error_flow_exit;
2744 
2745 	err = register_pernet_device(&ovs_net_ops);
2746 	if (err)
2747 		goto error_vport_exit;
2748 
2749 	err = register_netdevice_notifier(&ovs_dp_device_notifier);
2750 	if (err)
2751 		goto error_netns_exit;
2752 
2753 	err = ovs_netdev_init();
2754 	if (err)
2755 		goto error_unreg_notifier;
2756 
2757 	err = dp_register_genl();
2758 	if (err < 0)
2759 		goto error_unreg_netdev;
2760 
2761 	drop_reasons_register_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH,
2762 				     &drop_reason_list_ovs);
2763 
2764 	return 0;
2765 
2766 error_unreg_netdev:
2767 	ovs_netdev_exit();
2768 error_unreg_notifier:
2769 	unregister_netdevice_notifier(&ovs_dp_device_notifier);
2770 error_netns_exit:
2771 	unregister_pernet_device(&ovs_net_ops);
2772 error_vport_exit:
2773 	ovs_vport_exit();
2774 error_flow_exit:
2775 	ovs_flow_exit();
2776 error_unreg_rtnl_link:
2777 	ovs_internal_dev_rtnl_link_unregister();
2778 error_action_fifos_exit:
2779 	action_fifos_exit();
2780 error:
2781 	return err;
2782 }
2783 
dp_cleanup(void)2784 static void dp_cleanup(void)
2785 {
2786 	dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2787 	ovs_netdev_exit();
2788 	unregister_netdevice_notifier(&ovs_dp_device_notifier);
2789 	unregister_pernet_device(&ovs_net_ops);
2790 	drop_reasons_unregister_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH);
2791 	rcu_barrier();
2792 	ovs_vport_exit();
2793 	ovs_flow_exit();
2794 	ovs_internal_dev_rtnl_link_unregister();
2795 	action_fifos_exit();
2796 }
2797 
2798 module_init(dp_init);
2799 module_exit(dp_cleanup);
2800 
2801 MODULE_DESCRIPTION("Open vSwitch switching datapath");
2802 MODULE_LICENSE("GPL");
2803 MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
2804 MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
2805 MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
2806 MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
2807 MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
2808 MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);
2809