111 udp_tunnel_nic_entry_queue(struct udp_tunnel_nic *utn,  in udp_tunnel_nic_entry_queue()  argument
116 	utn->need_sync = 1;  in udp_tunnel_nic_entry_queue()
130 udp_tunnel_nic_is_empty(struct net_device *dev, struct udp_tunnel_nic *utn)  in udp_tunnel_nic_is_empty()  argument
135 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_is_empty()
137 			if (!udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))  in udp_tunnel_nic_is_empty()
143 udp_tunnel_nic_should_replay(struct net_device *dev, struct udp_tunnel_nic *utn)  in udp_tunnel_nic_should_replay()  argument
148 	if (!utn->missed)  in udp_tunnel_nic_should_replay()
151 	for (i = 0; i < utn->n_tables; i++) {  in udp_tunnel_nic_should_replay()
153 		if (!test_bit(i, &utn->missed))  in udp_tunnel_nic_should_replay()
157 			if (udp_tunnel_nic_entry_is_free(&utn->entries[i][j]))  in udp_tunnel_nic_should_replay()
169 	struct udp_tunnel_nic *utn;  in __udp_tunnel_nic_get_port()  local
171 	utn = dev->udp_tunnel_nic;  in __udp_tunnel_nic_get_port()
172 	entry = &utn->entries[table][idx];  in __udp_tunnel_nic_get_port()
210 			       struct udp_tunnel_nic *utn,  in udp_tunnel_nic_device_sync_one()  argument
217 	entry = &utn->entries[table][idx];  in udp_tunnel_nic_device_sync_one()
239 				   struct udp_tunnel_nic *utn)  in udp_tunnel_nic_device_sync_by_port()  argument
244 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_device_sync_by_port()
246 			udp_tunnel_nic_device_sync_one(dev, utn, i, j);  in udp_tunnel_nic_device_sync_by_port()
251 				    struct udp_tunnel_nic *utn)  in udp_tunnel_nic_device_sync_by_table()  argument
257 	for (i = 0; i < utn->n_tables; i++) {  in udp_tunnel_nic_device_sync_by_table()
260 			if (udp_tunnel_nic_entry_is_queued(&utn->entries[i][j]))  in udp_tunnel_nic_device_sync_by_table()
273 			entry = &utn->entries[i][j];  in udp_tunnel_nic_device_sync_by_table()
281 __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)  in __udp_tunnel_nic_device_sync()  argument
283 	if (!utn->need_sync)  in __udp_tunnel_nic_device_sync()
287 		udp_tunnel_nic_device_sync_by_table(dev, utn);  in __udp_tunnel_nic_device_sync()
289 		udp_tunnel_nic_device_sync_by_port(dev, utn);  in __udp_tunnel_nic_device_sync()
291 	utn->need_sync = 0;  in __udp_tunnel_nic_device_sync()
295 	utn->need_replay = udp_tunnel_nic_should_replay(dev, utn);  in __udp_tunnel_nic_device_sync()
299 udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn)  in udp_tunnel_nic_device_sync()  argument
304 	if (!utn->need_sync)  in udp_tunnel_nic_device_sync()
312 		__udp_tunnel_nic_device_sync(dev, utn);  in udp_tunnel_nic_device_sync()
313 	if (may_sleep || utn->need_replay) {  in udp_tunnel_nic_device_sync()
314 		queue_work(udp_tunnel_nic_workqueue, &utn->work);  in udp_tunnel_nic_device_sync()
315 		utn->work_pending = 1;  in udp_tunnel_nic_device_sync()
327 udp_tunnel_nic_is_capable(struct net_device *dev, struct udp_tunnel_nic *utn,  in udp_tunnel_nic_is_capable()  argument
338 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_is_capable()
345 udp_tunnel_nic_has_collision(struct net_device *dev, struct udp_tunnel_nic *utn,  in udp_tunnel_nic_has_collision()  argument
352 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_has_collision()
354 			entry =	&utn->entries[i][j];  in udp_tunnel_nic_has_collision()
359 				__set_bit(i, &utn->missed);  in udp_tunnel_nic_has_collision()
367 udp_tunnel_nic_entry_adj(struct udp_tunnel_nic *utn,  in udp_tunnel_nic_entry_adj()  argument
370 	struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];  in udp_tunnel_nic_entry_adj()
401 	udp_tunnel_nic_entry_queue(utn, entry, to);  in udp_tunnel_nic_entry_adj()
405 udp_tunnel_nic_entry_try_adj(struct udp_tunnel_nic *utn,  in udp_tunnel_nic_entry_try_adj()  argument
409 	struct udp_tunnel_nic_table_entry *entry =  &utn->entries[table][idx];  in udp_tunnel_nic_entry_try_adj()
419 	udp_tunnel_nic_entry_adj(utn, table, idx, use_cnt_adj);  in udp_tunnel_nic_entry_try_adj()
429 udp_tunnel_nic_try_existing(struct net_device *dev, struct udp_tunnel_nic *utn,  in udp_tunnel_nic_try_existing()  argument
435 	for (i = 0; i < utn->n_tables; i++) {  in udp_tunnel_nic_try_existing()
441 			if (udp_tunnel_nic_entry_try_adj(utn, i, j, ti,  in udp_tunnel_nic_try_existing()
450 udp_tunnel_nic_add_existing(struct net_device *dev, struct udp_tunnel_nic *utn,  in udp_tunnel_nic_add_existing()  argument
453 	return udp_tunnel_nic_try_existing(dev, utn, ti, +1);  in udp_tunnel_nic_add_existing()
457 udp_tunnel_nic_del_existing(struct net_device *dev, struct udp_tunnel_nic *utn,  in udp_tunnel_nic_del_existing()  argument
460 	return udp_tunnel_nic_try_existing(dev, utn, ti, -1);  in udp_tunnel_nic_del_existing()
464 udp_tunnel_nic_add_new(struct net_device *dev, struct udp_tunnel_nic *utn,  in udp_tunnel_nic_add_new()  argument
470 	for (i = 0; i < utn->n_tables; i++) {  in udp_tunnel_nic_add_new()
478 			entry = &utn->entries[i][j];  in udp_tunnel_nic_add_new()
485 			udp_tunnel_nic_entry_queue(utn, entry,  in udp_tunnel_nic_add_new()
494 		__set_bit(i, &utn->missed);  in udp_tunnel_nic_add_new()
504 	struct udp_tunnel_nic *utn;  in __udp_tunnel_nic_add_port()  local
506 	utn = dev->udp_tunnel_nic;  in __udp_tunnel_nic_add_port()
507 	if (!utn)  in __udp_tunnel_nic_add_port()
518 	if (!udp_tunnel_nic_is_capable(dev, utn, ti))  in __udp_tunnel_nic_add_port()
525 	if (udp_tunnel_nic_has_collision(dev, utn, ti))  in __udp_tunnel_nic_add_port()
528 	if (!udp_tunnel_nic_add_existing(dev, utn, ti))  in __udp_tunnel_nic_add_port()
529 		udp_tunnel_nic_add_new(dev, utn, ti);  in __udp_tunnel_nic_add_port()
531 	udp_tunnel_nic_device_sync(dev, utn);  in __udp_tunnel_nic_add_port()
537 	struct udp_tunnel_nic *utn;  in __udp_tunnel_nic_del_port()  local
539 	utn = dev->udp_tunnel_nic;  in __udp_tunnel_nic_del_port()
540 	if (!utn)  in __udp_tunnel_nic_del_port()
543 	if (!udp_tunnel_nic_is_capable(dev, utn, ti))  in __udp_tunnel_nic_del_port()
546 	udp_tunnel_nic_del_existing(dev, utn, ti);  in __udp_tunnel_nic_del_port()
548 	udp_tunnel_nic_device_sync(dev, utn);  in __udp_tunnel_nic_del_port()
554 	struct udp_tunnel_nic *utn;  in __udp_tunnel_nic_reset_ntf()  local
559 	utn = dev->udp_tunnel_nic;  in __udp_tunnel_nic_reset_ntf()
560 	if (!utn)  in __udp_tunnel_nic_reset_ntf()
563 	utn->need_sync = false;  in __udp_tunnel_nic_reset_ntf()
564 	for (i = 0; i < utn->n_tables; i++)  in __udp_tunnel_nic_reset_ntf()
568 			entry = &utn->entries[i][j];  in __udp_tunnel_nic_reset_ntf()
577 			udp_tunnel_nic_entry_queue(utn, entry,  in __udp_tunnel_nic_reset_ntf()
581 	__udp_tunnel_nic_device_sync(dev, utn);  in __udp_tunnel_nic_reset_ntf()
588 	struct udp_tunnel_nic *utn;  in __udp_tunnel_nic_dump_size()  local
592 	utn = dev->udp_tunnel_nic;  in __udp_tunnel_nic_dump_size()
593 	if (!utn)  in __udp_tunnel_nic_dump_size()
598 		if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))  in __udp_tunnel_nic_dump_size()
614 	struct udp_tunnel_nic *utn;  in __udp_tunnel_nic_dump_write()  local
618 	utn = dev->udp_tunnel_nic;  in __udp_tunnel_nic_dump_write()
619 	if (!utn)  in __udp_tunnel_nic_dump_write()
623 		if (!udp_tunnel_nic_entry_is_present(&utn->entries[table][j]))  in __udp_tunnel_nic_dump_write()
631 				 utn->entries[table][j].port) ||  in __udp_tunnel_nic_dump_write()
633 				ilog2(utn->entries[table][j].type)))  in __udp_tunnel_nic_dump_write()
657 udp_tunnel_nic_flush(struct net_device *dev, struct udp_tunnel_nic *utn)  in udp_tunnel_nic_flush()  argument
662 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_flush()
664 			int adj_cnt = -utn->entries[i][j].use_cnt;  in udp_tunnel_nic_flush()
667 				udp_tunnel_nic_entry_adj(utn, i, j, adj_cnt);  in udp_tunnel_nic_flush()
670 	__udp_tunnel_nic_device_sync(dev, utn);  in udp_tunnel_nic_flush()
672 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_flush()
673 		memset(utn->entries[i], 0, array_size(info->tables[i].n_entries,  in udp_tunnel_nic_flush()
674 						      sizeof(**utn->entries)));  in udp_tunnel_nic_flush()
675 	WARN_ON(utn->need_sync);  in udp_tunnel_nic_flush()
676 	utn->need_replay = 0;  in udp_tunnel_nic_flush()
680 udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn)  in udp_tunnel_nic_replay()  argument
689 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_replay()
691 			udp_tunnel_nic_entry_freeze_used(&utn->entries[i][j]);  in udp_tunnel_nic_replay()
692 	utn->missed = 0;  in udp_tunnel_nic_replay()
693 	utn->need_replay = 0;  in udp_tunnel_nic_replay()
702 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_replay()
704 			udp_tunnel_nic_entry_unfreeze(&utn->entries[i][j]);  in udp_tunnel_nic_replay()
709 	struct udp_tunnel_nic *utn =  in udp_tunnel_nic_device_sync_work()  local
713 	utn->work_pending = 0;  in udp_tunnel_nic_device_sync_work()
714 	__udp_tunnel_nic_device_sync(utn->dev, utn);  in udp_tunnel_nic_device_sync_work()
716 	if (utn->need_replay)  in udp_tunnel_nic_device_sync_work()
717 		udp_tunnel_nic_replay(utn->dev, utn);  in udp_tunnel_nic_device_sync_work()
725 	struct udp_tunnel_nic *utn;  in udp_tunnel_nic_alloc()  local
728 	utn = kzalloc(sizeof(*utn), GFP_KERNEL);  in udp_tunnel_nic_alloc()
729 	if (!utn)  in udp_tunnel_nic_alloc()
731 	utn->n_tables = n_tables;  in udp_tunnel_nic_alloc()
732 	INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work);  in udp_tunnel_nic_alloc()
734 	utn->entries = kmalloc_array(n_tables, sizeof(void *), GFP_KERNEL);  in udp_tunnel_nic_alloc()
735 	if (!utn->entries)  in udp_tunnel_nic_alloc()
739 		utn->entries[i] = kcalloc(info->tables[i].n_entries,  in udp_tunnel_nic_alloc()
740 					  sizeof(*utn->entries[i]), GFP_KERNEL);  in udp_tunnel_nic_alloc()
741 		if (!utn->entries[i])  in udp_tunnel_nic_alloc()
745 	return utn;  in udp_tunnel_nic_alloc()
749 		kfree(utn->entries[i]);  in udp_tunnel_nic_alloc()
750 	kfree(utn->entries);  in udp_tunnel_nic_alloc()
752 	kfree(utn);  in udp_tunnel_nic_alloc()
756 static void udp_tunnel_nic_free(struct udp_tunnel_nic *utn)  in udp_tunnel_nic_free()  argument
760 	for (i = 0; i < utn->n_tables; i++)  in udp_tunnel_nic_free()
761 		kfree(utn->entries[i]);  in udp_tunnel_nic_free()
762 	kfree(utn->entries);  in udp_tunnel_nic_free()
763 	kfree(utn);  in udp_tunnel_nic_free()
770 	struct udp_tunnel_nic *utn;  in udp_tunnel_nic_register()  local
773 	BUILD_BUG_ON(sizeof(utn->missed) * BITS_PER_BYTE <  in udp_tunnel_nic_register()
809 		utn = info->shared->udp_tunnel_nic_info;  in udp_tunnel_nic_register()
811 		utn = udp_tunnel_nic_alloc(info, n_tables);  in udp_tunnel_nic_register()
812 		if (!utn) {  in udp_tunnel_nic_register()
821 			info->shared->udp_tunnel_nic_info = utn;  in udp_tunnel_nic_register()
827 	utn->dev = dev;  in udp_tunnel_nic_register()
829 	dev->udp_tunnel_nic = utn;  in udp_tunnel_nic_register()
838 udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn)  in udp_tunnel_nic_unregister()  argument
861 			utn->dev = first->dev;  in udp_tunnel_nic_unregister()
871 	udp_tunnel_nic_flush(dev, utn);  in udp_tunnel_nic_unregister()
876 	if (utn->work_pending)  in udp_tunnel_nic_unregister()
879 	udp_tunnel_nic_free(utn);  in udp_tunnel_nic_unregister()
891 	struct udp_tunnel_nic *utn;  in udp_tunnel_nic_netdevice_event()  local
906 	utn = dev->udp_tunnel_nic;  in udp_tunnel_nic_netdevice_event()
907 	if (!utn)  in udp_tunnel_nic_netdevice_event()
911 		udp_tunnel_nic_unregister(dev, utn);  in udp_tunnel_nic_netdevice_event()
920 		WARN_ON(!udp_tunnel_nic_is_empty(dev, utn));  in udp_tunnel_nic_netdevice_event()
925 		udp_tunnel_nic_flush(dev, utn);  in udp_tunnel_nic_netdevice_event()