xref: /openbmc/linux/net/sched/sch_sfq.c (revision 9c1f8594)
1 /*
2  * net/sched/sch_sfq.c	Stochastic Fairness Queueing discipline.
3  *
4  *		This program is free software; you can redistribute it and/or
5  *		modify it under the terms of the GNU General Public License
6  *		as published by the Free Software Foundation; either version
7  *		2 of the License, or (at your option) any later version.
8  *
9  * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10  */
11 
12 #include <linux/module.h>
13 #include <linux/types.h>
14 #include <linux/kernel.h>
15 #include <linux/jiffies.h>
16 #include <linux/string.h>
17 #include <linux/in.h>
18 #include <linux/errno.h>
19 #include <linux/init.h>
20 #include <linux/ipv6.h>
21 #include <linux/skbuff.h>
22 #include <linux/jhash.h>
23 #include <linux/slab.h>
24 #include <linux/vmalloc.h>
25 #include <net/ip.h>
26 #include <net/netlink.h>
27 #include <net/pkt_sched.h>
28 
29 
30 /*	Stochastic Fairness Queuing algorithm.
31 	=======================================
32 
33 	Source:
34 	Paul E. McKenney "Stochastic Fairness Queuing",
35 	IEEE INFOCOMM'90 Proceedings, San Francisco, 1990.
36 
37 	Paul E. McKenney "Stochastic Fairness Queuing",
38 	"Interworking: Research and Experience", v.2, 1991, p.113-131.
39 
40 
41 	See also:
42 	M. Shreedhar and George Varghese "Efficient Fair
43 	Queuing using Deficit Round Robin", Proc. SIGCOMM 95.
44 
45 
46 	This is not the thing that is usually called (W)FQ nowadays.
47 	It does not use any timestamp mechanism, but instead
48 	processes queues in round-robin order.
49 
50 	ADVANTAGE:
51 
52 	- It is very cheap. Both CPU and memory requirements are minimal.
53 
54 	DRAWBACKS:
55 
56 	- "Stochastic" -> It is not 100% fair.
57 	When hash collisions occur, several flows are considered as one.
58 
59 	- "Round-robin" -> It introduces larger delays than virtual clock
60 	based schemes, and should not be used for isolating interactive
61 	traffic	from non-interactive. It means, that this scheduler
62 	should be used as leaf of CBQ or P3, which put interactive traffic
63 	to higher priority band.
64 
65 	We still need true WFQ for top level CSZ, but using WFQ
66 	for the best effort traffic is absolutely pointless:
67 	SFQ is superior for this purpose.
68 
69 	IMPLEMENTATION:
70 	This implementation limits maximal queue length to 128;
71 	max mtu to 2^18-1; max 128 flows, number of hash buckets to 1024.
72 	The only goal of this restrictions was that all data
73 	fit into one 4K page on 32bit arches.
74 
75 	It is easy to increase these values, but not in flight.  */
76 
77 #define SFQ_DEPTH		128 /* max number of packets per flow */
78 #define SFQ_SLOTS		128 /* max number of flows */
79 #define SFQ_EMPTY_SLOT		255
80 #define SFQ_DEFAULT_HASH_DIVISOR 1024
81 
82 /* We use 16 bits to store allot, and want to handle packets up to 64K
83  * Scale allot by 8 (1<<3) so that no overflow occurs.
84  */
85 #define SFQ_ALLOT_SHIFT		3
86 #define SFQ_ALLOT_SIZE(X)	DIV_ROUND_UP(X, 1 << SFQ_ALLOT_SHIFT)
87 
88 /* This type should contain at least SFQ_DEPTH + SFQ_SLOTS values */
89 typedef unsigned char sfq_index;
90 
91 /*
92  * We dont use pointers to save space.
93  * Small indexes [0 ... SFQ_SLOTS - 1] are 'pointers' to slots[] array
94  * while following values [SFQ_SLOTS ... SFQ_SLOTS + SFQ_DEPTH - 1]
95  * are 'pointers' to dep[] array
96  */
97 struct sfq_head {
98 	sfq_index	next;
99 	sfq_index	prev;
100 };
101 
102 struct sfq_slot {
103 	struct sk_buff	*skblist_next;
104 	struct sk_buff	*skblist_prev;
105 	sfq_index	qlen; /* number of skbs in skblist */
106 	sfq_index	next; /* next slot in sfq chain */
107 	struct sfq_head dep; /* anchor in dep[] chains */
108 	unsigned short	hash; /* hash value (index in ht[]) */
109 	short		allot; /* credit for this slot */
110 };
111 
112 struct sfq_sched_data {
113 /* Parameters */
114 	int		perturb_period;
115 	unsigned int	quantum;	/* Allotment per round: MUST BE >= MTU */
116 	int		limit;
117 	unsigned int	divisor;	/* number of slots in hash table */
118 /* Variables */
119 	struct tcf_proto *filter_list;
120 	struct timer_list perturb_timer;
121 	u32		perturbation;
122 	sfq_index	cur_depth;	/* depth of longest slot */
123 	unsigned short  scaled_quantum; /* SFQ_ALLOT_SIZE(quantum) */
124 	struct sfq_slot *tail;		/* current slot in round */
125 	sfq_index	*ht;		/* Hash table (divisor slots) */
126 	struct sfq_slot	slots[SFQ_SLOTS];
127 	struct sfq_head	dep[SFQ_DEPTH];	/* Linked list of slots, indexed by depth */
128 };
129 
130 /*
131  * sfq_head are either in a sfq_slot or in dep[] array
132  */
133 static inline struct sfq_head *sfq_dep_head(struct sfq_sched_data *q, sfq_index val)
134 {
135 	if (val < SFQ_SLOTS)
136 		return &q->slots[val].dep;
137 	return &q->dep[val - SFQ_SLOTS];
138 }
139 
140 static unsigned int sfq_fold_hash(struct sfq_sched_data *q, u32 h, u32 h1)
141 {
142 	return jhash_2words(h, h1, q->perturbation) & (q->divisor - 1);
143 }
144 
145 static unsigned int sfq_hash(struct sfq_sched_data *q, struct sk_buff *skb)
146 {
147 	u32 h, h2;
148 
149 	switch (skb->protocol) {
150 	case htons(ETH_P_IP):
151 	{
152 		const struct iphdr *iph;
153 		int poff;
154 
155 		if (!pskb_network_may_pull(skb, sizeof(*iph)))
156 			goto err;
157 		iph = ip_hdr(skb);
158 		h = (__force u32)iph->daddr;
159 		h2 = (__force u32)iph->saddr ^ iph->protocol;
160 		if (ip_is_fragment(iph))
161 			break;
162 		poff = proto_ports_offset(iph->protocol);
163 		if (poff >= 0 &&
164 		    pskb_network_may_pull(skb, iph->ihl * 4 + 4 + poff)) {
165 			iph = ip_hdr(skb);
166 			h2 ^= *(u32 *)((void *)iph + iph->ihl * 4 + poff);
167 		}
168 		break;
169 	}
170 	case htons(ETH_P_IPV6):
171 	{
172 		const struct ipv6hdr *iph;
173 		int poff;
174 
175 		if (!pskb_network_may_pull(skb, sizeof(*iph)))
176 			goto err;
177 		iph = ipv6_hdr(skb);
178 		h = (__force u32)iph->daddr.s6_addr32[3];
179 		h2 = (__force u32)iph->saddr.s6_addr32[3] ^ iph->nexthdr;
180 		poff = proto_ports_offset(iph->nexthdr);
181 		if (poff >= 0 &&
182 		    pskb_network_may_pull(skb, sizeof(*iph) + 4 + poff)) {
183 			iph = ipv6_hdr(skb);
184 			h2 ^= *(u32 *)((void *)iph + sizeof(*iph) + poff);
185 		}
186 		break;
187 	}
188 	default:
189 err:
190 		h = (unsigned long)skb_dst(skb) ^ (__force u32)skb->protocol;
191 		h2 = (unsigned long)skb->sk;
192 	}
193 
194 	return sfq_fold_hash(q, h, h2);
195 }
196 
197 static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
198 				 int *qerr)
199 {
200 	struct sfq_sched_data *q = qdisc_priv(sch);
201 	struct tcf_result res;
202 	int result;
203 
204 	if (TC_H_MAJ(skb->priority) == sch->handle &&
205 	    TC_H_MIN(skb->priority) > 0 &&
206 	    TC_H_MIN(skb->priority) <= q->divisor)
207 		return TC_H_MIN(skb->priority);
208 
209 	if (!q->filter_list)
210 		return sfq_hash(q, skb) + 1;
211 
212 	*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
213 	result = tc_classify(skb, q->filter_list, &res);
214 	if (result >= 0) {
215 #ifdef CONFIG_NET_CLS_ACT
216 		switch (result) {
217 		case TC_ACT_STOLEN:
218 		case TC_ACT_QUEUED:
219 			*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
220 		case TC_ACT_SHOT:
221 			return 0;
222 		}
223 #endif
224 		if (TC_H_MIN(res.classid) <= q->divisor)
225 			return TC_H_MIN(res.classid);
226 	}
227 	return 0;
228 }
229 
230 /*
231  * x : slot number [0 .. SFQ_SLOTS - 1]
232  */
233 static inline void sfq_link(struct sfq_sched_data *q, sfq_index x)
234 {
235 	sfq_index p, n;
236 	int qlen = q->slots[x].qlen;
237 
238 	p = qlen + SFQ_SLOTS;
239 	n = q->dep[qlen].next;
240 
241 	q->slots[x].dep.next = n;
242 	q->slots[x].dep.prev = p;
243 
244 	q->dep[qlen].next = x;		/* sfq_dep_head(q, p)->next = x */
245 	sfq_dep_head(q, n)->prev = x;
246 }
247 
248 #define sfq_unlink(q, x, n, p)			\
249 	n = q->slots[x].dep.next;		\
250 	p = q->slots[x].dep.prev;		\
251 	sfq_dep_head(q, p)->next = n;		\
252 	sfq_dep_head(q, n)->prev = p
253 
254 
255 static inline void sfq_dec(struct sfq_sched_data *q, sfq_index x)
256 {
257 	sfq_index p, n;
258 	int d;
259 
260 	sfq_unlink(q, x, n, p);
261 
262 	d = q->slots[x].qlen--;
263 	if (n == p && q->cur_depth == d)
264 		q->cur_depth--;
265 	sfq_link(q, x);
266 }
267 
268 static inline void sfq_inc(struct sfq_sched_data *q, sfq_index x)
269 {
270 	sfq_index p, n;
271 	int d;
272 
273 	sfq_unlink(q, x, n, p);
274 
275 	d = ++q->slots[x].qlen;
276 	if (q->cur_depth < d)
277 		q->cur_depth = d;
278 	sfq_link(q, x);
279 }
280 
281 /* helper functions : might be changed when/if skb use a standard list_head */
282 
283 /* remove one skb from tail of slot queue */
284 static inline struct sk_buff *slot_dequeue_tail(struct sfq_slot *slot)
285 {
286 	struct sk_buff *skb = slot->skblist_prev;
287 
288 	slot->skblist_prev = skb->prev;
289 	skb->prev->next = (struct sk_buff *)slot;
290 	skb->next = skb->prev = NULL;
291 	return skb;
292 }
293 
294 /* remove one skb from head of slot queue */
295 static inline struct sk_buff *slot_dequeue_head(struct sfq_slot *slot)
296 {
297 	struct sk_buff *skb = slot->skblist_next;
298 
299 	slot->skblist_next = skb->next;
300 	skb->next->prev = (struct sk_buff *)slot;
301 	skb->next = skb->prev = NULL;
302 	return skb;
303 }
304 
305 static inline void slot_queue_init(struct sfq_slot *slot)
306 {
307 	slot->skblist_prev = slot->skblist_next = (struct sk_buff *)slot;
308 }
309 
310 /* add skb to slot queue (tail add) */
311 static inline void slot_queue_add(struct sfq_slot *slot, struct sk_buff *skb)
312 {
313 	skb->prev = slot->skblist_prev;
314 	skb->next = (struct sk_buff *)slot;
315 	slot->skblist_prev->next = skb;
316 	slot->skblist_prev = skb;
317 }
318 
319 #define	slot_queue_walk(slot, skb)		\
320 	for (skb = slot->skblist_next;		\
321 	     skb != (struct sk_buff *)slot;	\
322 	     skb = skb->next)
323 
324 static unsigned int sfq_drop(struct Qdisc *sch)
325 {
326 	struct sfq_sched_data *q = qdisc_priv(sch);
327 	sfq_index x, d = q->cur_depth;
328 	struct sk_buff *skb;
329 	unsigned int len;
330 	struct sfq_slot *slot;
331 
332 	/* Queue is full! Find the longest slot and drop tail packet from it */
333 	if (d > 1) {
334 		x = q->dep[d].next;
335 		slot = &q->slots[x];
336 drop:
337 		skb = slot_dequeue_tail(slot);
338 		len = qdisc_pkt_len(skb);
339 		sfq_dec(q, x);
340 		kfree_skb(skb);
341 		sch->q.qlen--;
342 		sch->qstats.drops++;
343 		sch->qstats.backlog -= len;
344 		return len;
345 	}
346 
347 	if (d == 1) {
348 		/* It is difficult to believe, but ALL THE SLOTS HAVE LENGTH 1. */
349 		x = q->tail->next;
350 		slot = &q->slots[x];
351 		q->tail->next = slot->next;
352 		q->ht[slot->hash] = SFQ_EMPTY_SLOT;
353 		goto drop;
354 	}
355 
356 	return 0;
357 }
358 
359 static int
360 sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
361 {
362 	struct sfq_sched_data *q = qdisc_priv(sch);
363 	unsigned int hash;
364 	sfq_index x, qlen;
365 	struct sfq_slot *slot;
366 	int uninitialized_var(ret);
367 
368 	hash = sfq_classify(skb, sch, &ret);
369 	if (hash == 0) {
370 		if (ret & __NET_XMIT_BYPASS)
371 			sch->qstats.drops++;
372 		kfree_skb(skb);
373 		return ret;
374 	}
375 	hash--;
376 
377 	x = q->ht[hash];
378 	slot = &q->slots[x];
379 	if (x == SFQ_EMPTY_SLOT) {
380 		x = q->dep[0].next; /* get a free slot */
381 		q->ht[hash] = x;
382 		slot = &q->slots[x];
383 		slot->hash = hash;
384 	}
385 
386 	/* If selected queue has length q->limit, do simple tail drop,
387 	 * i.e. drop _this_ packet.
388 	 */
389 	if (slot->qlen >= q->limit)
390 		return qdisc_drop(skb, sch);
391 
392 	sch->qstats.backlog += qdisc_pkt_len(skb);
393 	slot_queue_add(slot, skb);
394 	sfq_inc(q, x);
395 	if (slot->qlen == 1) {		/* The flow is new */
396 		if (q->tail == NULL) {	/* It is the first flow */
397 			slot->next = x;
398 		} else {
399 			slot->next = q->tail->next;
400 			q->tail->next = x;
401 		}
402 		q->tail = slot;
403 		slot->allot = q->scaled_quantum;
404 	}
405 	if (++sch->q.qlen <= q->limit)
406 		return NET_XMIT_SUCCESS;
407 
408 	qlen = slot->qlen;
409 	sfq_drop(sch);
410 	/* Return Congestion Notification only if we dropped a packet
411 	 * from this flow.
412 	 */
413 	if (qlen != slot->qlen)
414 		return NET_XMIT_CN;
415 
416 	/* As we dropped a packet, better let upper stack know this */
417 	qdisc_tree_decrease_qlen(sch, 1);
418 	return NET_XMIT_SUCCESS;
419 }
420 
421 static struct sk_buff *
422 sfq_dequeue(struct Qdisc *sch)
423 {
424 	struct sfq_sched_data *q = qdisc_priv(sch);
425 	struct sk_buff *skb;
426 	sfq_index a, next_a;
427 	struct sfq_slot *slot;
428 
429 	/* No active slots */
430 	if (q->tail == NULL)
431 		return NULL;
432 
433 next_slot:
434 	a = q->tail->next;
435 	slot = &q->slots[a];
436 	if (slot->allot <= 0) {
437 		q->tail = slot;
438 		slot->allot += q->scaled_quantum;
439 		goto next_slot;
440 	}
441 	skb = slot_dequeue_head(slot);
442 	sfq_dec(q, a);
443 	qdisc_bstats_update(sch, skb);
444 	sch->q.qlen--;
445 	sch->qstats.backlog -= qdisc_pkt_len(skb);
446 
447 	/* Is the slot empty? */
448 	if (slot->qlen == 0) {
449 		q->ht[slot->hash] = SFQ_EMPTY_SLOT;
450 		next_a = slot->next;
451 		if (a == next_a) {
452 			q->tail = NULL; /* no more active slots */
453 			return skb;
454 		}
455 		q->tail->next = next_a;
456 	} else {
457 		slot->allot -= SFQ_ALLOT_SIZE(qdisc_pkt_len(skb));
458 	}
459 	return skb;
460 }
461 
462 static void
463 sfq_reset(struct Qdisc *sch)
464 {
465 	struct sk_buff *skb;
466 
467 	while ((skb = sfq_dequeue(sch)) != NULL)
468 		kfree_skb(skb);
469 }
470 
471 static void sfq_perturbation(unsigned long arg)
472 {
473 	struct Qdisc *sch = (struct Qdisc *)arg;
474 	struct sfq_sched_data *q = qdisc_priv(sch);
475 
476 	q->perturbation = net_random();
477 
478 	if (q->perturb_period)
479 		mod_timer(&q->perturb_timer, jiffies + q->perturb_period);
480 }
481 
482 static int sfq_change(struct Qdisc *sch, struct nlattr *opt)
483 {
484 	struct sfq_sched_data *q = qdisc_priv(sch);
485 	struct tc_sfq_qopt *ctl = nla_data(opt);
486 	unsigned int qlen;
487 
488 	if (opt->nla_len < nla_attr_size(sizeof(*ctl)))
489 		return -EINVAL;
490 
491 	if (ctl->divisor &&
492 	    (!is_power_of_2(ctl->divisor) || ctl->divisor > 65536))
493 		return -EINVAL;
494 
495 	sch_tree_lock(sch);
496 	q->quantum = ctl->quantum ? : psched_mtu(qdisc_dev(sch));
497 	q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
498 	q->perturb_period = ctl->perturb_period * HZ;
499 	if (ctl->limit)
500 		q->limit = min_t(u32, ctl->limit, SFQ_DEPTH - 1);
501 	if (ctl->divisor)
502 		q->divisor = ctl->divisor;
503 	qlen = sch->q.qlen;
504 	while (sch->q.qlen > q->limit)
505 		sfq_drop(sch);
506 	qdisc_tree_decrease_qlen(sch, qlen - sch->q.qlen);
507 
508 	del_timer(&q->perturb_timer);
509 	if (q->perturb_period) {
510 		mod_timer(&q->perturb_timer, jiffies + q->perturb_period);
511 		q->perturbation = net_random();
512 	}
513 	sch_tree_unlock(sch);
514 	return 0;
515 }
516 
517 static int sfq_init(struct Qdisc *sch, struct nlattr *opt)
518 {
519 	struct sfq_sched_data *q = qdisc_priv(sch);
520 	size_t sz;
521 	int i;
522 
523 	q->perturb_timer.function = sfq_perturbation;
524 	q->perturb_timer.data = (unsigned long)sch;
525 	init_timer_deferrable(&q->perturb_timer);
526 
527 	for (i = 0; i < SFQ_DEPTH; i++) {
528 		q->dep[i].next = i + SFQ_SLOTS;
529 		q->dep[i].prev = i + SFQ_SLOTS;
530 	}
531 
532 	q->limit = SFQ_DEPTH - 1;
533 	q->cur_depth = 0;
534 	q->tail = NULL;
535 	q->divisor = SFQ_DEFAULT_HASH_DIVISOR;
536 	if (opt == NULL) {
537 		q->quantum = psched_mtu(qdisc_dev(sch));
538 		q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
539 		q->perturb_period = 0;
540 		q->perturbation = net_random();
541 	} else {
542 		int err = sfq_change(sch, opt);
543 		if (err)
544 			return err;
545 	}
546 
547 	sz = sizeof(q->ht[0]) * q->divisor;
548 	q->ht = kmalloc(sz, GFP_KERNEL);
549 	if (!q->ht && sz > PAGE_SIZE)
550 		q->ht = vmalloc(sz);
551 	if (!q->ht)
552 		return -ENOMEM;
553 	for (i = 0; i < q->divisor; i++)
554 		q->ht[i] = SFQ_EMPTY_SLOT;
555 
556 	for (i = 0; i < SFQ_SLOTS; i++) {
557 		slot_queue_init(&q->slots[i]);
558 		sfq_link(q, i);
559 	}
560 	if (q->limit >= 1)
561 		sch->flags |= TCQ_F_CAN_BYPASS;
562 	else
563 		sch->flags &= ~TCQ_F_CAN_BYPASS;
564 	return 0;
565 }
566 
567 static void sfq_destroy(struct Qdisc *sch)
568 {
569 	struct sfq_sched_data *q = qdisc_priv(sch);
570 
571 	tcf_destroy_chain(&q->filter_list);
572 	q->perturb_period = 0;
573 	del_timer_sync(&q->perturb_timer);
574 	if (is_vmalloc_addr(q->ht))
575 		vfree(q->ht);
576 	else
577 		kfree(q->ht);
578 }
579 
580 static int sfq_dump(struct Qdisc *sch, struct sk_buff *skb)
581 {
582 	struct sfq_sched_data *q = qdisc_priv(sch);
583 	unsigned char *b = skb_tail_pointer(skb);
584 	struct tc_sfq_qopt opt;
585 
586 	opt.quantum = q->quantum;
587 	opt.perturb_period = q->perturb_period / HZ;
588 
589 	opt.limit = q->limit;
590 	opt.divisor = q->divisor;
591 	opt.flows = q->limit;
592 
593 	NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
594 
595 	return skb->len;
596 
597 nla_put_failure:
598 	nlmsg_trim(skb, b);
599 	return -1;
600 }
601 
602 static struct Qdisc *sfq_leaf(struct Qdisc *sch, unsigned long arg)
603 {
604 	return NULL;
605 }
606 
607 static unsigned long sfq_get(struct Qdisc *sch, u32 classid)
608 {
609 	return 0;
610 }
611 
612 static unsigned long sfq_bind(struct Qdisc *sch, unsigned long parent,
613 			      u32 classid)
614 {
615 	/* we cannot bypass queue discipline anymore */
616 	sch->flags &= ~TCQ_F_CAN_BYPASS;
617 	return 0;
618 }
619 
620 static void sfq_put(struct Qdisc *q, unsigned long cl)
621 {
622 }
623 
624 static struct tcf_proto **sfq_find_tcf(struct Qdisc *sch, unsigned long cl)
625 {
626 	struct sfq_sched_data *q = qdisc_priv(sch);
627 
628 	if (cl)
629 		return NULL;
630 	return &q->filter_list;
631 }
632 
633 static int sfq_dump_class(struct Qdisc *sch, unsigned long cl,
634 			  struct sk_buff *skb, struct tcmsg *tcm)
635 {
636 	tcm->tcm_handle |= TC_H_MIN(cl);
637 	return 0;
638 }
639 
640 static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
641 				struct gnet_dump *d)
642 {
643 	struct sfq_sched_data *q = qdisc_priv(sch);
644 	sfq_index idx = q->ht[cl - 1];
645 	struct gnet_stats_queue qs = { 0 };
646 	struct tc_sfq_xstats xstats = { 0 };
647 	struct sk_buff *skb;
648 
649 	if (idx != SFQ_EMPTY_SLOT) {
650 		const struct sfq_slot *slot = &q->slots[idx];
651 
652 		xstats.allot = slot->allot << SFQ_ALLOT_SHIFT;
653 		qs.qlen = slot->qlen;
654 		slot_queue_walk(slot, skb)
655 			qs.backlog += qdisc_pkt_len(skb);
656 	}
657 	if (gnet_stats_copy_queue(d, &qs) < 0)
658 		return -1;
659 	return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
660 }
661 
662 static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
663 {
664 	struct sfq_sched_data *q = qdisc_priv(sch);
665 	unsigned int i;
666 
667 	if (arg->stop)
668 		return;
669 
670 	for (i = 0; i < q->divisor; i++) {
671 		if (q->ht[i] == SFQ_EMPTY_SLOT ||
672 		    arg->count < arg->skip) {
673 			arg->count++;
674 			continue;
675 		}
676 		if (arg->fn(sch, i + 1, arg) < 0) {
677 			arg->stop = 1;
678 			break;
679 		}
680 		arg->count++;
681 	}
682 }
683 
684 static const struct Qdisc_class_ops sfq_class_ops = {
685 	.leaf		=	sfq_leaf,
686 	.get		=	sfq_get,
687 	.put		=	sfq_put,
688 	.tcf_chain	=	sfq_find_tcf,
689 	.bind_tcf	=	sfq_bind,
690 	.unbind_tcf	=	sfq_put,
691 	.dump		=	sfq_dump_class,
692 	.dump_stats	=	sfq_dump_class_stats,
693 	.walk		=	sfq_walk,
694 };
695 
696 static struct Qdisc_ops sfq_qdisc_ops __read_mostly = {
697 	.cl_ops		=	&sfq_class_ops,
698 	.id		=	"sfq",
699 	.priv_size	=	sizeof(struct sfq_sched_data),
700 	.enqueue	=	sfq_enqueue,
701 	.dequeue	=	sfq_dequeue,
702 	.peek		=	qdisc_peek_dequeued,
703 	.drop		=	sfq_drop,
704 	.init		=	sfq_init,
705 	.reset		=	sfq_reset,
706 	.destroy	=	sfq_destroy,
707 	.change		=	NULL,
708 	.dump		=	sfq_dump,
709 	.owner		=	THIS_MODULE,
710 };
711 
712 static int __init sfq_module_init(void)
713 {
714 	return register_qdisc(&sfq_qdisc_ops);
715 }
716 static void __exit sfq_module_exit(void)
717 {
718 	unregister_qdisc(&sfq_qdisc_ops);
719 }
720 module_init(sfq_module_init)
721 module_exit(sfq_module_exit)
722 MODULE_LICENSE("GPL");
723