Lines Matching refs:q
207 struct qfq_sched *q = qdisc_priv(sch); in qfq_find_class() local
210 clc = qdisc_class_find(&q->clhash, classid); in qfq_find_class()
257 static void qfq_init_agg(struct qfq_sched *q, struct qfq_aggregate *agg, in qfq_init_agg() argument
261 hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs); in qfq_init_agg()
267 static struct qfq_aggregate *qfq_find_agg(struct qfq_sched *q, in qfq_find_agg() argument
272 hlist_for_each_entry(agg, &q->nonfull_aggs, nonfull_next) in qfq_find_agg()
281 static void qfq_update_agg(struct qfq_sched *q, struct qfq_aggregate *agg, in qfq_update_agg() argument
286 if (new_num_classes == q->max_agg_classes) in qfq_update_agg()
290 new_num_classes == q->max_agg_classes - 1) /* agg no more full */ in qfq_update_agg()
291 hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs); in qfq_update_agg()
303 q->min_slot_shift); in qfq_update_agg()
304 agg->grp = &q->groups[i]; in qfq_update_agg()
307 q->wsum += in qfq_update_agg()
309 q->iwsum = ONE_FP / q->wsum; in qfq_update_agg()
315 static void qfq_add_to_agg(struct qfq_sched *q, in qfq_add_to_agg() argument
321 qfq_update_agg(q, agg, agg->num_classes+1); in qfq_add_to_agg()
322 if (cl->qdisc->q.qlen > 0) { /* adding an active class */ in qfq_add_to_agg()
325 cl && q->in_serv_agg != agg) /* agg was inactive */ in qfq_add_to_agg()
326 qfq_activate_agg(q, agg, enqueue); /* schedule agg */ in qfq_add_to_agg()
332 static void qfq_destroy_agg(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_destroy_agg() argument
335 q->wsum -= agg->class_weight; in qfq_destroy_agg()
336 if (q->wsum != 0) in qfq_destroy_agg()
337 q->iwsum = ONE_FP / q->wsum; in qfq_destroy_agg()
339 if (q->in_serv_agg == agg) in qfq_destroy_agg()
340 q->in_serv_agg = qfq_choose_next_agg(q); in qfq_destroy_agg()
345 static void qfq_deactivate_class(struct qfq_sched *q, struct qfq_class *cl) in qfq_deactivate_class() argument
352 qfq_deactivate_agg(q, agg); in qfq_deactivate_class()
356 static void qfq_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) in qfq_rm_from_agg() argument
362 qfq_destroy_agg(q, agg); in qfq_rm_from_agg()
365 qfq_update_agg(q, agg, agg->num_classes-1); in qfq_rm_from_agg()
369 static void qfq_deact_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) in qfq_deact_rm_from_agg() argument
371 if (cl->qdisc->q.qlen > 0) /* class is active */ in qfq_deact_rm_from_agg()
372 qfq_deactivate_class(q, cl); in qfq_deact_rm_from_agg()
374 qfq_rm_from_agg(q, cl); in qfq_deact_rm_from_agg()
381 struct qfq_sched *q = qdisc_priv(sch); in qfq_change_agg() local
388 new_agg = qfq_find_agg(q, lmax, weight); in qfq_change_agg()
393 qfq_init_agg(q, new_agg, lmax, weight); in qfq_change_agg()
395 qfq_deact_rm_from_agg(q, cl); in qfq_change_agg()
396 qfq_add_to_agg(q, new_agg, cl); in qfq_change_agg()
405 struct qfq_sched *q = qdisc_priv(sch); in qfq_change_class() local
451 if (q->wsum + delta_w > QFQ_MAX_WSUM) { in qfq_change_class()
454 delta_w, q->wsum); in qfq_change_class()
501 new_agg = qfq_find_agg(q, lmax, weight); in qfq_change_class()
511 qfq_init_agg(q, new_agg, lmax, weight); in qfq_change_class()
514 qfq_deact_rm_from_agg(q, cl); in qfq_change_class()
516 qdisc_class_hash_insert(&q->clhash, &cl->common); in qfq_change_class()
517 qfq_add_to_agg(q, new_agg, cl); in qfq_change_class()
519 qdisc_class_hash_grow(sch, &q->clhash); in qfq_change_class()
532 struct qfq_sched *q = qdisc_priv(sch); in qfq_destroy_class() local
534 qfq_rm_from_agg(q, cl); in qfq_destroy_class()
543 struct qfq_sched *q = qdisc_priv(sch); in qfq_delete_class() local
554 qdisc_class_hash_remove(&q->clhash, &cl->common); in qfq_delete_class()
570 struct qfq_sched *q = qdisc_priv(sch); in qfq_tcf_block() local
575 return q->block; in qfq_tcf_block()
664 struct qfq_sched *q = qdisc_priv(sch); in qfq_walk() local
671 for (i = 0; i < q->clhash.hashsize; i++) { in qfq_walk()
672 hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { in qfq_walk()
682 struct qfq_sched *q = qdisc_priv(sch); in qfq_classify() local
696 fl = rcu_dereference_bh(q->filter_list); in qfq_classify()
732 static inline struct qfq_group *qfq_ffs(struct qfq_sched *q, in qfq_ffs() argument
736 return &q->groups[index]; in qfq_ffs()
749 static int qfq_calc_state(struct qfq_sched *q, const struct qfq_group *grp) in qfq_calc_state() argument
752 unsigned int state = qfq_gt(grp->S, q->V); in qfq_calc_state()
753 unsigned long mask = mask_from(q->bitmaps[ER], grp->index); in qfq_calc_state()
757 next = qfq_ffs(q, mask); in qfq_calc_state()
772 static inline void qfq_move_groups(struct qfq_sched *q, unsigned long mask, in qfq_move_groups() argument
775 q->bitmaps[dst] |= q->bitmaps[src] & mask; in qfq_move_groups()
776 q->bitmaps[src] &= ~mask; in qfq_move_groups()
779 static void qfq_unblock_groups(struct qfq_sched *q, int index, u64 old_F) in qfq_unblock_groups() argument
781 unsigned long mask = mask_from(q->bitmaps[ER], index + 1); in qfq_unblock_groups()
785 next = qfq_ffs(q, mask); in qfq_unblock_groups()
791 qfq_move_groups(q, mask, EB, ER); in qfq_unblock_groups()
792 qfq_move_groups(q, mask, IB, IR); in qfq_unblock_groups()
805 static void qfq_make_eligible(struct qfq_sched *q) in qfq_make_eligible() argument
807 unsigned long vslot = q->V >> q->min_slot_shift; in qfq_make_eligible()
808 unsigned long old_vslot = q->oldV >> q->min_slot_shift; in qfq_make_eligible()
819 qfq_move_groups(q, mask, IR, ER); in qfq_make_eligible()
820 qfq_move_groups(q, mask, IB, EB); in qfq_make_eligible()
960 static void qfq_update_eligible(struct qfq_sched *q) in qfq_update_eligible() argument
965 ineligible = q->bitmaps[IR] | q->bitmaps[IB]; in qfq_update_eligible()
967 if (!q->bitmaps[ER]) { in qfq_update_eligible()
968 grp = qfq_ffs(q, ineligible); in qfq_update_eligible()
969 if (qfq_gt(grp->S, q->V)) in qfq_update_eligible()
970 q->V = grp->S; in qfq_update_eligible()
972 qfq_make_eligible(q); in qfq_update_eligible()
987 if (cl->qdisc->q.qlen == 0) /* no more packets, remove from list */ in agg_dequeue()
1039 static void qfq_update_start(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_update_start() argument
1046 limit = qfq_round_down(q->V, slot_shift) + (1ULL << slot_shift); in qfq_update_start()
1048 if (!qfq_gt(agg->F, q->V) || qfq_gt(roundedF, limit)) { in qfq_update_start()
1050 mask = mask_from(q->bitmaps[ER], agg->grp->index); in qfq_update_start()
1052 struct qfq_group *next = qfq_ffs(q, mask); in qfq_update_start()
1061 agg->S = q->V; in qfq_update_start()
1072 qfq_update_agg_ts(struct qfq_sched *q, in qfq_update_agg_ts() argument
1076 qfq_update_start(q, agg); in qfq_update_agg_ts()
1083 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg);
1087 struct qfq_sched *q = qdisc_priv(sch); in qfq_dequeue() local
1088 struct qfq_aggregate *in_serv_agg = q->in_serv_agg; in qfq_dequeue()
1123 qfq_update_agg_ts(q, in_serv_agg, requeue); in qfq_dequeue()
1124 qfq_schedule_agg(q, in_serv_agg); in qfq_dequeue()
1125 } else if (sch->q.qlen == 0) { /* no aggregate to serve */ in qfq_dequeue()
1126 q->in_serv_agg = NULL; in qfq_dequeue()
1134 in_serv_agg = q->in_serv_agg = qfq_choose_next_agg(q); in qfq_dequeue()
1140 sch->q.qlen--; in qfq_dequeue()
1145 sch->q.qlen++; in qfq_dequeue()
1161 q->V += (u64)len * q->iwsum; in qfq_dequeue()
1164 (unsigned long long) q->V); in qfq_dequeue()
1169 static struct qfq_aggregate *qfq_choose_next_agg(struct qfq_sched *q) in qfq_choose_next_agg() argument
1175 qfq_update_eligible(q); in qfq_choose_next_agg()
1176 q->oldV = q->V; in qfq_choose_next_agg()
1178 if (!q->bitmaps[ER]) in qfq_choose_next_agg()
1181 grp = qfq_ffs(q, q->bitmaps[ER]); in qfq_choose_next_agg()
1192 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_choose_next_agg()
1202 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_choose_next_agg()
1203 s = qfq_calc_state(q, grp); in qfq_choose_next_agg()
1204 __set_bit(grp->index, &q->bitmaps[s]); in qfq_choose_next_agg()
1207 qfq_unblock_groups(q, grp->index, old_F); in qfq_choose_next_agg()
1216 struct qfq_sched *q = qdisc_priv(sch); in qfq_enqueue() local
1242 first = !cl->qdisc->q.qlen; in qfq_enqueue()
1255 ++sch->q.qlen; in qfq_enqueue()
1273 q->in_serv_agg == agg) in qfq_enqueue()
1276 qfq_activate_agg(q, agg, enqueue); in qfq_enqueue()
1284 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_schedule_agg() argument
1308 __clear_bit(grp->index, &q->bitmaps[IR]); in qfq_schedule_agg()
1309 __clear_bit(grp->index, &q->bitmaps[IB]); in qfq_schedule_agg()
1310 } else if (!q->bitmaps[ER] && qfq_gt(roundedS, q->V) && in qfq_schedule_agg()
1311 q->in_serv_agg == NULL) in qfq_schedule_agg()
1312 q->V = roundedS; in qfq_schedule_agg()
1316 s = qfq_calc_state(q, grp); in qfq_schedule_agg()
1317 __set_bit(grp->index, &q->bitmaps[s]); in qfq_schedule_agg()
1320 s, q->bitmaps[s], in qfq_schedule_agg()
1323 (unsigned long long) q->V); in qfq_schedule_agg()
1331 static void qfq_activate_agg(struct qfq_sched *q, struct qfq_aggregate *agg, in qfq_activate_agg() argument
1336 qfq_update_agg_ts(q, agg, reason); in qfq_activate_agg()
1337 if (q->in_serv_agg == NULL) { /* no aggr. in service or scheduled */ in qfq_activate_agg()
1338 q->in_serv_agg = agg; /* start serving this aggregate */ in qfq_activate_agg()
1340 q->oldV = q->V = agg->S; in qfq_activate_agg()
1341 } else if (agg != q->in_serv_agg) in qfq_activate_agg()
1342 qfq_schedule_agg(q, agg); in qfq_activate_agg()
1345 static void qfq_slot_remove(struct qfq_sched *q, struct qfq_group *grp, in qfq_slot_remove() argument
1368 static void qfq_deactivate_agg(struct qfq_sched *q, struct qfq_aggregate *agg) in qfq_deactivate_agg() argument
1375 if (agg == q->in_serv_agg) { in qfq_deactivate_agg()
1377 q->in_serv_agg = qfq_choose_next_agg(q); in qfq_deactivate_agg()
1382 qfq_slot_remove(q, grp, agg); in qfq_deactivate_agg()
1385 __clear_bit(grp->index, &q->bitmaps[IR]); in qfq_deactivate_agg()
1386 __clear_bit(grp->index, &q->bitmaps[EB]); in qfq_deactivate_agg()
1387 __clear_bit(grp->index, &q->bitmaps[IB]); in qfq_deactivate_agg()
1389 if (test_bit(grp->index, &q->bitmaps[ER]) && in qfq_deactivate_agg()
1390 !(q->bitmaps[ER] & ~((1UL << grp->index) - 1))) { in qfq_deactivate_agg()
1391 mask = q->bitmaps[ER] & ((1UL << grp->index) - 1); in qfq_deactivate_agg()
1396 qfq_move_groups(q, mask, EB, ER); in qfq_deactivate_agg()
1397 qfq_move_groups(q, mask, IB, IR); in qfq_deactivate_agg()
1399 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_deactivate_agg()
1404 __clear_bit(grp->index, &q->bitmaps[ER]); in qfq_deactivate_agg()
1405 __clear_bit(grp->index, &q->bitmaps[IR]); in qfq_deactivate_agg()
1406 __clear_bit(grp->index, &q->bitmaps[EB]); in qfq_deactivate_agg()
1407 __clear_bit(grp->index, &q->bitmaps[IB]); in qfq_deactivate_agg()
1410 s = qfq_calc_state(q, grp); in qfq_deactivate_agg()
1411 __set_bit(grp->index, &q->bitmaps[s]); in qfq_deactivate_agg()
1418 struct qfq_sched *q = qdisc_priv(sch); in qfq_qlen_notify() local
1421 qfq_deactivate_class(q, cl); in qfq_qlen_notify()
1427 struct qfq_sched *q = qdisc_priv(sch); in qfq_init_qdisc() local
1432 err = tcf_block_get(&q->block, &q->filter_list, sch, extack); in qfq_init_qdisc()
1436 err = qdisc_class_hash_init(&q->clhash); in qfq_init_qdisc()
1444 q->max_agg_classes = 1<<max_cl_shift; in qfq_init_qdisc()
1448 q->min_slot_shift = FRAC_BITS + maxbudg_shift - QFQ_MAX_INDEX; in qfq_init_qdisc()
1451 grp = &q->groups[i]; in qfq_init_qdisc()
1453 grp->slot_shift = q->min_slot_shift + i; in qfq_init_qdisc()
1458 INIT_HLIST_HEAD(&q->nonfull_aggs); in qfq_init_qdisc()
1465 struct qfq_sched *q = qdisc_priv(sch); in qfq_reset_qdisc() local
1469 for (i = 0; i < q->clhash.hashsize; i++) { in qfq_reset_qdisc()
1470 hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { in qfq_reset_qdisc()
1471 if (cl->qdisc->q.qlen > 0) in qfq_reset_qdisc()
1472 qfq_deactivate_class(q, cl); in qfq_reset_qdisc()
1481 struct qfq_sched *q = qdisc_priv(sch); in qfq_destroy_qdisc() local
1486 tcf_block_put(q->block); in qfq_destroy_qdisc()
1488 for (i = 0; i < q->clhash.hashsize; i++) { in qfq_destroy_qdisc()
1489 hlist_for_each_entry_safe(cl, next, &q->clhash.hash[i], in qfq_destroy_qdisc()
1494 qdisc_class_hash_destroy(&q->clhash); in qfq_destroy_qdisc()