177 static void poll_timer_fn(struct timer_list *t);
440 	struct psi_trigger *t;  in update_triggers()  local
458 	list_for_each_entry(t, triggers, node) {  in update_triggers()
462 		new_stall = aggregator_total[t->state] != total[t->state];  in update_triggers()
465 		if (!new_stall && !t->pending_event)  in update_triggers()
483 			growth = window_update(&t->win, now, total[t->state]);  in update_triggers()
484 			if (!t->pending_event) {  in update_triggers()
485 				if (growth < t->threshold)  in update_triggers()
488 				t->pending_event = true;  in update_triggers()
492 		if (now < t->last_event_time + t->win.size)  in update_triggers()
496 		if (cmpxchg(&t->event, 0, 1) == 0) {  in update_triggers()
497 			if (t->of)  in update_triggers()
498 				kernfs_notify(t->of->kn);  in update_triggers()
500 				wake_up_interruptible(&t->event_wait);  in update_triggers()
502 		t->last_event_time = now;  in update_triggers()
504 		t->pending_event = false;  in update_triggers()
601 	struct psi_trigger *t;  in init_rtpoll_triggers()  local
603 	list_for_each_entry(t, &group->rtpoll_triggers, node)  in init_rtpoll_triggers()
604 		window_reset(&t->win, now,  in init_rtpoll_triggers()
605 				group->total[PSI_POLL][t->state], 0);  in init_rtpoll_triggers()
741 static void poll_timer_fn(struct timer_list *t)  in poll_timer_fn()  argument
743 	struct psi_group *group = from_timer(group, t, rtpoll_timer);  in poll_timer_fn()
783 	unsigned int t, m;  in psi_group_change()  local
823 	for (t = 0, m = clear; m; m &= ~(1 << t), t++) {  in psi_group_change()
824 		if (!(m & (1 << t)))  in psi_group_change()
826 		if (groupc->tasks[t]) {  in psi_group_change()
827 			groupc->tasks[t]--;  in psi_group_change()
830 					cpu, t, groupc->tasks[0],  in psi_group_change()
837 	for (t = 0; set; set &= ~(1 << t), t++)  in psi_group_change()
838 		if (set & (1 << t))  in psi_group_change()
839 			groupc->tasks[t]++;  in psi_group_change()
1291 	struct psi_trigger *t;  in psi_trigger_create()  local
1335 	t = kmalloc(sizeof(*t), GFP_KERNEL);  in psi_trigger_create()
1336 	if (!t)  in psi_trigger_create()
1339 	t->group = group;  in psi_trigger_create()
1340 	t->state = state;  in psi_trigger_create()
1341 	t->threshold = threshold_us * NSEC_PER_USEC;  in psi_trigger_create()
1342 	t->win.size = window_us * NSEC_PER_USEC;  in psi_trigger_create()
1343 	window_reset(&t->win, sched_clock(),  in psi_trigger_create()
1344 			group->total[PSI_POLL][t->state], 0);  in psi_trigger_create()
1346 	t->event = 0;  in psi_trigger_create()
1347 	t->last_event_time = 0;  in psi_trigger_create()
1348 	t->of = of;  in psi_trigger_create()
1350 		init_waitqueue_head(&t->event_wait);  in psi_trigger_create()
1351 	t->pending_event = false;  in psi_trigger_create()
1352 	t->aggregator = privileged ? PSI_POLL : PSI_AVGS;  in psi_trigger_create()
1362 				kfree(t);  in psi_trigger_create()
1371 		list_add(&t->node, &group->rtpoll_triggers);  in psi_trigger_create()
1373 			div_u64(t->win.size, UPDATES_PER_WINDOW));  in psi_trigger_create()
1374 		group->rtpoll_nr_triggers[t->state]++;  in psi_trigger_create()
1375 		group->rtpoll_states |= (1 << t->state);  in psi_trigger_create()
1381 		list_add(&t->node, &group->avg_triggers);  in psi_trigger_create()
1382 		group->avg_nr_triggers[t->state]++;  in psi_trigger_create()
1386 	return t;  in psi_trigger_create()
1389 void psi_trigger_destroy(struct psi_trigger *t)  in psi_trigger_destroy()  argument
1398 	if (!t)  in psi_trigger_destroy()
1401 	group = t->group;  in psi_trigger_destroy()
1407 	if (t->of)  in psi_trigger_destroy()
1408 		kernfs_notify(t->of->kn);  in psi_trigger_destroy()
1410 		wake_up_interruptible(&t->event_wait);  in psi_trigger_destroy()
1412 	if (t->aggregator == PSI_AVGS) {  in psi_trigger_destroy()
1414 		if (!list_empty(&t->node)) {  in psi_trigger_destroy()
1415 			list_del(&t->node);  in psi_trigger_destroy()
1416 			group->avg_nr_triggers[t->state]--;  in psi_trigger_destroy()
1421 		if (!list_empty(&t->node)) {  in psi_trigger_destroy()
1425 			list_del(&t->node);  in psi_trigger_destroy()
1426 			group->rtpoll_nr_triggers[t->state]--;  in psi_trigger_destroy()
1427 			if (!group->rtpoll_nr_triggers[t->state])  in psi_trigger_destroy()
1428 				group->rtpoll_states &= ~(1 << t->state);  in psi_trigger_destroy()
1433 			if (group->rtpoll_min_period == div_u64(t->win.size, UPDATES_PER_WINDOW)) {  in psi_trigger_destroy()
1471 	kfree(t);  in psi_trigger_destroy()
1478 	struct psi_trigger *t;  in psi_trigger_poll()  local
1483 	t = smp_load_acquire(trigger_ptr);  in psi_trigger_poll()
1484 	if (!t)  in psi_trigger_poll()
1487 	if (t->of)  in psi_trigger_poll()
1488 		kernfs_generic_poll(t->of, wait);  in psi_trigger_poll()
1490 		poll_wait(file, &t->event_wait, wait);  in psi_trigger_poll()
1492 	if (cmpxchg(&t->event, 1, 0) == 1)  in psi_trigger_poll()