47 static void padata_free_pd(struct parallel_data *pd);
50 static inline void padata_get_pd(struct parallel_data *pd)  in padata_get_pd()  argument
52 	refcount_inc(&pd->refcnt);  in padata_get_pd()
55 static inline void padata_put_pd_cnt(struct parallel_data *pd, int cnt)  in padata_put_pd_cnt()  argument
57 	if (refcount_sub_and_test(cnt, &pd->refcnt))  in padata_put_pd_cnt()
58 		padata_free_pd(pd);  in padata_put_pd_cnt()
61 static inline void padata_put_pd(struct parallel_data *pd)  in padata_put_pd()  argument
63 	padata_put_pd_cnt(pd, 1);  in padata_put_pd()
66 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)  in padata_index_to_cpu()  argument
70 	target_cpu = cpumask_first(pd->cpumask.pcpu);  in padata_index_to_cpu()
72 		target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu);  in padata_index_to_cpu()
77 static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr)  in padata_cpu_hash()  argument
83 	int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);  in padata_cpu_hash()
85 	return padata_index_to_cpu(pd, cpu_index);  in padata_cpu_hash()
196 	struct parallel_data *pd;  in padata_do_parallel()  local
201 	pd = rcu_dereference_bh(ps->pd);  in padata_do_parallel()
207 	if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) {  in padata_do_parallel()
208 		if (cpumask_empty(pd->cpumask.cbcpu))  in padata_do_parallel()
212 		cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu);  in padata_do_parallel()
214 		cpu = cpumask_first(pd->cpumask.cbcpu);  in padata_do_parallel()
216 			cpu = cpumask_next(cpu, pd->cpumask.cbcpu);  in padata_do_parallel()
225 	padata_get_pd(pd);  in padata_do_parallel()
226 	padata->pd = pd;  in padata_do_parallel()
230 	padata->seq_nr = ++pd->seq_nr;  in padata_do_parallel()
264 static struct padata_priv *padata_find_next(struct parallel_data *pd,  in padata_find_next()  argument
269 	int cpu = pd->cpu;  in padata_find_next()
271 	reorder = per_cpu_ptr(pd->reorder_list, cpu);  in padata_find_next()
285 	if (padata->seq_nr != pd->processed) {  in padata_find_next()
292 		++pd->processed;  in padata_find_next()
293 		pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false);  in padata_find_next()
300 static void padata_reorder(struct parallel_data *pd)  in padata_reorder()  argument
302 	struct padata_instance *pinst = pd->ps->pinst;  in padata_reorder()
318 	if (!spin_trylock_bh(&pd->lock))  in padata_reorder()
322 		padata = padata_find_next(pd, true);  in padata_reorder()
333 		squeue = per_cpu_ptr(pd->squeue, cb_cpu);  in padata_reorder()
342 	spin_unlock_bh(&pd->lock);  in padata_reorder()
348 	 * Ensure reorder queue is read after pd->lock is dropped so we see  in padata_reorder()
354 	reorder = per_cpu_ptr(pd->reorder_list, pd->cpu);  in padata_reorder()
355 	if (!list_empty(&reorder->list) && padata_find_next(pd, false)) {  in padata_reorder()
358 		 * To avoid UAF issue, add pd ref here, and put pd ref after reorder_work finish.  in padata_reorder()
360 		padata_get_pd(pd);  in padata_reorder()
361 		queue_work(pinst->serial_wq, &pd->reorder_work);  in padata_reorder()
367 	struct parallel_data *pd;  in invoke_padata_reorder()  local
370 	pd = container_of(work, struct parallel_data, reorder_work);  in invoke_padata_reorder()
371 	padata_reorder(pd);  in invoke_padata_reorder()
374 	padata_put_pd(pd);  in invoke_padata_reorder()
380 	struct parallel_data *pd;  in padata_serial_worker()  local
386 	pd = squeue->pd;  in padata_serial_worker()
407 	padata_put_pd_cnt(pd, cnt);  in padata_serial_worker()
420 	struct parallel_data *pd = padata->pd;  in padata_do_serial()  local
421 	int hashed_cpu = padata_cpu_hash(pd, padata->seq_nr);  in padata_do_serial()
422 	struct padata_list *reorder = per_cpu_ptr(pd->reorder_list, hashed_cpu);  in padata_do_serial()
439 	 * with the trylock of pd->lock in padata_reorder.  Pairs with smp_mb  in padata_do_serial()
444 	padata_reorder(pd);  in padata_do_serial()
457 	/* Restrict parallel_wq workers to pd->cpumask.pcpu. */  in padata_setup_cpumasks()
574 static void padata_init_squeues(struct parallel_data *pd)  in padata_init_squeues()  argument
579 	for_each_cpu(cpu, pd->cpumask.cbcpu) {  in padata_init_squeues()
580 		squeue = per_cpu_ptr(pd->squeue, cpu);  in padata_init_squeues()
581 		squeue->pd = pd;  in padata_init_squeues()
588 static void padata_init_reorder_list(struct parallel_data *pd)  in padata_init_reorder_list()  argument
593 	for_each_cpu(cpu, pd->cpumask.pcpu) {  in padata_init_reorder_list()
594 		list = per_cpu_ptr(pd->reorder_list, cpu);  in padata_init_reorder_list()
603 	struct parallel_data *pd;  in padata_alloc_pd()  local
605 	pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL);  in padata_alloc_pd()
606 	if (!pd)  in padata_alloc_pd()
609 	pd->reorder_list = alloc_percpu(struct padata_list);  in padata_alloc_pd()
610 	if (!pd->reorder_list)  in padata_alloc_pd()
613 	pd->squeue = alloc_percpu(struct padata_serial_queue);  in padata_alloc_pd()
614 	if (!pd->squeue)  in padata_alloc_pd()
617 	pd->ps = ps;  in padata_alloc_pd()
619 	if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))  in padata_alloc_pd()
621 	if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL))  in padata_alloc_pd()
624 	cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask);  in padata_alloc_pd()
625 	cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask);  in padata_alloc_pd()
627 	padata_init_reorder_list(pd);  in padata_alloc_pd()
628 	padata_init_squeues(pd);  in padata_alloc_pd()
629 	pd->seq_nr = -1;  in padata_alloc_pd()
630 	refcount_set(&pd->refcnt, 1);  in padata_alloc_pd()
631 	spin_lock_init(&pd->lock);  in padata_alloc_pd()
632 	pd->cpu = cpumask_first(pd->cpumask.pcpu);  in padata_alloc_pd()
633 	INIT_WORK(&pd->reorder_work, invoke_padata_reorder);  in padata_alloc_pd()
635 	return pd;  in padata_alloc_pd()
638 	free_cpumask_var(pd->cpumask.pcpu);  in padata_alloc_pd()
640 	free_percpu(pd->squeue);  in padata_alloc_pd()
642 	free_percpu(pd->reorder_list);  in padata_alloc_pd()
644 	kfree(pd);  in padata_alloc_pd()
649 static void padata_free_pd(struct parallel_data *pd)  in padata_free_pd()  argument
651 	free_cpumask_var(pd->cpumask.pcpu);  in padata_free_pd()
652 	free_cpumask_var(pd->cpumask.cbcpu);  in padata_free_pd()
653 	free_percpu(pd->reorder_list);  in padata_free_pd()
654 	free_percpu(pd->squeue);  in padata_free_pd()
655 	kfree(pd);  in padata_free_pd()
682 	ps->opd = rcu_dereference_protected(ps->pd, 1);  in padata_replace_one()
683 	rcu_assign_pointer(ps->pd, pd_new);  in padata_replace_one()
1101 	struct parallel_data *pd;  in padata_alloc_shell()  local
1111 	pd = padata_alloc_pd(ps);  in padata_alloc_shell()
1114 	if (!pd)  in padata_alloc_shell()
1118 	RCU_INIT_POINTER(ps->pd, pd);  in padata_alloc_shell()
1138 	struct parallel_data *pd;  in padata_free_shell()  local
1145 	 * freed pd's and ps's.  in padata_free_shell()
1151 	pd = rcu_dereference_protected(ps->pd, 1);  in padata_free_shell()
1152 	padata_put_pd(pd);  in padata_free_shell()