392 static struct rcu_torture_ops *cur_ops;  variable
424 		started = cur_ops->get_gp_seq();  in rcu_read_delay()
430 		completed = cur_ops->get_gp_seq();  in rcu_read_delay()
431 		do_trace_rcu_torture_read(cur_ops->name, NULL, ts,  in rcu_read_delay()
508 		cur_ops->deferred_free(rp);  in rcu_torture_cb()
631 	if (cur_ops == &srcud_ops)  in srcu_torture_read_lock()
658 	if (cur_ops == &srcud_ops)  in srcu_torture_read_unlock()
994 	if (!cur_ops->gp_diff)  in rcutorture_seq_diff()
996 	return cur_ops->gp_diff(new, old);  in rcutorture_seq_diff()
1046 		if (cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost_failed()
1048 		if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) {  in rcu_torture_boost_failed()
1058 		if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) {  in rcu_torture_boost_failed()
1061 			cur_ops->gp_kthread_dbg();  in rcu_torture_boost_failed()
1063 			gp_done = cur_ops->poll_gp_state(gp_state);  in rcu_torture_boost_failed()
1070 	} else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) {  in rcu_torture_boost_failed()
1111 			if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost()
1114 			if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) {  in rcu_torture_boost()
1115 				gp_state = cur_ops->start_gp_poll();  in rcu_torture_boost()
1124 				if (cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost()
1132 		if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))  in rcu_torture_boost()
1191 			cur_ops->fqs();  in rcu_torture_fqs()
1240 	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {  in rcu_torture_write_types()
1243 	} else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {  in rcu_torture_write_types()
1246 	if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) {  in rcu_torture_write_types()
1249 	} else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {  in rcu_torture_write_types()
1252 	if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) {  in rcu_torture_write_types()
1255 	} else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {  in rcu_torture_write_types()
1258 	if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {  in rcu_torture_write_types()
1262 		   (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {  in rcu_torture_write_types()
1265 	if (gp_exp1 && cur_ops->exp_sync) {  in rcu_torture_write_types()
1268 	} else if (gp_exp && !cur_ops->exp_sync) {  in rcu_torture_write_types()
1271 	if (gp_normal1 && cur_ops->deferred_free) {  in rcu_torture_write_types()
1274 	} else if (gp_normal && !cur_ops->deferred_free) {  in rcu_torture_write_types()
1277 	if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state &&  in rcu_torture_write_types()
1278 	    cur_ops->start_gp_poll && cur_ops->poll_gp_state) {  in rcu_torture_write_types()
1281 	} else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {  in rcu_torture_write_types()
1284 	if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full  in rcu_torture_write_types()
1285 	    && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {  in rcu_torture_write_types()
1288 	} else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {  in rcu_torture_write_types()
1291 	if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {  in rcu_torture_write_types()
1294 	} else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {  in rcu_torture_write_types()
1297 	if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {  in rcu_torture_write_types()
1301 		   (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {  in rcu_torture_write_types()
1304 	if (gp_sync1 && cur_ops->sync) {  in rcu_torture_write_types()
1307 	} else if (gp_sync && !cur_ops->sync) {  in rcu_torture_write_types()
1325 	dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300);  in do_rtws_sync()
1326 	dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00);  in do_rtws_sync()
1330 		cookie = cur_ops->get_gp_state();  in do_rtws_sync()
1332 		cur_ops->get_gp_state_full(&cookie_full);  in do_rtws_sync()
1333 	if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full))  in do_rtws_sync()
1336 	WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),  in do_rtws_sync()
1339 	WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full),  in do_rtws_sync()
1377 			 torture_type, cur_ops->name);  in rcu_torture_writer()
1414 			if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {  in rcu_torture_writer()
1415 				idx = cur_ops->readlock();  in rcu_torture_writer()
1416 				cookie = cur_ops->get_gp_state();  in rcu_torture_writer()
1417 				WARN_ONCE(cur_ops->poll_gp_state(cookie),  in rcu_torture_writer()
1422 					  cookie, cur_ops->get_gp_state());  in rcu_torture_writer()
1423 				if (cur_ops->get_gp_completed) {  in rcu_torture_writer()
1424 					cookie = cur_ops->get_gp_completed();  in rcu_torture_writer()
1425 					WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));  in rcu_torture_writer()
1427 				cur_ops->readunlock(idx);  in rcu_torture_writer()
1429 			if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) {  in rcu_torture_writer()
1430 				idx = cur_ops->readlock();  in rcu_torture_writer()
1431 				cur_ops->get_gp_state_full(&cookie_full);  in rcu_torture_writer()
1432 				WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),  in rcu_torture_writer()
1438 				if (cur_ops->get_gp_completed_full) {  in rcu_torture_writer()
1439 					cur_ops->get_gp_completed_full(&cookie_full);  in rcu_torture_writer()
1440 					WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));  in rcu_torture_writer()
1442 				cur_ops->readunlock(idx);  in rcu_torture_writer()
1447 				cur_ops->deferred_free(old_rp);  in rcu_torture_writer()
1451 				do_rtws_sync(&rand, cur_ops->exp_sync);  in rcu_torture_writer()
1456 				gp_snap = cur_ops->get_gp_state();  in rcu_torture_writer()
1459 				cur_ops->cond_sync(gp_snap);  in rcu_torture_writer()
1464 				gp_snap = cur_ops->get_gp_state_exp();  in rcu_torture_writer()
1467 				cur_ops->cond_sync_exp(gp_snap);  in rcu_torture_writer()
1472 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_writer()
1475 				cur_ops->cond_sync_full(&gp_snap_full);  in rcu_torture_writer()
1480 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_writer()
1483 				cur_ops->cond_sync_exp_full(&gp_snap_full);  in rcu_torture_writer()
1489 					ulo[i] = cur_ops->get_comp_state();  in rcu_torture_writer()
1490 				gp_snap = cur_ops->start_gp_poll();  in rcu_torture_writer()
1492 				while (!cur_ops->poll_gp_state(gp_snap)) {  in rcu_torture_writer()
1493 					gp_snap1 = cur_ops->get_gp_state();  in rcu_torture_writer()
1495 						if (cur_ops->poll_gp_state(ulo[i]) ||  in rcu_torture_writer()
1496 						    cur_ops->same_gp_state(ulo[i], gp_snap1)) {  in rcu_torture_writer()
1509 					cur_ops->get_comp_state_full(&rgo[i]);  in rcu_torture_writer()
1510 				cur_ops->start_gp_poll_full(&gp_snap_full);  in rcu_torture_writer()
1512 				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {  in rcu_torture_writer()
1513 					cur_ops->get_gp_state_full(&gp_snap1_full);  in rcu_torture_writer()
1515 						if (cur_ops->poll_gp_state_full(&rgo[i]) ||  in rcu_torture_writer()
1516 						    cur_ops->same_gp_state_full(&rgo[i],  in rcu_torture_writer()
1529 				gp_snap = cur_ops->start_gp_poll_exp();  in rcu_torture_writer()
1531 				while (!cur_ops->poll_gp_state_exp(gp_snap))  in rcu_torture_writer()
1538 				cur_ops->start_gp_poll_exp_full(&gp_snap_full);  in rcu_torture_writer()
1540 				while (!cur_ops->poll_gp_state_full(&gp_snap_full))  in rcu_torture_writer()
1547 				do_rtws_sync(&rand, cur_ops->sync);  in rcu_torture_writer()
1576 		    !cur_ops->slow_gps &&  in rcu_torture_writer()
1634 		if (cur_ops->cb_barrier != NULL &&  in rcu_torture_fakewriter()
1636 			cur_ops->cb_barrier();  in rcu_torture_fakewriter()
1642 				cur_ops->exp_sync();  in rcu_torture_fakewriter()
1645 				gp_snap = cur_ops->get_gp_state();  in rcu_torture_fakewriter()
1647 				cur_ops->cond_sync(gp_snap);  in rcu_torture_fakewriter()
1650 				gp_snap = cur_ops->get_gp_state_exp();  in rcu_torture_fakewriter()
1652 				cur_ops->cond_sync_exp(gp_snap);  in rcu_torture_fakewriter()
1655 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_fakewriter()
1657 				cur_ops->cond_sync_full(&gp_snap_full);  in rcu_torture_fakewriter()
1660 				cur_ops->get_gp_state_full(&gp_snap_full);  in rcu_torture_fakewriter()
1662 				cur_ops->cond_sync_exp_full(&gp_snap_full);  in rcu_torture_fakewriter()
1665 				gp_snap = cur_ops->start_gp_poll();  in rcu_torture_fakewriter()
1666 				while (!cur_ops->poll_gp_state(gp_snap)) {  in rcu_torture_fakewriter()
1672 				cur_ops->start_gp_poll_full(&gp_snap_full);  in rcu_torture_fakewriter()
1673 				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {  in rcu_torture_fakewriter()
1679 				gp_snap = cur_ops->start_gp_poll_exp();  in rcu_torture_fakewriter()
1680 				while (!cur_ops->poll_gp_state_exp(gp_snap)) {  in rcu_torture_fakewriter()
1686 				cur_ops->start_gp_poll_exp_full(&gp_snap_full);  in rcu_torture_fakewriter()
1687 				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {  in rcu_torture_fakewriter()
1693 				cur_ops->sync();  in rcu_torture_fakewriter()
1805 		idxnew1 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_1;  in rcutorture_one_extend()
1807 		idxnew2 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_2;  in rcutorture_one_extend()
1827 		cur_ops->readunlock((idxold2 >> RCUTORTURE_RDR_SHIFT_2) & 0x1);  in rcutorture_one_extend()
1834 		lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff);  in rcutorture_one_extend()
1837 		cur_ops->readunlock((idxold1 >> RCUTORTURE_RDR_SHIFT_1) & 0x1);  in rcutorture_one_extend()
1846 		cur_ops->read_delay(trsp, rtrsp);  in rcutorture_one_extend()
1870 	mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;  in rcutorture_extend_mask_max()
1974 		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)  in rcu_torture_one_read()
1975 			cookie = cur_ops->get_gp_state();  in rcu_torture_one_read()
1976 		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)  in rcu_torture_one_read()
1977 			cur_ops->get_gp_state_full(&cookie_full);  in rcu_torture_one_read()
1979 	started = cur_ops->get_gp_seq();  in rcu_torture_one_read()
1982 				  !cur_ops->readlock_held || cur_ops->readlock_held());  in rcu_torture_one_read()
1999 	completed = cur_ops->get_gp_seq();  in rcu_torture_one_read()
2001 		do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,  in rcu_torture_one_read()
2014 		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)  in rcu_torture_one_read()
2015 			WARN_ONCE(cur_ops->poll_gp_state(cookie),  in rcu_torture_one_read()
2020 				  cookie, cur_ops->get_gp_state());  in rcu_torture_one_read()
2021 		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)  in rcu_torture_one_read()
2022 			WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),  in rcu_torture_one_read()
2060 	if (cur_ops->call) {  in rcu_torture_timer()
2064 			cur_ops->call(rhp, rcu_torture_timer_cb);  in rcu_torture_timer()
2085 	if (irqreader && cur_ops->irq_capable)  in rcu_torture_reader()
2089 		if (irqreader && cur_ops->irq_capable) {  in rcu_torture_reader()
2103 	if (irqreader && cur_ops->irq_capable) {  in rcu_torture_reader()
2250 	if (cur_ops->stats)  in rcu_torture_stats_print()
2251 		cur_ops->stats();  in rcu_torture_stats_print()
2258 		rcutorture_get_gp_data(cur_ops->ttype,  in rcu_torture_stats_print()
2260 		srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp,  in rcu_torture_stats_print()
2272 		if (cur_ops->gp_kthread_dbg)  in rcu_torture_stats_print()
2273 			cur_ops->gp_kthread_dbg();  in rcu_torture_stats_print()
2347 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)  in rcu_torture_print_module_parms()  argument
2366 		 test_boost, cur_ops->can_boost,  in rcu_torture_print_module_parms()
2460 		idx = cur_ops->readlock();  in rcu_torture_stall()
2482 		cur_ops->readunlock(idx);  in rcu_torture_stall()
2518 	cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb);  in rcu_torture_fwd_prog_cb()
2601 	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();  in rcu_torture_fwd_cb_cr()
2667 	if (!cur_ops->sync)  in rcu_torture_fwd_prog_nr()
2669 	if (cur_ops->call && cur_ops->cb_barrier) {  in rcu_torture_fwd_prog_nr()
2676 	cur_ops->sync(); /* Later readers see above write. */  in rcu_torture_fwd_prog_nr()
2679 		cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);  in rcu_torture_fwd_prog_nr()
2682 	gps = cur_ops->get_gp_seq();  in rcu_torture_fwd_prog_nr()
2683 	sd = cur_ops->stall_dur() + 1;  in rcu_torture_fwd_prog_nr()
2691 		idx = cur_ops->readlock();  in rcu_torture_fwd_prog_nr()
2693 		cur_ops->readunlock(idx);  in rcu_torture_fwd_prog_nr()
2703 		gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);  in rcu_torture_fwd_prog_nr()
2710 		cur_ops->sync(); /* Wait for running CB to complete. */  in rcu_torture_fwd_prog_nr()
2711 		pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_nr()
2712 		cur_ops->cb_barrier(); /* Wait for queued callbacks. */  in rcu_torture_fwd_prog_nr()
2743 	if (!cur_ops->call)  in rcu_torture_fwd_prog_cr()
2748 	cur_ops->sync(); /* Later readers see above write. */  in rcu_torture_fwd_prog_cr()
2759 	gps = cur_ops->get_gp_seq();  in rcu_torture_fwd_prog_cr()
2776 		} else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) {  in rcu_torture_fwd_prog_cr()
2790 			cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);  in rcu_torture_fwd_prog_cr()
2801 	gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);  in rcu_torture_fwd_prog_cr()
2802 	pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);  in rcu_torture_fwd_prog_cr()
2803 	cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */  in rcu_torture_fwd_prog_cr()
2855 	cur_ops->cb_barrier();  in rcutorture_oom_notify()
2860 	cur_ops->cb_barrier();  in rcutorture_oom_notify()
2909 		if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) &&  in rcu_torture_fwd_prog()
2943 	if ((!cur_ops->sync && !cur_ops->call) ||  in rcu_torture_fwd_prog_init()
2944 	    (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) ||  in rcu_torture_fwd_prog_init()
2945 	    cur_ops == &rcu_busted_ops) {  in rcu_torture_fwd_prog_init()
3020 	cur_ops->call(rhp, rcu_torture_barrier_cbf);  in rcu_torture_barrier1cb()
3048 			cur_ops->call(&rcu, rcu_torture_barrier_cbf);  in rcu_torture_barrier_cbs()
3053 	if (cur_ops->cb_barrier != NULL)  in rcu_torture_barrier_cbs()
3054 		cur_ops->cb_barrier();  in rcu_torture_barrier_cbs()
3079 		cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */  in rcu_torture_barrier()
3092 				cur_ops->cb_barrier();  in rcu_torture_barrier()
3117 	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {  in rcu_torture_barrier_init()
3120 			 torture_type, cur_ops->name);  in rcu_torture_barrier_init()
3170 	if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2)  in rcu_torture_can_boost()
3172 	if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)  in rcu_torture_can_boost()
3311 		if (cur_ops->cb_barrier != NULL) {  in rcu_torture_cleanup()
3312 			pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);  in rcu_torture_cleanup()
3313 			cur_ops->cb_barrier();  in rcu_torture_cleanup()
3318 	if (!cur_ops) {  in rcu_torture_cleanup()
3326 	if (cur_ops->gp_kthread_dbg)  in rcu_torture_cleanup()
3327 		cur_ops->gp_kthread_dbg();  in rcu_torture_cleanup()
3359 	rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq);  in rcu_torture_cleanup()
3360 	srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq);  in rcu_torture_cleanup()
3362 		 cur_ops->name, (long)gp_seq, flags,  in rcu_torture_cleanup()
3373 	if (cur_ops->cb_barrier != NULL) {  in rcu_torture_cleanup()
3374 		pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);  in rcu_torture_cleanup()
3375 		cur_ops->cb_barrier();  in rcu_torture_cleanup()
3377 	if (cur_ops->cleanup != NULL)  in rcu_torture_cleanup()
3378 		cur_ops->cleanup();  in rcu_torture_cleanup()
3412 		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");  in rcu_torture_cleanup()
3414 		rcu_torture_print_module_parms(cur_ops,  in rcu_torture_cleanup()
3417 		rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");  in rcu_torture_cleanup()
3488 	if (cur_ops->sync && !(++n & 0xfff))  in rcutorture_sync()
3489 		cur_ops->sync();  in rcutorture_sync()
3693 		cur_ops = torture_ops[i];  in rcu_torture_init()
3694 		if (strcmp(torture_type, cur_ops->name) == 0)  in rcu_torture_init()
3705 		cur_ops = NULL;  in rcu_torture_init()
3708 	if (cur_ops->fqs == NULL && fqs_duration != 0) {  in rcu_torture_init()
3712 	if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops ||  in rcu_torture_init()
3715 			 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU));  in rcu_torture_init()
3718 	if (cur_ops->init)  in rcu_torture_init()
3719 		cur_ops->init();  in rcu_torture_init()
3730 	rcu_torture_print_module_parms(cur_ops, "Start of test");  in rcu_torture_init()
3731 	rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq);  in rcu_torture_init()
3732 	srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq);  in rcu_torture_init()
3735 		 cur_ops->name, (long)gp_seq, flags);  in rcu_torture_init()
3847 		t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;  in rcu_torture_init()