187 	pair->param = (pair->param ? pair->param : def);		      \
188 	if (pair->param < low)						      \
190 	if (pair->param > hi)						      \
203 static inline void tst_fzsync_pair_init(struct tst_fzsync_pair *pair)  in tst_fzsync_pair_init()  argument
211 		pair->yield_in_wait = 1;  in tst_fzsync_pair_init()
222 static inline void tst_fzsync_pair_cleanup(struct tst_fzsync_pair *pair)  in tst_fzsync_pair_cleanup()  argument
224 	if (pair->thread_b) {  in tst_fzsync_pair_cleanup()
226 		if (!pair->exit)  in tst_fzsync_pair_cleanup()
227 			tst_atomic_store(1, &pair->exit);  in tst_fzsync_pair_cleanup()
228 		SAFE_PTHREAD_JOIN(pair->thread_b, NULL);  in tst_fzsync_pair_cleanup()
229 		pair->thread_b = 0;  in tst_fzsync_pair_cleanup()
262 static inline void tst_fzsync_pair_reset(struct tst_fzsync_pair *pair,  in tst_fzsync_pair_reset()  argument
265 	tst_fzsync_pair_cleanup(pair);  in tst_fzsync_pair_reset()
267 	tst_init_stat(&pair->diff_ss);  in tst_fzsync_pair_reset()
268 	tst_init_stat(&pair->diff_sa);  in tst_fzsync_pair_reset()
269 	tst_init_stat(&pair->diff_sb);  in tst_fzsync_pair_reset()
270 	tst_init_stat(&pair->diff_ab);  in tst_fzsync_pair_reset()
271 	tst_init_stat(&pair->spins_avg);  in tst_fzsync_pair_reset()
272 	pair->delay = 0;  in tst_fzsync_pair_reset()
273 	pair->delay_bias = 0;  in tst_fzsync_pair_reset()
274 	pair->sampling = pair->min_samples;  in tst_fzsync_pair_reset()
276 	pair->exec_loop = 0;  in tst_fzsync_pair_reset()
278 	pair->a_cntr = 0;  in tst_fzsync_pair_reset()
279 	pair->b_cntr = 0;  in tst_fzsync_pair_reset()
280 	pair->exit = 0;  in tst_fzsync_pair_reset()
282 		SAFE_PTHREAD_CREATE(&pair->thread_b, 0, run_b, 0);  in tst_fzsync_pair_reset()
284 	pair->exec_time_start = (float)tst_remaining_runtime();  in tst_fzsync_pair_reset()
305 static inline void tst_fzsync_pair_info(struct tst_fzsync_pair *pair)  in tst_fzsync_pair_info()  argument
308 		pair->exec_loop, pair->delay_bias);  in tst_fzsync_pair_info()
309 	tst_fzsync_stat_info(pair->diff_ss, "ns", "start_a - start_b");  in tst_fzsync_pair_info()
310 	tst_fzsync_stat_info(pair->diff_sa, "ns", "end_a - start_a");  in tst_fzsync_pair_info()
311 	tst_fzsync_stat_info(pair->diff_sb, "ns", "end_b - start_b");  in tst_fzsync_pair_info()
312 	tst_fzsync_stat_info(pair->diff_ab, "ns", "end_a - end_b");  in tst_fzsync_pair_info()
313 	tst_fzsync_stat_info(pair->spins_avg, "  ", "spins");  in tst_fzsync_pair_info()
458 static inline void tst_fzsync_pair_update(struct tst_fzsync_pair *pair)  in tst_fzsync_pair_update()  argument
460 	float alpha = pair->avg_alpha;  in tst_fzsync_pair_update()
462 	float max_dev = pair->max_dev_ratio;  in tst_fzsync_pair_update()
465 	pair->delay = pair->delay_bias;  in tst_fzsync_pair_update()
467 	over_max_dev = pair->diff_ss.dev_ratio > max_dev  in tst_fzsync_pair_update()
468 		|| pair->diff_sa.dev_ratio > max_dev  in tst_fzsync_pair_update()
469 		|| pair->diff_sb.dev_ratio > max_dev  in tst_fzsync_pair_update()
470 		|| pair->diff_ab.dev_ratio > max_dev  in tst_fzsync_pair_update()
471 		|| pair->spins_avg.dev_ratio > max_dev;  in tst_fzsync_pair_update()
473 	if (pair->sampling > 0 || over_max_dev) {  in tst_fzsync_pair_update()
474 		tst_upd_diff_stat(&pair->diff_ss, alpha,  in tst_fzsync_pair_update()
475 				  pair->a_start, pair->b_start);  in tst_fzsync_pair_update()
476 		tst_upd_diff_stat(&pair->diff_sa, alpha,  in tst_fzsync_pair_update()
477 				  pair->a_end, pair->a_start);  in tst_fzsync_pair_update()
478 		tst_upd_diff_stat(&pair->diff_sb, alpha,  in tst_fzsync_pair_update()
479 				  pair->b_end, pair->b_start);  in tst_fzsync_pair_update()
480 		tst_upd_diff_stat(&pair->diff_ab, alpha,  in tst_fzsync_pair_update()
481 				  pair->a_end, pair->b_end);  in tst_fzsync_pair_update()
482 		tst_upd_stat(&pair->spins_avg, alpha, pair->spins);  in tst_fzsync_pair_update()
483 		if (pair->sampling > 0 && --pair->sampling == 0) {  in tst_fzsync_pair_update()
485 			tst_fzsync_pair_info(pair);  in tst_fzsync_pair_update()
487 	} else if (fabsf(pair->diff_ab.avg) >= 1) {  in tst_fzsync_pair_update()
488 		per_spin_time = fabsf(pair->diff_ab.avg) / MAX(pair->spins_avg.avg, 1.0f);  in tst_fzsync_pair_update()
489 		time_delay = drand48() * (pair->diff_sa.avg + pair->diff_sb.avg)  in tst_fzsync_pair_update()
490 			- pair->diff_sb.avg;  in tst_fzsync_pair_update()
491 		pair->delay += (int)(1.1 * time_delay / per_spin_time);  in tst_fzsync_pair_update()
493 		if (!pair->sampling) {  in tst_fzsync_pair_update()
496 				pair->max_dev_ratio);  in tst_fzsync_pair_update()
498 				-(int)(pair->diff_sb.avg / per_spin_time) + pair->delay_bias,  in tst_fzsync_pair_update()
499 				(int)(pair->diff_sa.avg / per_spin_time) + pair->delay_bias);  in tst_fzsync_pair_update()
500 			tst_fzsync_pair_info(pair);  in tst_fzsync_pair_update()
501 			pair->sampling = -1;  in tst_fzsync_pair_update()
503 	} else if (!pair->sampling) {  in tst_fzsync_pair_update()
505 		tst_fzsync_pair_info(pair);  in tst_fzsync_pair_update()
506 		pair->sampling = -1;  in tst_fzsync_pair_update()
509 	pair->spins = 0;  in tst_fzsync_pair_update()
606 static inline void tst_fzsync_wait_a(struct tst_fzsync_pair *pair)  in tst_fzsync_wait_a()  argument
608 	tst_fzsync_pair_wait(&pair->a_cntr, &pair->b_cntr,  in tst_fzsync_wait_a()
609 			     NULL, &pair->exit, pair->yield_in_wait);  in tst_fzsync_wait_a()
618 static inline void tst_fzsync_wait_b(struct tst_fzsync_pair *pair)  in tst_fzsync_wait_b()  argument
620 	tst_fzsync_pair_wait(&pair->b_cntr, &pair->a_cntr,  in tst_fzsync_wait_b()
621 			     NULL, &pair->exit, pair->yield_in_wait);  in tst_fzsync_wait_b()
635 static inline int tst_fzsync_run_a(struct tst_fzsync_pair *pair)  in tst_fzsync_run_a()  argument
637 	float rem_p = 1 - tst_remaining_runtime() / pair->exec_time_start;  in tst_fzsync_run_a()
639 	if ((SAMPLING_SLICE < rem_p) && (pair->sampling > 0)) {  in tst_fzsync_run_a()
642 			pair->exec_loop, pair->min_samples);  in tst_fzsync_run_a()
643 		pair->sampling = 0;  in tst_fzsync_run_a()
644 		tst_fzsync_pair_info(pair);  in tst_fzsync_run_a()
650 		tst_atomic_store(1, &pair->exit);  in tst_fzsync_run_a()
653 	if (++pair->exec_loop > pair->exec_loops) {  in tst_fzsync_run_a()
656 		tst_atomic_store(1, &pair->exit);  in tst_fzsync_run_a()
659 	tst_fzsync_wait_a(pair);  in tst_fzsync_run_a()
661 	if (pair->exit) {  in tst_fzsync_run_a()
662 		tst_fzsync_pair_cleanup(pair);  in tst_fzsync_run_a()
675 static inline int tst_fzsync_run_b(struct tst_fzsync_pair *pair)  in tst_fzsync_run_b()  argument
677 	tst_fzsync_wait_b(pair);  in tst_fzsync_run_b()
678 	return !tst_atomic_load(&pair->exit);  in tst_fzsync_run_b()
699 static inline void tst_fzsync_start_race_a(struct tst_fzsync_pair *pair)  in tst_fzsync_start_race_a()  argument
703 	tst_fzsync_pair_update(pair);  in tst_fzsync_start_race_a()
705 	tst_fzsync_wait_a(pair);  in tst_fzsync_start_race_a()
707 	delay = pair->delay;  in tst_fzsync_start_race_a()
708 	if (pair->yield_in_wait) {  in tst_fzsync_start_race_a()
718 	tst_fzsync_time(&pair->a_start);  in tst_fzsync_start_race_a()
727 static inline void tst_fzsync_end_race_a(struct tst_fzsync_pair *pair)  in tst_fzsync_end_race_a()  argument
729 	tst_fzsync_time(&pair->a_end);  in tst_fzsync_end_race_a()
730 	tst_fzsync_pair_wait(&pair->a_cntr, &pair->b_cntr,  in tst_fzsync_end_race_a()
731 			     &pair->spins, &pair->exit, pair->yield_in_wait);  in tst_fzsync_end_race_a()
740 static inline void tst_fzsync_start_race_b(struct tst_fzsync_pair *pair)  in tst_fzsync_start_race_b()  argument
744 	tst_fzsync_wait_b(pair);  in tst_fzsync_start_race_b()
746 	delay = pair->delay;  in tst_fzsync_start_race_b()
747 	if (pair->yield_in_wait) {  in tst_fzsync_start_race_b()
757 	tst_fzsync_time(&pair->b_start);  in tst_fzsync_start_race_b()
766 static inline void tst_fzsync_end_race_b(struct tst_fzsync_pair *pair)  in tst_fzsync_end_race_b()  argument
768 	tst_fzsync_time(&pair->b_end);  in tst_fzsync_end_race_b()
769 	tst_fzsync_pair_wait(&pair->b_cntr, &pair->a_cntr,  in tst_fzsync_end_race_b()
770 			     &pair->spins, &pair->exit, pair->yield_in_wait);  in tst_fzsync_end_race_b()
797 static inline void tst_fzsync_pair_add_bias(struct tst_fzsync_pair *pair, int change)  in tst_fzsync_pair_add_bias()  argument
799 	if (pair->sampling > 0)  in tst_fzsync_pair_add_bias()
800 		pair->delay_bias += change;  in tst_fzsync_pair_add_bias()