139 static fault_buffer_t *get_fault_buffer(test_globals_t* globals);
145 static uint64_t grab_and_fault_pages(test_globals_t* globals);
147 static bool worker_thread_iteration_setup(size_t current_iteration, test_globals_t *globals);
148 static void worker_thread_iteration_complete(test_globals_t *globals);
156 static pthread_t* setup_test(test_globals_t *globals, const test_args_t *args, size_t memory_size, …
159 static void init_globals(test_globals_t *globals, const test_args_t *args);
160 static inline _Atomic size_t *next_fault_buffer_index_ptr(test_globals_t *globals);
167 static uint64_t start_iteration(test_globals_t* globals, test_variant_t variant, bool verbose);
173 static uint64_t finish_iteration(test_globals_t *globals, uint64_t start_time);
178 static void setup_memory(test_globals_t* globals, test_variant_t variant);
183 static void output_results(const test_globals_t *globals, double walltime_elapsed_seconds, double c…
184 static void cleanup_test(test_globals_t *globals);
190 static uint64_t join_background_threads(test_globals_t *globals, pthread_t *threads);
191 static void unmap_fault_buffers(test_globals_t *globals);
195 static size_t fault_buffer_stride(const test_globals_t *globals);
207 	test_globals_t *globals = allocate_test_globals();  in main()  local
215 	pthread_t* threads = setup_test(globals, &args, kMemSize, args.verbose);  in main()
219 …benchmark_log(args.verbose, "----Starting Iteration %lu-----\n", globals->tg_current_iteration + 1…  in main()
220 		start_time_ns = start_iteration(globals, args.variant, args.verbose);  in main()
221 		wall_time_elapsed_ns += finish_iteration(globals, start_time_ns);  in main()
222 		benchmark_log(args.verbose, "----Completed Iteration %lu----\n", globals->tg_current_iteration);  in main()
226 	cpu_time_faulting_us = join_background_threads(globals, threads);  in main()
228 	output_results(globals, (double) wall_time_elapsed_ns / kNumNanosecondsInSecond,  in main()
230 	cleanup_test(globals);  in main()
240 	test_globals_t* globals = ((faulting_thread_args_t *)arg)->test_globals;  in faulting_thread()  local
244 	if (globals->pin_threads) {  in faulting_thread()
251 		bool should_continue = worker_thread_iteration_setup(current_iteration, globals);  in faulting_thread()
255 		on_cpu_time_faulting += grab_and_fault_pages(globals);  in faulting_thread()
256 		worker_thread_iteration_complete(globals);  in faulting_thread()
268 worker_thread_iteration_setup(size_t current_iteration, test_globals_t *globals)  in worker_thread_iteration_setup()  argument
273 	ret = pthread_mutex_lock(&globals->tg_lock);  in worker_thread_iteration_setup()
275 	globals->tg_running_count++;  in worker_thread_iteration_setup()
276 	if (globals->tg_running_count == globals->tg_num_threads) {  in worker_thread_iteration_setup()
279 		ret = pthread_cond_broadcast(&globals->tg_cv);  in worker_thread_iteration_setup()
287 	while (!globals->tg_done && globals->tg_current_iteration != current_iteration) {  in worker_thread_iteration_setup()
288 		ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in worker_thread_iteration_setup()
291 	should_continue = !globals->tg_done;  in worker_thread_iteration_setup()
292 	ret = pthread_mutex_unlock(&globals->tg_lock);  in worker_thread_iteration_setup()
302 worker_thread_iteration_complete(test_globals_t *globals)  in worker_thread_iteration_complete()  argument
306 	ret = pthread_mutex_lock(&globals->tg_lock);  in worker_thread_iteration_complete()
308 	globals->tg_running_count--;  in worker_thread_iteration_complete()
309 	if (globals->tg_running_count == 0) {  in worker_thread_iteration_complete()
311 		globals->tg_iterations_completed++;  in worker_thread_iteration_complete()
312 		ret = pthread_cond_broadcast(&globals->tg_cv);  in worker_thread_iteration_complete()
316 		while (globals->tg_iterations_completed != globals->tg_current_iteration) {  in worker_thread_iteration_complete()
317 			ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in worker_thread_iteration_complete()
321 	ret = pthread_mutex_unlock(&globals->tg_lock);  in worker_thread_iteration_complete()
335 get_fault_buffer(test_globals_t* globals)  in get_fault_buffer()  argument
337 …size_t index = atomic_fetch_add_explicit(next_fault_buffer_index_ptr(globals), 1UL, memory_order_a…  in get_fault_buffer()
338 	if (index < globals->tg_fault_buffer_arr_length) {  in get_fault_buffer()
339 		return &globals->tg_fault_buffer_arr[index];  in get_fault_buffer()
345 grab_and_fault_pages(test_globals_t* globals)  in grab_and_fault_pages()  argument
350 	size_t stride = fault_buffer_stride(globals) * kPageSize;  in grab_and_fault_pages()
352 		fault_buffer_t *object = get_fault_buffer(globals);  in grab_and_fault_pages()
369 start_iteration(test_globals_t* globals, test_variant_t variant, bool verbose)  in start_iteration()  argument
373 	ret = pthread_mutex_lock(&globals->tg_lock);  in start_iteration()
377 	while (globals->tg_running_count != globals->tg_num_threads) {  in start_iteration()
378 		ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in start_iteration()
381 	setup_memory(globals, variant);  in start_iteration()
385 	globals->tg_current_iteration++;  in start_iteration()
386 	ret = pthread_mutex_unlock(&globals->tg_lock);  in start_iteration()
388 	ret = pthread_cond_broadcast(&globals->tg_cv);  in start_iteration()
394 finish_iteration(test_globals_t* globals, uint64_t start_time)  in finish_iteration()  argument
398 	ret = pthread_mutex_lock(&globals->tg_lock);  in finish_iteration()
400 	while (globals->tg_iterations_completed != globals->tg_current_iteration) {  in finish_iteration()
401 		ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in finish_iteration()
404 	ret = pthread_mutex_unlock(&globals->tg_lock);  in finish_iteration()
405 	unmap_fault_buffers(globals);  in finish_iteration()
411 setup_memory(test_globals_t* globals, test_variant_t variant)  in setup_memory()  argument
413 	size_t stride = fault_buffer_stride(globals);  in setup_memory()
414 	for (size_t i = 0; i < globals->tg_fault_buffer_arr_length; i += stride) {  in setup_memory()
415 		fault_buffer_t *object = &globals->tg_fault_buffer_arr[i];  in setup_memory()
424 			for (size_t j = 1; j < globals->tg_num_threads; j++) {  in setup_memory()
426 				fault_buffer_t *offset_object = &globals->tg_fault_buffer_arr[i + j];  in setup_memory()
435 	atomic_store_explicit(next_fault_buffer_index_ptr(globals), 0, memory_order_release);  in setup_memory()
439 unmap_fault_buffers(test_globals_t* globals)  in unmap_fault_buffers()  argument
441 	size_t stride = fault_buffer_stride(globals);  in unmap_fault_buffers()
442 	for (size_t i = 0; i < globals->tg_fault_buffer_arr_length; i += stride) {  in unmap_fault_buffers()
443 		fault_buffer_t *buffer = &globals->tg_fault_buffer_arr[i];  in unmap_fault_buffers()
452 	test_globals_t *globals = NULL;  in allocate_test_globals()  local
467 	globals = malloc(test_globals_size);  in allocate_test_globals()
468 	assert(globals != NULL);  in allocate_test_globals()
469 	memset(globals, 0, test_globals_size);  in allocate_test_globals()
470 	return globals;  in allocate_test_globals()
474 init_globals(test_globals_t *globals, const test_args_t *args)  in init_globals()  argument
479 	memset(globals, 0, sizeof(test_globals_t));  in init_globals()
483 	ret = pthread_mutex_init(&globals->tg_lock, &mutex_attrs);  in init_globals()
487 	ret = pthread_cond_init(&globals->tg_cv, &cond_attrs);  in init_globals()
494 	globals->tg_num_threads = args->n_threads;  in init_globals()
495 	globals->tg_variant = args->variant;  in init_globals()
496 	globals->pin_threads = args->pin_threads;  in init_globals()
500 init_fault_buffer_arr(test_globals_t *globals, const test_args_t *args, size_t memory_size)  in init_fault_buffer_arr()  argument
504 		globals->tg_fault_buffer_arr_length = memory_size / kVmObjectSize;  in init_fault_buffer_arr()
508 		globals->tg_fault_buffer_arr_length = memory_size / kVmObjectSize * globals->tg_num_threads;  in init_fault_buffer_arr()
516 	assert(globals->tg_fault_buffer_arr_length >= globals->tg_num_threads);  in init_fault_buffer_arr()
517 	globals->tg_fault_buffer_arr = calloc(sizeof(fault_buffer_t), globals->tg_fault_buffer_arr_length);  in init_fault_buffer_arr()
518 	assert(globals->tg_fault_buffer_arr);  in init_fault_buffer_arr()
522 spawn_worker_threads(test_globals_t *globals, unsigned int num_threads, unsigned int first_cpu)  in spawn_worker_threads()  argument
526 	globals->tg_num_threads = num_threads;  in spawn_worker_threads()
534 		if (globals->pin_threads) {  in spawn_worker_threads()
537 		faulting_thread_args[i].test_globals = globals;  in spawn_worker_threads()
547 setup_test(test_globals_t *globals, const test_args_t *args, size_t memory_size, bool verbose)  in setup_test()  argument
549 	init_globals(globals, args);  in setup_test()
550 	init_fault_buffer_arr(globals, args, memory_size);  in setup_test()
552 	pthread_t *workers = spawn_worker_threads(globals, args->n_threads, args->first_cpu);  in setup_test()
558 join_background_threads(test_globals_t *globals, pthread_t *threads)  in join_background_threads()  argument
563 	ret = pthread_mutex_lock(&globals->tg_lock);  in join_background_threads()
565 	globals->tg_done = true;  in join_background_threads()
566 	ret = pthread_cond_broadcast(&globals->tg_cv);  in join_background_threads()
568 	ret = pthread_mutex_unlock(&globals->tg_lock);  in join_background_threads()
572 	for (unsigned int i = 0; i < globals->tg_num_threads; i++) {  in join_background_threads()
584 cleanup_test(test_globals_t* globals)  in cleanup_test()  argument
587 	ret = pthread_mutex_destroy(&globals->tg_lock);  in cleanup_test()
589 	ret = pthread_cond_destroy(&globals->tg_cv);  in cleanup_test()
591 	free(globals->tg_fault_buffer_arr);  in cleanup_test()
592 	free(globals);  in cleanup_test()
596 output_results(const test_globals_t* globals, double walltime_elapsed_seconds, double cputime_elaps…  in output_results()  argument
604 	size_t stride = fault_buffer_stride(globals);  in output_results()
605 	for (size_t i = 0; i < globals->tg_fault_buffer_arr_length; i += stride) {  in output_results()
606 		num_pages += globals->tg_fault_buffer_arr[i].fb_size / pgsize;  in output_results()
608 	num_pages *= globals->tg_iterations_completed;  in output_results()
680 next_fault_buffer_index_ptr(test_globals_t *globals)  in next_fault_buffer_index_ptr()  argument
682 	return (_Atomic size_t *) (((ptrdiff_t)(globals + 1)) + (int64_t)kCacheLineSize);  in next_fault_buffer_index_ptr()
685 fault_buffer_stride(const test_globals_t *globals)  in fault_buffer_stride()  argument
688 	if (globals->tg_variant == VARIANT_SEPARATE_VM_OBJECTS) {  in fault_buffer_stride()
690 	} else if (globals->tg_variant == VARIANT_SHARE_VM_OBJECTS) {  in fault_buffer_stride()
691 		stride = globals->tg_num_threads;  in fault_buffer_stride()