138 static fault_buffer_t *get_fault_buffer(test_globals_t* globals);
144 static uint64_t grab_and_fault_pages(test_globals_t* globals);
146 static bool worker_thread_iteration_setup(size_t current_iteration, test_globals_t *globals);
147 static void worker_thread_iteration_complete(test_globals_t *globals);
155 static pthread_t* setup_test(test_globals_t *globals, const test_args_t *args, size_t memory_size, …
158 static void init_globals(test_globals_t *globals, const test_args_t *args);
159 static inline _Atomic size_t *next_fault_buffer_index_ptr(test_globals_t *globals);
166 static uint64_t start_iteration(test_globals_t* globals, test_variant_t variant, bool verbose);
172 static uint64_t finish_iteration(test_globals_t *globals, uint64_t start_time);
177 static void setup_memory(test_globals_t* globals, test_variant_t variant);
182 static void output_results(const test_globals_t *globals, double walltime_elapsed_seconds, double c…
183 static void cleanup_test(test_globals_t *globals);
189 static uint64_t join_background_threads(test_globals_t *globals, pthread_t *threads);
190 static void unmap_fault_buffers(test_globals_t *globals);
194 static size_t fault_buffer_stride(const test_globals_t *globals);
206 	test_globals_t *globals = allocate_test_globals();  in main()  local
214 	pthread_t* threads = setup_test(globals, &args, kMemSize, args.verbose);  in main()
218 …benchmark_log(args.verbose, "----Starting Iteration %lu-----\n", globals->tg_current_iteration + 1…  in main()
219 		start_time_ns = start_iteration(globals, args.variant, args.verbose);  in main()
220 		wall_time_elapsed_ns += finish_iteration(globals, start_time_ns);  in main()
221 		benchmark_log(args.verbose, "----Completed Iteration %lu----\n", globals->tg_current_iteration);  in main()
225 	cpu_time_faulting_us = join_background_threads(globals, threads);  in main()
227 	output_results(globals, (double) wall_time_elapsed_ns / kNumNanosecondsInSecond,  in main()
229 	cleanup_test(globals);  in main()
239 	test_globals_t* globals = ((faulting_thread_args_t *)arg)->test_globals;  in faulting_thread()  local
243 	if (globals->pin_threads) {  in faulting_thread()
250 		bool should_continue = worker_thread_iteration_setup(current_iteration, globals);  in faulting_thread()
254 		on_cpu_time_faulting += grab_and_fault_pages(globals);  in faulting_thread()
255 		worker_thread_iteration_complete(globals);  in faulting_thread()
267 worker_thread_iteration_setup(size_t current_iteration, test_globals_t *globals)  in worker_thread_iteration_setup()  argument
272 	ret = pthread_mutex_lock(&globals->tg_lock);  in worker_thread_iteration_setup()
274 	globals->tg_running_count++;  in worker_thread_iteration_setup()
275 	if (globals->tg_running_count == globals->tg_num_threads) {  in worker_thread_iteration_setup()
278 		ret = pthread_cond_broadcast(&globals->tg_cv);  in worker_thread_iteration_setup()
286 	while (!globals->tg_done && globals->tg_current_iteration != current_iteration) {  in worker_thread_iteration_setup()
287 		ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in worker_thread_iteration_setup()
290 	should_continue = !globals->tg_done;  in worker_thread_iteration_setup()
291 	ret = pthread_mutex_unlock(&globals->tg_lock);  in worker_thread_iteration_setup()
301 worker_thread_iteration_complete(test_globals_t *globals)  in worker_thread_iteration_complete()  argument
305 	ret = pthread_mutex_lock(&globals->tg_lock);  in worker_thread_iteration_complete()
307 	globals->tg_running_count--;  in worker_thread_iteration_complete()
308 	if (globals->tg_running_count == 0) {  in worker_thread_iteration_complete()
310 		globals->tg_iterations_completed++;  in worker_thread_iteration_complete()
311 		ret = pthread_cond_broadcast(&globals->tg_cv);  in worker_thread_iteration_complete()
315 		while (globals->tg_iterations_completed != globals->tg_current_iteration) {  in worker_thread_iteration_complete()
316 			ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in worker_thread_iteration_complete()
320 	ret = pthread_mutex_unlock(&globals->tg_lock);  in worker_thread_iteration_complete()
334 get_fault_buffer(test_globals_t* globals)  in get_fault_buffer()  argument
336 …size_t index = atomic_fetch_add_explicit(next_fault_buffer_index_ptr(globals), 1UL, memory_order_a…  in get_fault_buffer()
337 	if (index < globals->tg_fault_buffer_arr_length) {  in get_fault_buffer()
338 		return &globals->tg_fault_buffer_arr[index];  in get_fault_buffer()
344 grab_and_fault_pages(test_globals_t* globals)  in grab_and_fault_pages()  argument
349 	size_t stride = fault_buffer_stride(globals) * kPageSize;  in grab_and_fault_pages()
351 		fault_buffer_t *object = get_fault_buffer(globals);  in grab_and_fault_pages()
368 start_iteration(test_globals_t* globals, test_variant_t variant, bool verbose)  in start_iteration()  argument
372 	ret = pthread_mutex_lock(&globals->tg_lock);  in start_iteration()
376 	while (globals->tg_running_count != globals->tg_num_threads) {  in start_iteration()
377 		ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in start_iteration()
380 	setup_memory(globals, variant);  in start_iteration()
384 	globals->tg_current_iteration++;  in start_iteration()
385 	ret = pthread_mutex_unlock(&globals->tg_lock);  in start_iteration()
387 	ret = pthread_cond_broadcast(&globals->tg_cv);  in start_iteration()
393 finish_iteration(test_globals_t* globals, uint64_t start_time)  in finish_iteration()  argument
397 	ret = pthread_mutex_lock(&globals->tg_lock);  in finish_iteration()
399 	while (globals->tg_iterations_completed != globals->tg_current_iteration) {  in finish_iteration()
400 		ret = pthread_cond_wait(&globals->tg_cv, &globals->tg_lock);  in finish_iteration()
403 	ret = pthread_mutex_unlock(&globals->tg_lock);  in finish_iteration()
404 	unmap_fault_buffers(globals);  in finish_iteration()
410 setup_memory(test_globals_t* globals, test_variant_t variant)  in setup_memory()  argument
412 	size_t stride = fault_buffer_stride(globals);  in setup_memory()
413 	for (size_t i = 0; i < globals->tg_fault_buffer_arr_length; i += stride) {  in setup_memory()
414 		fault_buffer_t *object = &globals->tg_fault_buffer_arr[i];  in setup_memory()
423 			for (size_t j = 1; j < globals->tg_num_threads; j++) {  in setup_memory()
425 				fault_buffer_t *offset_object = &globals->tg_fault_buffer_arr[i + j];  in setup_memory()
434 	atomic_store_explicit(next_fault_buffer_index_ptr(globals), 0, memory_order_release);  in setup_memory()
438 unmap_fault_buffers(test_globals_t* globals)  in unmap_fault_buffers()  argument
440 	size_t stride = fault_buffer_stride(globals);  in unmap_fault_buffers()
441 	for (size_t i = 0; i < globals->tg_fault_buffer_arr_length; i += stride) {  in unmap_fault_buffers()
442 		fault_buffer_t *buffer = &globals->tg_fault_buffer_arr[i];  in unmap_fault_buffers()
451 	test_globals_t *globals = NULL;  in allocate_test_globals()  local
466 	globals = malloc(test_globals_size);  in allocate_test_globals()
467 	assert(globals != NULL);  in allocate_test_globals()
468 	memset(globals, 0, test_globals_size);  in allocate_test_globals()
469 	return globals;  in allocate_test_globals()
473 init_globals(test_globals_t *globals, const test_args_t *args)  in init_globals()  argument
478 	memset(globals, 0, sizeof(test_globals_t));  in init_globals()
482 	ret = pthread_mutex_init(&globals->tg_lock, &mutex_attrs);  in init_globals()
486 	ret = pthread_cond_init(&globals->tg_cv, &cond_attrs);  in init_globals()
493 	globals->tg_num_threads = args->n_threads;  in init_globals()
494 	globals->tg_variant = args->variant;  in init_globals()
495 	globals->pin_threads = args->pin_threads;  in init_globals()
499 init_fault_buffer_arr(test_globals_t *globals, const test_args_t *args, size_t memory_size)  in init_fault_buffer_arr()  argument
503 		globals->tg_fault_buffer_arr_length = memory_size / kVmObjectSize;  in init_fault_buffer_arr()
507 		globals->tg_fault_buffer_arr_length = memory_size / kVmObjectSize * globals->tg_num_threads;  in init_fault_buffer_arr()
515 	assert(globals->tg_fault_buffer_arr_length >= globals->tg_num_threads);  in init_fault_buffer_arr()
516 	globals->tg_fault_buffer_arr = calloc(sizeof(fault_buffer_t), globals->tg_fault_buffer_arr_length);  in init_fault_buffer_arr()
517 	assert(globals->tg_fault_buffer_arr);  in init_fault_buffer_arr()
521 spawn_worker_threads(test_globals_t *globals, unsigned int num_threads, unsigned int first_cpu)  in spawn_worker_threads()  argument
525 	globals->tg_num_threads = num_threads;  in spawn_worker_threads()
533 		if (globals->pin_threads) {  in spawn_worker_threads()
536 		faulting_thread_args[i].test_globals = globals;  in spawn_worker_threads()
546 setup_test(test_globals_t *globals, const test_args_t *args, size_t memory_size, bool verbose)  in setup_test()  argument
548 	init_globals(globals, args);  in setup_test()
549 	init_fault_buffer_arr(globals, args, memory_size);  in setup_test()
551 	pthread_t *workers = spawn_worker_threads(globals, args->n_threads, args->first_cpu);  in setup_test()
557 join_background_threads(test_globals_t *globals, pthread_t *threads)  in join_background_threads()  argument
562 	ret = pthread_mutex_lock(&globals->tg_lock);  in join_background_threads()
564 	globals->tg_done = true;  in join_background_threads()
565 	ret = pthread_cond_broadcast(&globals->tg_cv);  in join_background_threads()
567 	ret = pthread_mutex_unlock(&globals->tg_lock);  in join_background_threads()
571 	for (unsigned int i = 0; i < globals->tg_num_threads; i++) {  in join_background_threads()
583 cleanup_test(test_globals_t* globals)  in cleanup_test()  argument
586 	ret = pthread_mutex_destroy(&globals->tg_lock);  in cleanup_test()
588 	ret = pthread_cond_destroy(&globals->tg_cv);  in cleanup_test()
590 	free(globals->tg_fault_buffer_arr);  in cleanup_test()
591 	free(globals);  in cleanup_test()
595 output_results(const test_globals_t* globals, double walltime_elapsed_seconds, double cputime_elaps…  in output_results()  argument
603 	size_t stride = fault_buffer_stride(globals);  in output_results()
604 	for (size_t i = 0; i < globals->tg_fault_buffer_arr_length; i += stride) {  in output_results()
605 		num_pages += globals->tg_fault_buffer_arr[i].fb_size / pgsize;  in output_results()
607 	num_pages *= globals->tg_iterations_completed;  in output_results()
679 next_fault_buffer_index_ptr(test_globals_t *globals)  in next_fault_buffer_index_ptr()  argument
681 	return (_Atomic size_t *) (((ptrdiff_t)(globals + 1)) + (int64_t)kCacheLineSize);  in next_fault_buffer_index_ptr()
684 fault_buffer_stride(const test_globals_t *globals)  in fault_buffer_stride()  argument
687 	if (globals->tg_variant == VARIANT_SEPARATE_VM_OBJECTS) {  in fault_buffer_stride()
689 	} else if (globals->tg_variant == VARIANT_SHARE_VM_OBJECTS) {  in fault_buffer_stride()
690 		stride = globals->tg_num_threads;  in fault_buffer_stride()