1 /* 2 * Copyright (c) 2023 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _VM_VM_PAGEOUT_XNU_H_ 30 #define _VM_VM_PAGEOUT_XNU_H_ 31 32 #include <sys/cdefs.h> 33 34 __BEGIN_DECLS 35 #include <vm/vm_pageout.h> 36 37 #ifdef XNU_KERNEL_PRIVATE 38 39 extern void memoryshot(unsigned int event, unsigned int control); 40 41 extern void update_vm_info(void); 42 43 44 45 #if CONFIG_IOSCHED 46 extern int upl_get_cached_tier( 47 upl_t upl); 48 #endif 49 50 extern void upl_set_iodone(upl_t, void *); 51 extern void upl_set_iodone_error(upl_t, int); 52 extern void upl_callout_iodone(upl_t); 53 54 extern ppnum_t upl_get_highest_page( 55 upl_t upl); 56 57 extern upl_t upl_associated_upl(upl_t upl); 58 extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl); 59 extern void upl_set_map_exclusive(upl_t upl); 60 extern void upl_clear_map_exclusive(upl_t upl); 61 62 63 #include <vm/vm_kern_xnu.h> 64 65 66 extern upl_size_t upl_adjusted_size( 67 upl_t upl, 68 vm_map_offset_t page_mask); 69 extern vm_object_offset_t upl_adjusted_offset( 70 upl_t upl, 71 vm_map_offset_t page_mask); 72 extern vm_object_offset_t upl_get_data_offset( 73 upl_t upl); 74 75 extern kern_return_t vm_map_create_upl( 76 vm_map_t map, 77 vm_map_address_t offset, 78 upl_size_t *upl_size, 79 upl_t *upl, 80 upl_page_info_array_t page_list, 81 unsigned int *count, 82 upl_control_flags_t *flags, 83 vm_tag_t tag); 84 85 extern void vm_page_free_list( 86 vm_page_t mem, 87 boolean_t prepare_object); 88 89 extern kern_return_t vm_page_alloc_list( 90 vm_size_t page_count, 91 kma_flags_t flags, 92 vm_page_t *list); 93 #if XNU_TARGET_OS_OSX 94 extern kern_return_t vm_pageout_wait(uint64_t deadline); 95 #endif /* XNU_TARGET_OS_OSX */ 96 97 98 #ifdef MACH_KERNEL_PRIVATE 99 100 #include <vm/vm_page.h> 101 102 extern unsigned int vm_pageout_scan_event_counter; 103 extern unsigned int vm_page_anonymous_count; 104 extern thread_t vm_pageout_scan_thread; 105 extern thread_t vm_pageout_gc_thread; 106 107 /* 108 * must hold the page queues lock to 109 * manipulate this structure 110 */ 111 struct vm_pageout_queue { 112 vm_page_queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */ 113 unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */ 114 unsigned int pgo_maxlaundry; 115 116 uint32_t 117 pgo_busy:1, /* iothread is currently processing request from pgo_pending */ 118 pgo_throttled:1, /* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */ 119 pgo_lowpriority:1, /* iothread is set to use low priority I/O */ 120 pgo_draining:1, 121 pgo_inited:1, 122 pgo_unused_bits:26; 123 }; 124 125 #define VM_PAGE_Q_THROTTLED(q) \ 126 ((q)->pgo_laundry >= (q)->pgo_maxlaundry) 127 128 extern struct vm_pageout_queue vm_pageout_queue_internal; 129 extern struct vm_pageout_queue vm_pageout_queue_external; 130 131 /* 132 * This function is redeclared with slightly different parameter types in vfs_cluster.c 133 * This should be fixed at a later time. 134 */ 135 extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t); 136 137 /* 138 * Routines exported to Mach. 139 */ 140 extern void vm_pageout(void); 141 142 __startup_func extern void vm_config_init(void); 143 144 extern kern_return_t vm_pageout_internal_start(void); 145 146 extern void vm_pageout_object_terminate( 147 vm_object_t object); 148 149 extern void vm_pageout_cluster( 150 vm_page_t m); 151 152 extern void vm_pageout_initialize_page( 153 vm_page_t m); 154 155 156 struct _vector_upl_iostates { 157 upl_offset_t offset; 158 upl_size_t size; 159 }; 160 161 typedef struct _vector_upl_iostates vector_upl_iostates_t; 162 163 struct _vector_upl { 164 upl_size_t size; 165 uint32_t num_upls; 166 uint32_t invalid_upls; 167 uint32_t max_upls; 168 vm_map_t submap; 169 vm_offset_t submap_dst_addr; 170 vm_object_offset_t offset; 171 upl_page_info_array_t pagelist; 172 struct { 173 upl_t elem; 174 vector_upl_iostates_t iostate; 175 } upls[]; 176 }; 177 178 typedef struct _vector_upl* vector_upl_t; 179 180 /* universal page list structure */ 181 182 #if UPL_DEBUG 183 #define UPL_DEBUG_COMMIT_RECORDS 4 184 185 struct ucd { 186 upl_offset_t c_beg; 187 upl_offset_t c_end; 188 int c_aborted; 189 uint32_t c_btref; /* btref_t */ 190 }; 191 #endif 192 193 struct upl_io_completion { 194 void *io_context; 195 void (*io_done)(void *, int); 196 197 int io_error; 198 }; 199 200 201 struct upl { 202 decl_lck_mtx_data(, Lock); /* Synchronization */ 203 int ref_count; 204 int ext_ref_count; 205 int flags; 206 ctid_t map_addr_owner; /* owning thread for upl_map_range */ 207 /* 208 * XXX CAUTION: to accomodate devices with "mixed page sizes", 209 * u_offset and u_size are now byte-aligned and no longer 210 * page-aligned, on all devices. 211 */ 212 vm_object_offset_t u_offset; 213 upl_size_t u_size; /* size in bytes of the address space */ 214 upl_size_t u_mapped_size; /* size in bytes of the UPL that is mapped */ 215 vm_offset_t kaddr; /* secondary mapping in kernel */ 216 vm_object_t map_object; 217 vector_upl_t vector_upl; 218 upl_t associated_upl; 219 struct upl_io_completion *upl_iodone; 220 ppnum_t highest_page; 221 #if CONFIG_IOSCHED 222 int upl_priority; 223 uint64_t *upl_reprio_info; 224 void *decmp_io_upl; 225 #endif 226 #if CONFIG_IOSCHED || UPL_DEBUG 227 thread_t upl_creator; 228 queue_chain_t uplq; /* List of outstanding upls on an obj */ 229 #endif 230 #if UPL_DEBUG 231 uintptr_t ubc_alias1; 232 uintptr_t ubc_alias2; 233 234 uint32_t upl_state; 235 uint32_t upl_commit_index; 236 uint32_t upl_create_btref; /* btref_t */ 237 238 struct ucd upl_commit_records[UPL_DEBUG_COMMIT_RECORDS]; 239 #endif /* UPL_DEBUG */ 240 241 bitmap_t *lite_list; 242 struct upl_page_info page_list[]; 243 }; 244 245 /* upl struct flags */ 246 #define UPL_PAGE_LIST_MAPPED 0x1 247 #define UPL_KERNEL_MAPPED 0x2 248 #define UPL_CLEAR_DIRTY 0x4 249 #define UPL_COMPOSITE_LIST 0x8 250 #define UPL_INTERNAL 0x10 251 #define UPL_PAGE_SYNC_DONE 0x20 252 #define UPL_DEVICE_MEMORY 0x40 253 #define UPL_PAGEOUT 0x80 254 #define UPL_LITE 0x100 255 #define UPL_IO_WIRE 0x200 256 #define UPL_ACCESS_BLOCKED 0x400 257 #define UPL_SHADOWED 0x1000 258 #define UPL_KERNEL_OBJECT 0x2000 259 #define UPL_VECTOR 0x4000 260 #define UPL_SET_DIRTY 0x8000 261 #define UPL_HAS_BUSY 0x10000 262 #define UPL_TRACKED_BY_OBJECT 0x20000 263 #define UPL_EXPEDITE_SUPPORTED 0x40000 264 #define UPL_DECMP_REQ 0x80000 265 #define UPL_DECMP_REAL_IO 0x100000 266 #define UPL_MAP_EXCLUSIVE_WAIT 0x200000 267 268 /* flags for upl_create flags parameter */ 269 #define UPL_CREATE_EXTERNAL 0 270 #define UPL_CREATE_INTERNAL 0x1 271 #define UPL_CREATE_LITE 0x2 272 #define UPL_CREATE_IO_TRACKING 0x4 273 #define UPL_CREATE_EXPEDITE_SUP 0x8 274 275 extern void vector_upl_deallocate(upl_t); 276 extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t); 277 extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*); 278 extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*); 279 extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*); 280 extern upl_t vector_upl_subupl_byindex(upl_t, uint32_t); 281 extern upl_t vector_upl_subupl_byoffset(upl_t, upl_offset_t*, upl_size_t*); 282 283 284 extern kern_return_t vm_object_iopl_request( 285 vm_object_t object, 286 vm_object_offset_t offset, 287 upl_size_t size, 288 upl_t *upl_ptr, 289 upl_page_info_array_t user_page_list, 290 unsigned int *page_list_count, 291 upl_control_flags_t cntrl_flags, 292 vm_tag_t tag); 293 294 extern kern_return_t vm_object_super_upl_request( 295 vm_object_t object, 296 vm_object_offset_t offset, 297 upl_size_t size, 298 upl_size_t super_cluster, 299 upl_t *upl, 300 upl_page_info_t *user_page_list, 301 unsigned int *page_list_count, 302 upl_control_flags_t cntrl_flags, 303 vm_tag_t tag); 304 305 306 307 308 309 extern void vm_page_free_reserve(int pages); 310 311 #endif /* MACH_KERNEL_PRIVATE */ 312 313 314 struct vm_pageout_state { 315 boolean_t vm_pressure_thread_running; 316 boolean_t vm_pressure_changed; 317 boolean_t vm_restricted_to_single_processor; 318 int vm_compressor_thread_count; 319 320 unsigned int vm_page_speculative_q_age_ms; 321 unsigned int vm_page_speculative_percentage; 322 unsigned int vm_page_speculative_target; 323 324 unsigned int vm_pageout_swap_wait; 325 unsigned int vm_pageout_idle_wait; /* milliseconds */ 326 unsigned int vm_pageout_empty_wait; /* milliseconds */ 327 unsigned int vm_pageout_burst_wait; /* milliseconds */ 328 unsigned int vm_pageout_deadlock_wait; /* milliseconds */ 329 unsigned int vm_pageout_deadlock_relief; 330 unsigned int vm_pageout_burst_inactive_throttle; 331 332 unsigned int vm_pageout_inactive; 333 unsigned int vm_pageout_inactive_used; /* debugging */ 334 unsigned int vm_pageout_inactive_clean; /* debugging */ 335 336 uint32_t vm_page_filecache_min; 337 uint32_t vm_page_filecache_min_divisor; 338 uint32_t vm_page_xpmapped_min; 339 uint32_t vm_page_xpmapped_min_divisor; 340 uint64_t vm_pageout_considered_page_last; 341 342 int vm_page_free_count_init; 343 344 unsigned int vm_memory_pressure; 345 346 int memorystatus_purge_on_critical; 347 int memorystatus_purge_on_warning; 348 int memorystatus_purge_on_urgent; 349 350 thread_t vm_pageout_early_swapout_iothread; 351 }; 352 353 extern struct vm_pageout_state vm_pageout_state; 354 355 /* 356 * This structure is used to track the VM_INFO instrumentation 357 */ 358 struct vm_pageout_vminfo { 359 unsigned long vm_pageout_considered_page; 360 unsigned long vm_pageout_considered_bq_internal; 361 unsigned long vm_pageout_considered_bq_external; 362 unsigned long vm_pageout_skipped_external; 363 unsigned long vm_pageout_skipped_internal; 364 365 unsigned long vm_pageout_pages_evicted; 366 unsigned long vm_pageout_pages_purged; 367 unsigned long vm_pageout_freed_cleaned; 368 unsigned long vm_pageout_freed_speculative; 369 unsigned long vm_pageout_freed_external; 370 unsigned long vm_pageout_freed_internal; 371 unsigned long vm_pageout_inactive_dirty_internal; 372 unsigned long vm_pageout_inactive_dirty_external; 373 unsigned long vm_pageout_inactive_referenced; 374 unsigned long vm_pageout_reactivation_limit_exceeded; 375 unsigned long vm_pageout_inactive_force_reclaim; 376 unsigned long vm_pageout_inactive_nolock; 377 unsigned long vm_pageout_filecache_min_reactivated; 378 unsigned long vm_pageout_scan_inactive_throttled_internal; 379 unsigned long vm_pageout_scan_inactive_throttled_external; 380 381 uint64_t vm_pageout_compressions; 382 uint64_t vm_compressor_pages_grabbed; 383 unsigned long vm_compressor_failed; 384 385 unsigned long vm_page_pages_freed; 386 387 unsigned long vm_phantom_cache_found_ghost; 388 unsigned long vm_phantom_cache_added_ghost; 389 390 unsigned long vm_pageout_protected_sharedcache; 391 unsigned long vm_pageout_forcereclaimed_sharedcache; 392 unsigned long vm_pageout_protected_realtime; 393 unsigned long vm_pageout_forcereclaimed_realtime; 394 }; 395 396 extern struct vm_pageout_vminfo vm_pageout_vminfo; 397 398 extern void vm_swapout_thread(void); 399 400 #if DEVELOPMENT || DEBUG 401 402 /* 403 * This structure records the pageout daemon's actions: 404 * how many pages it looks at and what happens to those pages. 405 * No locking needed because only one thread modifies the fields. 406 */ 407 struct vm_pageout_debug { 408 uint32_t vm_pageout_balanced; 409 uint32_t vm_pageout_scan_event_counter; 410 uint32_t vm_pageout_speculative_dirty; 411 412 uint32_t vm_pageout_inactive_busy; 413 uint32_t vm_pageout_inactive_absent; 414 uint32_t vm_pageout_inactive_notalive; 415 uint32_t vm_pageout_inactive_error; 416 uint32_t vm_pageout_inactive_deactivated; 417 418 uint32_t vm_pageout_enqueued_cleaned; 419 420 uint32_t vm_pageout_cleaned_busy; 421 uint32_t vm_pageout_cleaned_nolock; 422 uint32_t vm_pageout_cleaned_reference_reactivated; 423 uint32_t vm_pageout_cleaned_volatile_reactivated; 424 uint32_t vm_pageout_cleaned_reactivated; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */ 425 uint32_t vm_pageout_cleaned_fault_reactivated; 426 427 uint32_t vm_pageout_dirty_no_pager; 428 uint32_t vm_pageout_purged_objects; 429 430 uint32_t vm_pageout_scan_throttle; 431 uint32_t vm_pageout_scan_reclaimed_throttled; 432 uint32_t vm_pageout_scan_burst_throttle; 433 uint32_t vm_pageout_scan_empty_throttle; 434 uint32_t vm_pageout_scan_swap_throttle; 435 uint32_t vm_pageout_scan_deadlock_detected; 436 uint32_t vm_pageout_scan_inactive_throttle_success; 437 uint32_t vm_pageout_scan_throttle_deferred; 438 439 uint32_t vm_pageout_inactive_external_forced_jetsam_count; 440 441 uint32_t vm_grab_anon_overrides; 442 uint32_t vm_grab_anon_nops; 443 444 uint32_t vm_pageout_no_victim; 445 uint32_t vm_pageout_yield_for_free_pages; 446 unsigned long vm_pageout_throttle_up_count; 447 uint32_t vm_page_steal_pageout_page; 448 449 uint32_t vm_cs_validated_resets; 450 uint32_t vm_object_iopl_request_sleep_for_cleaning; 451 uint32_t vm_page_slide_counter; 452 uint32_t vm_page_slide_errors; 453 uint32_t vm_page_throttle_count; 454 /* 455 * Statistics about UPL enforcement of copy-on-write obligations. 456 */ 457 unsigned long upl_cow; 458 unsigned long upl_cow_again; 459 unsigned long upl_cow_pages; 460 unsigned long upl_cow_again_pages; 461 unsigned long iopl_cow; 462 unsigned long iopl_cow_pages; 463 }; 464 465 extern struct vm_pageout_debug vm_pageout_debug; 466 467 #define VM_PAGEOUT_DEBUG(member, value) \ 468 MACRO_BEGIN \ 469 vm_pageout_debug.member += value; \ 470 MACRO_END 471 #else /* DEVELOPMENT || DEBUG */ 472 #define VM_PAGEOUT_DEBUG(member, value) 473 #endif /* DEVELOPMENT || DEBUG */ 474 475 #define MAX_COMPRESSOR_THREAD_COUNT 8 476 477 /* 478 * Forward declarations for internal routines. 479 */ 480 481 /* 482 * Contains relevant state for pageout iothreads. Some state is unused by 483 * external (file-backed) thread. 484 */ 485 struct pgo_iothread_state { 486 struct vm_pageout_queue *q; 487 // cheads unused by external thread 488 void *current_early_swapout_chead; 489 void *current_regular_swapout_chead; 490 void *current_late_swapout_chead; 491 char *scratch_buf; 492 int id; 493 thread_t pgo_iothread; // holds a +1 ref 494 sched_cond_atomic_t pgo_wakeup; 495 #if DEVELOPMENT || DEBUG 496 // for perf_compressor benchmark 497 struct vm_pageout_queue *benchmark_q; 498 #endif /* DEVELOPMENT || DEBUG */ 499 }; 500 501 extern struct pgo_iothread_state pgo_iothread_internal_state[MAX_COMPRESSOR_THREAD_COUNT]; 502 503 extern struct pgo_iothread_state pgo_iothread_external_state; 504 505 struct vm_compressor_swapper_stats { 506 uint64_t unripe_under_30s; 507 uint64_t unripe_under_60s; 508 uint64_t unripe_under_300s; 509 uint64_t reclaim_swapins; 510 uint64_t defrag_swapins; 511 uint64_t compressor_swap_threshold_exceeded; 512 uint64_t external_q_throttled; 513 uint64_t free_count_below_reserve; 514 uint64_t thrashing_detected; 515 uint64_t fragmentation_detected; 516 }; 517 extern struct vm_compressor_swapper_stats vmcs_stats; 518 519 #if DEVELOPMENT || DEBUG 520 typedef struct vmct_stats_s { 521 uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT]; 522 uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT]; 523 uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT]; 524 // total mach absolute time that compressor threads has been running 525 uint64_t vmct_cthreads_total; 526 int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT]; 527 int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT]; 528 } vmct_stats_t; 529 530 kern_return_t 531 run_compressor_perf_test( 532 user_addr_t buf, 533 size_t buffer_size, 534 uint64_t *time, 535 uint64_t *bytes_compressed, 536 uint64_t *compressor_growth); 537 538 #endif /* DEVELOPMENT || DEBUG */ 539 540 #endif /* XNU_KERNEL_PRIVATE */ 541 __END_DECLS 542 543 #endif /* _VM_VM_PAGEOUT_XNU_H_ */ 544