1 /* 2 * Copyright (c) 2023 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _VM_VM_PAGEOUT_XNU_H_ 30 #define _VM_VM_PAGEOUT_XNU_H_ 31 32 #include <sys/cdefs.h> 33 34 __BEGIN_DECLS 35 #include <vm/vm_pageout.h> 36 37 #ifdef XNU_KERNEL_PRIVATE 38 39 extern void memoryshot(unsigned int event, unsigned int control); 40 41 extern void update_vm_info(void); 42 43 44 45 #if CONFIG_IOSCHED 46 extern int upl_get_cached_tier( 47 upl_t upl); 48 #endif 49 50 extern void upl_set_iodone(upl_t, void *); 51 extern void upl_set_iodone_error(upl_t, int); 52 extern void upl_callout_iodone(upl_t); 53 54 extern ppnum_t upl_get_highest_page( 55 upl_t upl); 56 57 extern upl_t upl_associated_upl(upl_t upl); 58 extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl); 59 extern void upl_set_map_exclusive(upl_t upl); 60 extern void upl_clear_map_exclusive(upl_t upl); 61 extern void upl_set_fs_verify_info(upl_t upl, uint32_t size_per_page); 62 extern bool upl_has_fs_verify_info(upl_t upl); 63 extern uint8_t * upl_fs_verify_buf(upl_t upl, uint32_t *size); 64 65 66 #include <vm/vm_kern_xnu.h> 67 68 69 extern upl_size_t upl_adjusted_size( 70 upl_t upl, 71 vm_map_offset_t page_mask); 72 extern vm_object_offset_t upl_adjusted_offset( 73 upl_t upl, 74 vm_map_offset_t page_mask); 75 extern vm_object_offset_t upl_get_data_offset( 76 upl_t upl); 77 78 extern kern_return_t vm_map_create_upl( 79 vm_map_t map, 80 vm_map_address_t offset, 81 upl_size_t *upl_size, 82 upl_t *upl, 83 upl_page_info_array_t page_list, 84 unsigned int *count, 85 upl_control_flags_t *flags, 86 vm_tag_t tag); 87 88 extern void vm_page_free_list( 89 vm_page_t mem, 90 bool prepare_object); 91 92 extern kern_return_t vm_page_alloc_list( 93 vm_size_t page_count, 94 kma_flags_t flags, 95 vm_page_t *list); 96 #if XNU_TARGET_OS_OSX 97 extern kern_return_t vm_pageout_wait(uint64_t deadline); 98 #endif /* XNU_TARGET_OS_OSX */ 99 100 101 #ifdef MACH_KERNEL_PRIVATE 102 103 #include <vm/vm_page.h> 104 105 extern unsigned int vm_pageout_scan_event_counter; 106 extern unsigned int vm_page_anonymous_count; 107 extern thread_t vm_pageout_scan_thread; 108 extern thread_t vm_pageout_gc_thread; 109 extern sched_cond_atomic_t vm_pageout_gc_cond; 110 111 /* 112 * must hold the page queues lock to 113 * manipulate this structure 114 */ 115 struct vm_pageout_queue { 116 vm_page_queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */ 117 unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */ 118 unsigned int pgo_maxlaundry; 119 120 uint32_t 121 pgo_busy:1, /* iothread is currently processing request from pgo_pending */ 122 pgo_throttled:1, /* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */ 123 pgo_lowpriority:1, /* iothread is set to use low priority I/O */ 124 pgo_draining:1, 125 pgo_inited:1, 126 pgo_unused_bits:26; 127 }; 128 129 #define VM_PAGE_Q_THROTTLED(q) \ 130 ((q)->pgo_laundry >= (q)->pgo_maxlaundry) 131 132 extern struct vm_pageout_queue vm_pageout_queue_internal; 133 extern struct vm_pageout_queue vm_pageout_queue_external; 134 135 /* 136 * This function is redeclared with slightly different parameter types in vfs_cluster.c 137 * This should be fixed at a later time. 138 */ 139 extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t); 140 141 /* 142 * Routines exported to Mach. 143 */ 144 extern void vm_pageout(void); 145 146 __startup_func extern void vm_config_init(void); 147 148 extern kern_return_t vm_pageout_internal_start(void); 149 150 extern void vm_pageout_object_terminate( 151 vm_object_t object); 152 153 extern void vm_pageout_cluster( 154 vm_page_t m); 155 156 extern void vm_pageout_initialize_page( 157 vm_page_t m); 158 159 160 struct _vector_upl_iostates { 161 upl_offset_t offset; 162 upl_size_t size; 163 }; 164 165 typedef struct _vector_upl_iostates vector_upl_iostates_t; 166 167 struct _vector_upl { 168 upl_size_t size; 169 uint32_t num_upls; 170 uint32_t invalid_upls; 171 uint32_t max_upls; 172 vm_offset_t dst_addr; 173 vm_object_offset_t offset; 174 upl_page_info_array_t pagelist; 175 struct { 176 upl_t elem; 177 vector_upl_iostates_t iostate; 178 } upls[]; 179 }; 180 181 typedef struct _vector_upl* vector_upl_t; 182 183 /* universal page list structure */ 184 185 #if UPL_DEBUG 186 #define UPL_DEBUG_COMMIT_RECORDS 4 187 188 struct ucd { 189 upl_offset_t c_beg; 190 upl_offset_t c_end; 191 int c_aborted; 192 uint32_t c_btref; /* btref_t */ 193 }; 194 #endif 195 196 struct upl_io_completion { 197 void *io_context; 198 void (*io_done)(void *, int); 199 200 int io_error; 201 }; 202 203 struct upl_fs_verify_info { 204 uint8_t *verify_data_ptr; /* verification data (hashes) for the data pages in the UPL */ 205 uint32_t verify_data_len; /* the digest size per page (can vary depending on the type of hash) */ 206 }; 207 208 struct upl { 209 decl_lck_mtx_data(, Lock); /* Synchronization */ 210 int ref_count; 211 int ext_ref_count; 212 int flags; 213 ctid_t map_addr_owner; /* owning thread for upl_map_range */ 214 /* 215 * XXX CAUTION: to accomodate devices with "mixed page sizes", 216 * u_offset and u_size are now byte-aligned and no longer 217 * page-aligned, on all devices. 218 */ 219 vm_object_offset_t u_offset; 220 upl_size_t u_size; /* size in bytes of the address space */ 221 upl_size_t u_mapped_size; /* size in bytes of the UPL that is mapped */ 222 vm_offset_t kaddr; /* secondary mapping in kernel */ 223 vm_object_t map_object; 224 vector_upl_t vector_upl; 225 union { 226 upl_t associated_upl; 227 struct upl_fs_verify_info *verify_info; /* verification data for the data pages in the UPL */ 228 } u_fs_un; 229 struct upl_io_completion *upl_iodone; 230 ppnum_t highest_page; 231 #if CONFIG_IOSCHED 232 int upl_priority; 233 uint64_t *upl_reprio_info; 234 void *decmp_io_upl; 235 #endif 236 #if CONFIG_IOSCHED || UPL_DEBUG 237 thread_t upl_creator; 238 queue_chain_t uplq; /* List of outstanding upls on an obj */ 239 #endif 240 #if UPL_DEBUG 241 uintptr_t ubc_alias1; 242 uintptr_t ubc_alias2; 243 244 uint32_t upl_state; 245 uint32_t upl_commit_index; 246 uint32_t upl_create_btref; /* btref_t */ 247 248 struct ucd upl_commit_records[UPL_DEBUG_COMMIT_RECORDS]; 249 #endif /* UPL_DEBUG */ 250 251 bitmap_t *lite_list; 252 struct upl_page_info page_list[]; 253 }; 254 255 /* upl struct flags */ 256 #define UPL_PAGE_LIST_MAPPED 0x00000001 257 #define UPL_KERNEL_MAPPED 0x00000002 258 #define UPL_CLEAR_DIRTY 0x00000004 259 #define UPL_COMPOSITE_LIST 0x00000008 260 #define UPL_INTERNAL 0x00000010 261 #define UPL_PAGE_SYNC_DONE 0x00000020 262 #define UPL_DEVICE_MEMORY 0x00000040 263 #define UPL_PAGEOUT 0x00000080 264 #define UPL_LITE 0x00000100 265 #define UPL_IO_WIRE 0x00000200 266 #define UPL_ACCESS_BLOCKED 0x00000400 267 #define UPL_PAGEIN 0x00000800 268 #define UPL_SHADOWED 0x00001000 269 #define UPL_KERNEL_OBJECT 0x00002000 270 #define UPL_VECTOR 0x00004000 271 #define UPL_SET_DIRTY 0x00008000 272 #define UPL_HAS_BUSY 0x00010000 273 #define UPL_TRACKED_BY_OBJECT 0x00020000 274 #define UPL_EXPEDITE_SUPPORTED 0x00040000 275 #define UPL_DECMP_REQ 0x00080000 276 #define UPL_DECMP_REAL_IO 0x00100000 277 #define UPL_MAP_EXCLUSIVE_WAIT 0x00200000 278 #define UPL_HAS_FS_VERIFY_INFO 0x00400000 279 #define UPL_HAS_WIRED 0x00800000 280 281 /* flags for upl_create flags parameter */ 282 #define UPL_CREATE_EXTERNAL 0 283 #define UPL_CREATE_INTERNAL 0x1 284 #define UPL_CREATE_LITE 0x2 285 #define UPL_CREATE_IO_TRACKING 0x4 286 #define UPL_CREATE_EXPEDITE_SUP 0x8 287 288 extern void vector_upl_deallocate(upl_t); 289 extern void vector_upl_set_addr(upl_t, vm_offset_t); 290 extern void vector_upl_get_addr(upl_t, vm_offset_t*); 291 extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*); 292 extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*); 293 extern upl_t vector_upl_subupl_byindex(upl_t, uint32_t); 294 extern upl_t vector_upl_subupl_byoffset(upl_t, upl_offset_t*, upl_size_t*); 295 296 297 extern void vm_page_free_reserve(int pages); 298 299 #endif /* MACH_KERNEL_PRIVATE */ 300 301 302 struct vm_pageout_state { 303 boolean_t vm_pressure_thread_running; 304 boolean_t vm_pressure_changed; 305 boolean_t vm_restricted_to_single_processor; 306 int vm_compressor_thread_count; 307 308 unsigned int vm_page_speculative_q_age_ms; 309 unsigned int vm_page_speculative_percentage; 310 unsigned int vm_page_speculative_target; 311 312 unsigned int vm_pageout_swap_wait; 313 unsigned int vm_pageout_idle_wait; /* milliseconds */ 314 unsigned int vm_pageout_empty_wait; /* milliseconds */ 315 unsigned int vm_pageout_burst_wait; /* milliseconds */ 316 unsigned int vm_pageout_deadlock_wait; /* milliseconds */ 317 unsigned int vm_pageout_deadlock_relief; 318 unsigned int vm_pageout_burst_inactive_throttle; 319 320 unsigned int vm_pageout_inactive; 321 unsigned int vm_pageout_inactive_used; /* debugging */ 322 unsigned int vm_pageout_inactive_clean; /* debugging */ 323 324 uint32_t vm_page_filecache_min; 325 uint32_t vm_page_filecache_min_divisor; 326 uint32_t vm_page_xpmapped_min; 327 uint32_t vm_page_xpmapped_min_divisor; 328 uint64_t vm_pageout_considered_page_last; 329 330 int vm_page_free_count_init; 331 332 unsigned int vm_memory_pressure; 333 334 int memorystatus_purge_on_critical; 335 int memorystatus_purge_on_warning; 336 int memorystatus_purge_on_urgent; 337 338 thread_t vm_pageout_early_swapout_iothread; 339 }; 340 341 extern struct vm_pageout_state vm_pageout_state; 342 343 /* 344 * This structure is used to track the VM_INFO instrumentation 345 */ 346 struct vm_pageout_vminfo { 347 unsigned long vm_pageout_considered_page; 348 unsigned long vm_pageout_considered_bq_internal; 349 unsigned long vm_pageout_considered_bq_external; 350 unsigned long vm_pageout_skipped_external; 351 unsigned long vm_pageout_skipped_internal; 352 353 unsigned long vm_pageout_pages_evicted; 354 unsigned long vm_pageout_pages_purged; 355 unsigned long vm_pageout_freed_cleaned; 356 unsigned long vm_pageout_freed_speculative; 357 unsigned long vm_pageout_freed_external; 358 unsigned long vm_pageout_freed_internal; 359 unsigned long vm_pageout_inactive_dirty_internal; 360 unsigned long vm_pageout_inactive_dirty_external; 361 unsigned long vm_pageout_inactive_referenced; 362 unsigned long vm_pageout_reactivation_limit_exceeded; 363 unsigned long vm_pageout_inactive_force_reclaim; 364 unsigned long vm_pageout_inactive_nolock; 365 unsigned long vm_pageout_filecache_min_reactivated; 366 unsigned long vm_pageout_scan_inactive_throttled_internal; 367 unsigned long vm_pageout_scan_inactive_throttled_external; 368 369 uint64_t vm_pageout_compressions; 370 uint64_t vm_compressor_pages_grabbed; 371 unsigned long vm_compressor_failed; 372 373 unsigned long vm_page_pages_freed; 374 375 unsigned long vm_phantom_cache_found_ghost; 376 unsigned long vm_phantom_cache_added_ghost; 377 378 unsigned long vm_pageout_protected_sharedcache; 379 unsigned long vm_pageout_forcereclaimed_sharedcache; 380 unsigned long vm_pageout_protected_realtime; 381 unsigned long vm_pageout_forcereclaimed_realtime; 382 383 uint64_t vm_compactor_major_compactions_completed; 384 uint64_t vm_compactor_major_compactions_considered; 385 uint64_t vm_compactor_major_compactions_bailed; 386 uint64_t vm_compactor_major_compaction_bytes_freed; 387 uint64_t vm_compactor_major_compaction_bytes_moved; 388 uint64_t vm_compactor_major_compaction_slots_moved; 389 uint64_t vm_compactor_major_compaction_segments_freed; 390 uint64_t vm_compactor_swapouts_queued; 391 uint64_t vm_compactor_swapout_bytes_wasted; 392 }; 393 394 extern struct vm_pageout_vminfo vm_pageout_vminfo; 395 396 extern void vm_swapout_thread(void); 397 398 #if DEVELOPMENT || DEBUG 399 400 /* 401 * This structure records the pageout daemon's actions: 402 * how many pages it looks at and what happens to those pages. 403 * No locking needed because only one thread modifies the fields. 404 */ 405 struct vm_pageout_debug { 406 uint32_t vm_pageout_balanced; 407 uint32_t vm_pageout_scan_event_counter; 408 uint32_t vm_pageout_speculative_dirty; 409 410 uint32_t vm_pageout_inactive_busy; 411 uint32_t vm_pageout_inactive_absent; 412 uint32_t vm_pageout_inactive_notalive; 413 uint32_t vm_pageout_inactive_error; 414 uint32_t vm_pageout_inactive_deactivated; 415 416 uint32_t vm_pageout_enqueued_cleaned; 417 418 uint32_t vm_pageout_cleaned_busy; 419 uint32_t vm_pageout_cleaned_nolock; 420 uint32_t vm_pageout_cleaned_reference_reactivated; 421 uint32_t vm_pageout_cleaned_volatile_reactivated; 422 uint32_t vm_pageout_cleaned_reactivated; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */ 423 uint32_t vm_pageout_cleaned_fault_reactivated; 424 425 uint32_t vm_pageout_dirty_no_pager; 426 uint32_t vm_pageout_purged_objects; 427 428 uint32_t vm_pageout_scan_throttle; 429 uint32_t vm_pageout_scan_reclaimed_throttled; 430 uint32_t vm_pageout_scan_burst_throttle; 431 uint32_t vm_pageout_scan_empty_throttle; 432 uint32_t vm_pageout_scan_swap_throttle; 433 uint32_t vm_pageout_scan_deadlock_detected; 434 uint32_t vm_pageout_scan_inactive_throttle_success; 435 uint32_t vm_pageout_scan_throttle_deferred; 436 437 uint32_t vm_pageout_inactive_external_forced_jetsam_count; 438 439 uint32_t vm_grab_anon_overrides; 440 uint32_t vm_grab_anon_nops; 441 442 uint32_t vm_pageout_no_victim; 443 uint32_t vm_pageout_yield_for_free_pages; 444 unsigned long vm_pageout_throttle_up_count; 445 uint32_t vm_page_steal_pageout_page; 446 447 uint32_t vm_cs_validated_resets; 448 uint32_t vm_object_iopl_request_sleep_for_cleaning; 449 uint32_t vm_page_slide_counter; 450 uint32_t vm_page_slide_errors; 451 uint32_t vm_page_throttle_count; 452 /* 453 * Statistics about UPL enforcement of copy-on-write obligations. 454 */ 455 unsigned long upl_cow; 456 unsigned long upl_cow_again; 457 unsigned long upl_cow_pages; 458 unsigned long upl_cow_again_pages; 459 unsigned long iopl_cow; 460 unsigned long iopl_cow_pages; 461 }; 462 463 extern struct vm_pageout_debug vm_pageout_debug; 464 465 #define VM_PAGEOUT_DEBUG(member, value) \ 466 MACRO_BEGIN \ 467 vm_pageout_debug.member += value; \ 468 MACRO_END 469 #else /* DEVELOPMENT || DEBUG */ 470 #define VM_PAGEOUT_DEBUG(member, value) 471 #endif /* DEVELOPMENT || DEBUG */ 472 473 #define MAX_COMPRESSOR_THREAD_COUNT 8 474 475 /* 476 * Forward declarations for internal routines. 477 */ 478 479 /* 480 * Contains relevant state for pageout iothreads. Some state is unused by 481 * external (file-backed) thread. 482 */ 483 struct pgo_iothread_state { 484 struct vm_pageout_queue *q; 485 // cheads unused by external thread 486 void *current_early_swapout_chead; 487 void *current_regular_swapout_cheads[COMPRESSOR_PAGEOUT_CHEADS_MAX_COUNT]; 488 void *current_late_swapout_chead; 489 char *scratch_buf; 490 int id; 491 thread_t pgo_iothread; // holds a +1 ref 492 sched_cond_atomic_t pgo_wakeup; 493 #if DEVELOPMENT || DEBUG 494 // for perf_compressor benchmark 495 struct vm_pageout_queue *benchmark_q; 496 #endif /* DEVELOPMENT || DEBUG */ 497 }; 498 499 extern struct pgo_iothread_state pgo_iothread_internal_state[MAX_COMPRESSOR_THREAD_COUNT]; 500 501 extern struct pgo_iothread_state pgo_iothread_external_state; 502 503 struct vm_compressor_swapper_stats { 504 uint64_t unripe_under_30s; 505 uint64_t unripe_under_60s; 506 uint64_t unripe_under_300s; 507 uint64_t reclaim_swapins; 508 uint64_t defrag_swapins; 509 uint64_t compressor_swap_threshold_exceeded; 510 uint64_t external_q_throttled; 511 uint64_t free_count_below_reserve; 512 uint64_t thrashing_detected; 513 uint64_t fragmentation_detected; 514 }; 515 extern struct vm_compressor_swapper_stats vmcs_stats; 516 517 #if DEVELOPMENT || DEBUG 518 typedef struct vmct_stats_s { 519 uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT]; 520 uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT]; 521 uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT]; 522 // total mach absolute time that compressor threads has been running 523 uint64_t vmct_cthreads_total; 524 int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT]; 525 int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT]; 526 } vmct_stats_t; 527 528 kern_return_t 529 run_compressor_perf_test( 530 user_addr_t buf, 531 size_t buffer_size, 532 uint64_t *time, 533 uint64_t *bytes_compressed, 534 uint64_t *compressor_growth); 535 536 #endif /* DEVELOPMENT || DEBUG */ 537 538 #endif /* XNU_KERNEL_PRIVATE */ 539 __END_DECLS 540 541 #endif /* _VM_VM_PAGEOUT_XNU_H_ */ 542