1 /* 2 * Copyright (c) 2023 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _VM_VM_PAGEOUT_XNU_H_ 30 #define _VM_VM_PAGEOUT_XNU_H_ 31 32 #include <sys/cdefs.h> 33 34 __BEGIN_DECLS 35 #include <vm/vm_pageout.h> 36 37 #ifdef XNU_KERNEL_PRIVATE 38 39 extern void memoryshot(unsigned int event, unsigned int control); 40 41 extern void update_vm_info(void); 42 43 44 45 #if CONFIG_IOSCHED 46 extern int upl_get_cached_tier( 47 upl_t upl); 48 #endif 49 50 extern void upl_set_iodone(upl_t, void *); 51 extern void upl_set_iodone_error(upl_t, int); 52 extern void upl_callout_iodone(upl_t); 53 54 extern ppnum_t upl_get_highest_page( 55 upl_t upl); 56 57 extern upl_t upl_associated_upl(upl_t upl); 58 extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl); 59 extern void upl_set_map_exclusive(upl_t upl); 60 extern void upl_clear_map_exclusive(upl_t upl); 61 62 63 #include <vm/vm_kern_xnu.h> 64 65 66 extern upl_size_t upl_adjusted_size( 67 upl_t upl, 68 vm_map_offset_t page_mask); 69 extern vm_object_offset_t upl_adjusted_offset( 70 upl_t upl, 71 vm_map_offset_t page_mask); 72 extern vm_object_offset_t upl_get_data_offset( 73 upl_t upl); 74 75 extern kern_return_t vm_map_create_upl( 76 vm_map_t map, 77 vm_map_address_t offset, 78 upl_size_t *upl_size, 79 upl_t *upl, 80 upl_page_info_array_t page_list, 81 unsigned int *count, 82 upl_control_flags_t *flags, 83 vm_tag_t tag); 84 85 extern void vm_page_free_list( 86 vm_page_t mem, 87 boolean_t prepare_object); 88 89 extern kern_return_t vm_page_alloc_list( 90 vm_size_t page_count, 91 kma_flags_t flags, 92 vm_page_t *list); 93 #if XNU_TARGET_OS_OSX 94 extern kern_return_t vm_pageout_wait(uint64_t deadline); 95 #endif /* XNU_TARGET_OS_OSX */ 96 97 98 #ifdef MACH_KERNEL_PRIVATE 99 100 #include <vm/vm_page.h> 101 102 extern unsigned int vm_pageout_scan_event_counter; 103 extern unsigned int vm_page_anonymous_count; 104 extern thread_t vm_pageout_scan_thread; 105 extern thread_t vm_pageout_gc_thread; 106 extern sched_cond_atomic_t vm_pageout_gc_cond; 107 108 /* 109 * must hold the page queues lock to 110 * manipulate this structure 111 */ 112 struct vm_pageout_queue { 113 vm_page_queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */ 114 unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */ 115 unsigned int pgo_maxlaundry; 116 117 uint32_t 118 pgo_busy:1, /* iothread is currently processing request from pgo_pending */ 119 pgo_throttled:1, /* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */ 120 pgo_lowpriority:1, /* iothread is set to use low priority I/O */ 121 pgo_draining:1, 122 pgo_inited:1, 123 pgo_unused_bits:26; 124 }; 125 126 #define VM_PAGE_Q_THROTTLED(q) \ 127 ((q)->pgo_laundry >= (q)->pgo_maxlaundry) 128 129 extern struct vm_pageout_queue vm_pageout_queue_internal; 130 extern struct vm_pageout_queue vm_pageout_queue_external; 131 132 /* 133 * This function is redeclared with slightly different parameter types in vfs_cluster.c 134 * This should be fixed at a later time. 135 */ 136 extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t); 137 138 /* 139 * Routines exported to Mach. 140 */ 141 extern void vm_pageout(void); 142 143 __startup_func extern void vm_config_init(void); 144 145 extern kern_return_t vm_pageout_internal_start(void); 146 147 extern void vm_pageout_object_terminate( 148 vm_object_t object); 149 150 extern void vm_pageout_cluster( 151 vm_page_t m); 152 153 extern void vm_pageout_initialize_page( 154 vm_page_t m); 155 156 157 struct _vector_upl_iostates { 158 upl_offset_t offset; 159 upl_size_t size; 160 }; 161 162 typedef struct _vector_upl_iostates vector_upl_iostates_t; 163 164 struct _vector_upl { 165 upl_size_t size; 166 uint32_t num_upls; 167 uint32_t invalid_upls; 168 uint32_t max_upls; 169 vm_map_t submap; 170 vm_offset_t submap_dst_addr; 171 vm_object_offset_t offset; 172 upl_page_info_array_t pagelist; 173 struct { 174 upl_t elem; 175 vector_upl_iostates_t iostate; 176 } upls[]; 177 }; 178 179 typedef struct _vector_upl* vector_upl_t; 180 181 /* universal page list structure */ 182 183 #if UPL_DEBUG 184 #define UPL_DEBUG_COMMIT_RECORDS 4 185 186 struct ucd { 187 upl_offset_t c_beg; 188 upl_offset_t c_end; 189 int c_aborted; 190 uint32_t c_btref; /* btref_t */ 191 }; 192 #endif 193 194 struct upl_io_completion { 195 void *io_context; 196 void (*io_done)(void *, int); 197 198 int io_error; 199 }; 200 201 202 struct upl { 203 decl_lck_mtx_data(, Lock); /* Synchronization */ 204 int ref_count; 205 int ext_ref_count; 206 int flags; 207 ctid_t map_addr_owner; /* owning thread for upl_map_range */ 208 /* 209 * XXX CAUTION: to accomodate devices with "mixed page sizes", 210 * u_offset and u_size are now byte-aligned and no longer 211 * page-aligned, on all devices. 212 */ 213 vm_object_offset_t u_offset; 214 upl_size_t u_size; /* size in bytes of the address space */ 215 upl_size_t u_mapped_size; /* size in bytes of the UPL that is mapped */ 216 vm_offset_t kaddr; /* secondary mapping in kernel */ 217 vm_object_t map_object; 218 vector_upl_t vector_upl; 219 upl_t associated_upl; 220 struct upl_io_completion *upl_iodone; 221 ppnum_t highest_page; 222 #if CONFIG_IOSCHED 223 int upl_priority; 224 uint64_t *upl_reprio_info; 225 void *decmp_io_upl; 226 #endif 227 #if CONFIG_IOSCHED || UPL_DEBUG 228 thread_t upl_creator; 229 queue_chain_t uplq; /* List of outstanding upls on an obj */ 230 #endif 231 #if UPL_DEBUG 232 uintptr_t ubc_alias1; 233 uintptr_t ubc_alias2; 234 235 uint32_t upl_state; 236 uint32_t upl_commit_index; 237 uint32_t upl_create_btref; /* btref_t */ 238 239 struct ucd upl_commit_records[UPL_DEBUG_COMMIT_RECORDS]; 240 #endif /* UPL_DEBUG */ 241 242 bitmap_t *lite_list; 243 struct upl_page_info page_list[]; 244 }; 245 246 /* upl struct flags */ 247 #define UPL_PAGE_LIST_MAPPED 0x1 248 #define UPL_KERNEL_MAPPED 0x2 249 #define UPL_CLEAR_DIRTY 0x4 250 #define UPL_COMPOSITE_LIST 0x8 251 #define UPL_INTERNAL 0x10 252 #define UPL_PAGE_SYNC_DONE 0x20 253 #define UPL_DEVICE_MEMORY 0x40 254 #define UPL_PAGEOUT 0x80 255 #define UPL_LITE 0x100 256 #define UPL_IO_WIRE 0x200 257 #define UPL_ACCESS_BLOCKED 0x400 258 #define UPL_SHADOWED 0x1000 259 #define UPL_KERNEL_OBJECT 0x2000 260 #define UPL_VECTOR 0x4000 261 #define UPL_SET_DIRTY 0x8000 262 #define UPL_HAS_BUSY 0x10000 263 #define UPL_TRACKED_BY_OBJECT 0x20000 264 #define UPL_EXPEDITE_SUPPORTED 0x40000 265 #define UPL_DECMP_REQ 0x80000 266 #define UPL_DECMP_REAL_IO 0x100000 267 #define UPL_MAP_EXCLUSIVE_WAIT 0x200000 268 #define UPL_HAS_WIRED 0x400000 269 270 /* flags for upl_create flags parameter */ 271 #define UPL_CREATE_EXTERNAL 0 272 #define UPL_CREATE_INTERNAL 0x1 273 #define UPL_CREATE_LITE 0x2 274 #define UPL_CREATE_IO_TRACKING 0x4 275 #define UPL_CREATE_EXPEDITE_SUP 0x8 276 277 extern void vector_upl_deallocate(upl_t); 278 extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t); 279 extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*); 280 extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*); 281 extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*); 282 extern upl_t vector_upl_subupl_byindex(upl_t, uint32_t); 283 extern upl_t vector_upl_subupl_byoffset(upl_t, upl_offset_t*, upl_size_t*); 284 285 286 extern void vm_page_free_reserve(int pages); 287 288 #endif /* MACH_KERNEL_PRIVATE */ 289 290 291 struct vm_pageout_state { 292 boolean_t vm_pressure_thread_running; 293 boolean_t vm_pressure_changed; 294 boolean_t vm_restricted_to_single_processor; 295 int vm_compressor_thread_count; 296 297 unsigned int vm_page_speculative_q_age_ms; 298 unsigned int vm_page_speculative_percentage; 299 unsigned int vm_page_speculative_target; 300 301 unsigned int vm_pageout_swap_wait; 302 unsigned int vm_pageout_idle_wait; /* milliseconds */ 303 unsigned int vm_pageout_empty_wait; /* milliseconds */ 304 unsigned int vm_pageout_burst_wait; /* milliseconds */ 305 unsigned int vm_pageout_deadlock_wait; /* milliseconds */ 306 unsigned int vm_pageout_deadlock_relief; 307 unsigned int vm_pageout_burst_inactive_throttle; 308 309 unsigned int vm_pageout_inactive; 310 unsigned int vm_pageout_inactive_used; /* debugging */ 311 unsigned int vm_pageout_inactive_clean; /* debugging */ 312 313 uint32_t vm_page_filecache_min; 314 uint32_t vm_page_filecache_min_divisor; 315 uint32_t vm_page_xpmapped_min; 316 uint32_t vm_page_xpmapped_min_divisor; 317 uint64_t vm_pageout_considered_page_last; 318 319 int vm_page_free_count_init; 320 321 unsigned int vm_memory_pressure; 322 323 int memorystatus_purge_on_critical; 324 int memorystatus_purge_on_warning; 325 int memorystatus_purge_on_urgent; 326 327 thread_t vm_pageout_early_swapout_iothread; 328 }; 329 330 extern struct vm_pageout_state vm_pageout_state; 331 332 /* 333 * This structure is used to track the VM_INFO instrumentation 334 */ 335 struct vm_pageout_vminfo { 336 unsigned long vm_pageout_considered_page; 337 unsigned long vm_pageout_considered_bq_internal; 338 unsigned long vm_pageout_considered_bq_external; 339 unsigned long vm_pageout_skipped_external; 340 unsigned long vm_pageout_skipped_internal; 341 342 unsigned long vm_pageout_pages_evicted; 343 unsigned long vm_pageout_pages_purged; 344 unsigned long vm_pageout_freed_cleaned; 345 unsigned long vm_pageout_freed_speculative; 346 unsigned long vm_pageout_freed_external; 347 unsigned long vm_pageout_freed_internal; 348 unsigned long vm_pageout_inactive_dirty_internal; 349 unsigned long vm_pageout_inactive_dirty_external; 350 unsigned long vm_pageout_inactive_referenced; 351 unsigned long vm_pageout_reactivation_limit_exceeded; 352 unsigned long vm_pageout_inactive_force_reclaim; 353 unsigned long vm_pageout_inactive_nolock; 354 unsigned long vm_pageout_filecache_min_reactivated; 355 unsigned long vm_pageout_scan_inactive_throttled_internal; 356 unsigned long vm_pageout_scan_inactive_throttled_external; 357 358 uint64_t vm_pageout_compressions; 359 uint64_t vm_compressor_pages_grabbed; 360 unsigned long vm_compressor_failed; 361 362 unsigned long vm_page_pages_freed; 363 364 unsigned long vm_phantom_cache_found_ghost; 365 unsigned long vm_phantom_cache_added_ghost; 366 367 unsigned long vm_pageout_protected_sharedcache; 368 unsigned long vm_pageout_forcereclaimed_sharedcache; 369 unsigned long vm_pageout_protected_realtime; 370 unsigned long vm_pageout_forcereclaimed_realtime; 371 372 }; 373 374 extern struct vm_pageout_vminfo vm_pageout_vminfo; 375 376 extern void vm_swapout_thread(void); 377 378 #if DEVELOPMENT || DEBUG 379 380 /* 381 * This structure records the pageout daemon's actions: 382 * how many pages it looks at and what happens to those pages. 383 * No locking needed because only one thread modifies the fields. 384 */ 385 struct vm_pageout_debug { 386 uint32_t vm_pageout_balanced; 387 uint32_t vm_pageout_scan_event_counter; 388 uint32_t vm_pageout_speculative_dirty; 389 390 uint32_t vm_pageout_inactive_busy; 391 uint32_t vm_pageout_inactive_absent; 392 uint32_t vm_pageout_inactive_notalive; 393 uint32_t vm_pageout_inactive_error; 394 uint32_t vm_pageout_inactive_deactivated; 395 396 uint32_t vm_pageout_enqueued_cleaned; 397 398 uint32_t vm_pageout_cleaned_busy; 399 uint32_t vm_pageout_cleaned_nolock; 400 uint32_t vm_pageout_cleaned_reference_reactivated; 401 uint32_t vm_pageout_cleaned_volatile_reactivated; 402 uint32_t vm_pageout_cleaned_reactivated; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */ 403 uint32_t vm_pageout_cleaned_fault_reactivated; 404 405 uint32_t vm_pageout_dirty_no_pager; 406 uint32_t vm_pageout_purged_objects; 407 408 uint32_t vm_pageout_scan_throttle; 409 uint32_t vm_pageout_scan_reclaimed_throttled; 410 uint32_t vm_pageout_scan_burst_throttle; 411 uint32_t vm_pageout_scan_empty_throttle; 412 uint32_t vm_pageout_scan_swap_throttle; 413 uint32_t vm_pageout_scan_deadlock_detected; 414 uint32_t vm_pageout_scan_inactive_throttle_success; 415 uint32_t vm_pageout_scan_throttle_deferred; 416 417 uint32_t vm_pageout_inactive_external_forced_jetsam_count; 418 419 uint32_t vm_grab_anon_overrides; 420 uint32_t vm_grab_anon_nops; 421 422 uint32_t vm_pageout_no_victim; 423 uint32_t vm_pageout_yield_for_free_pages; 424 unsigned long vm_pageout_throttle_up_count; 425 uint32_t vm_page_steal_pageout_page; 426 427 uint32_t vm_cs_validated_resets; 428 uint32_t vm_object_iopl_request_sleep_for_cleaning; 429 uint32_t vm_page_slide_counter; 430 uint32_t vm_page_slide_errors; 431 uint32_t vm_page_throttle_count; 432 /* 433 * Statistics about UPL enforcement of copy-on-write obligations. 434 */ 435 unsigned long upl_cow; 436 unsigned long upl_cow_again; 437 unsigned long upl_cow_pages; 438 unsigned long upl_cow_again_pages; 439 unsigned long iopl_cow; 440 unsigned long iopl_cow_pages; 441 }; 442 443 extern struct vm_pageout_debug vm_pageout_debug; 444 445 #define VM_PAGEOUT_DEBUG(member, value) \ 446 MACRO_BEGIN \ 447 vm_pageout_debug.member += value; \ 448 MACRO_END 449 #else /* DEVELOPMENT || DEBUG */ 450 #define VM_PAGEOUT_DEBUG(member, value) 451 #endif /* DEVELOPMENT || DEBUG */ 452 453 #define MAX_COMPRESSOR_THREAD_COUNT 8 454 455 /* 456 * Forward declarations for internal routines. 457 */ 458 459 /* 460 * Contains relevant state for pageout iothreads. Some state is unused by 461 * external (file-backed) thread. 462 */ 463 struct pgo_iothread_state { 464 struct vm_pageout_queue *q; 465 // cheads unused by external thread 466 void *current_early_swapout_chead; 467 void *current_regular_swapout_chead; 468 void *current_late_swapout_chead; 469 char *scratch_buf; 470 int id; 471 thread_t pgo_iothread; // holds a +1 ref 472 sched_cond_atomic_t pgo_wakeup; 473 #if DEVELOPMENT || DEBUG 474 // for perf_compressor benchmark 475 struct vm_pageout_queue *benchmark_q; 476 #endif /* DEVELOPMENT || DEBUG */ 477 }; 478 479 extern struct pgo_iothread_state pgo_iothread_internal_state[MAX_COMPRESSOR_THREAD_COUNT]; 480 481 extern struct pgo_iothread_state pgo_iothread_external_state; 482 483 struct vm_compressor_swapper_stats { 484 uint64_t unripe_under_30s; 485 uint64_t unripe_under_60s; 486 uint64_t unripe_under_300s; 487 uint64_t reclaim_swapins; 488 uint64_t defrag_swapins; 489 uint64_t compressor_swap_threshold_exceeded; 490 uint64_t external_q_throttled; 491 uint64_t free_count_below_reserve; 492 uint64_t thrashing_detected; 493 uint64_t fragmentation_detected; 494 }; 495 extern struct vm_compressor_swapper_stats vmcs_stats; 496 497 #if DEVELOPMENT || DEBUG 498 typedef struct vmct_stats_s { 499 uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT]; 500 uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT]; 501 uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT]; 502 // total mach absolute time that compressor threads has been running 503 uint64_t vmct_cthreads_total; 504 int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT]; 505 int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT]; 506 } vmct_stats_t; 507 508 kern_return_t 509 run_compressor_perf_test( 510 user_addr_t buf, 511 size_t buffer_size, 512 uint64_t *time, 513 uint64_t *bytes_compressed, 514 uint64_t *compressor_growth); 515 516 #endif /* DEVELOPMENT || DEBUG */ 517 518 #endif /* XNU_KERNEL_PRIVATE */ 519 __END_DECLS 520 521 #endif /* _VM_VM_PAGEOUT_XNU_H_ */ 522