1 /* 2 * Copyright (c) 2006-2019 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 * 28 */ 29 30 #ifndef _KERN_MEMORYSTATUS_INTERNAL_H_ 31 #define _KERN_MEMORYSTATUS_INTERNAL_H_ 32 33 /* 34 * Contains memorystatus subsystem definitions that are not 35 * exported outside of the memorystatus subsystem. 36 * 37 * For example, all of the mechanisms used by kern_memorystatus_policy.c 38 * should be defined in this header. 39 */ 40 41 #if BSD_KERNEL_PRIVATE 42 43 #include <mach/boolean.h> 44 #include <stdbool.h> 45 #include <os/base.h> 46 #include <os/log.h> 47 #include <kern/sched_prim.h> 48 49 #if CONFIG_FREEZE 50 #include <sys/kern_memorystatus_freeze.h> 51 #endif /* CONFIG_FREEZE */ 52 53 /* 54 * memorystatus subsystem globals 55 */ 56 #if CONFIG_JETSAM 57 extern unsigned int memorystatus_available_pages; 58 extern unsigned int memorystatus_available_pages_pressure; 59 extern unsigned int memorystatus_available_pages_critical; 60 extern uint32_t jetsam_kill_on_low_swap; 61 #else /* CONFIG_JETSAM */ 62 extern uint64_t memorystatus_available_pages; 63 extern uint64_t memorystatus_available_pages_pressure; 64 extern uint64_t memorystatus_available_pages_critical; 65 #endif /* CONFIG_JETSAM */ 66 extern int block_corpses; /* counter to block new corpses if jetsam purges them */ 67 extern int system_procs_aging_band; 68 extern int applications_aging_band; 69 /* 70 * TODO(jason): This should really be calculated dynamically by the zalloc 71 * subsystem before we do a zone map exhaustion kill. But the zone_gc 72 * logic is non-trivial, so for now it just sets this global. 73 */ 74 extern _Atomic bool memorystatus_zone_map_is_exhausted; 75 /* 76 * TODO(jason): We should get rid of this global 77 * and have the memorystatus thread check for compressor space shortages 78 * itself. However, there are 3 async call sites remaining that require more work to get us there: 79 * 2 of them are in vm_swap_defragment. When it's about to swap in a segment, it checks if that 80 * will cause a compressor space shortage & pre-emptively triggers jetsam. vm_compressor_backing_store 81 * needs to keep track of in-flight swapins due to defrag so we can perform those checks 82 * in the memorystatus thread. 83 * The other is in no_paging_space_action. This is only on macOS right now, but will 84 * be needed on iPad when we run out of swap space. This should be a new kill 85 * reason and we need to add a new health check for it. 86 * We need to maintain the macOS behavior though that we kill no more than 1 process 87 * every 5 seconds. 88 */ 89 extern _Atomic bool memorystatus_compressor_space_shortage; 90 /* 91 * TODO(jason): We should also get rid of this global 92 * and check for phantom cache pressure from the memorystatus 93 * thread. But first we need to fix the syncronization in 94 * vm_phantom_cache_check_pressure 95 */ 96 extern _Atomic bool memorystatus_phantom_cache_pressure; 97 98 extern _Atomic bool memorystatus_pageout_starved; 99 /* 100 * The actions that the memorystatus thread can perform 101 * when we're low on memory. 102 * See memorystatus_pick_action to see when each action is deployed. 103 */ 104 OS_CLOSED_ENUM(memorystatus_action, uint32_t, 105 MEMORYSTATUS_KILL_HIWATER, // Kill 1 highwatermark process 106 MEMORYSTATUS_KILL_AGGRESSIVE, // Do aggressive jetsam 107 MEMORYSTATUS_KILL_TOP_PROCESS, // Kill based on jetsam priority 108 MEMORYSTATUS_WAKE_SWAPPER, // Wake up the swap thread 109 MEMORYSTATUS_PROCESS_SWAPIN_QUEUE, // Compact the swapin queue and move segments to the swapout queue 110 MEMORYSTATUS_KILL_SUSPENDED_SWAPPABLE, // Kill a suspended swap-eligible processes based on jetsam priority 111 MEMORYSTATUS_KILL_SWAPPABLE, // Kill a swap-eligible process (even if it's running) based on jetsam priority 112 MEMORYSTATUS_KILL_NONE, // Do nothing 113 ); 114 115 /* 116 * Structure to hold state for a jetsam thread. 117 * Typically there should be a single jetsam thread 118 * unless parallel jetsam is enabled. 119 */ 120 typedef struct jetsam_thread_state { 121 uint8_t inited; /* boolean - if the thread is initialized */ 122 uint8_t limit_to_low_bands; /* boolean */ 123 int index; /* jetsam thread index */ 124 thread_t thread; /* jetsam thread pointer */ 125 int jld_idle_kills; /* idle jetsam kill counter for this session */ 126 uint32_t errors; /* Error accumulator */ 127 bool sort_flag; /* Sort the fg band (idle on macOS) before killing? */ 128 bool corpse_list_purged; /* Has the corpse list been purged? */ 129 bool post_snapshot; /* Do we need to post a jetsam snapshot after this session? */ 130 uint64_t memory_reclaimed; /* Amount of memory that was just reclaimed */ 131 uint32_t hwm_kills; /* hwm kill counter for this session */ 132 sched_cond_atomic_t jt_wakeup_cond; /* condition var used to synchronize wake/sleep operations for this jetsam thread */ 133 } jetsam_thread_state_t; 134 135 /* 136 * The memorystatus thread monitors these conditions 137 * and will continue to act until the system is considered 138 * healthy. 139 */ 140 typedef struct memorystatus_system_health { 141 #if CONFIG_JETSAM 142 bool msh_available_pages_below_pressure; 143 bool msh_available_pages_below_critical; 144 bool msh_compressor_needs_to_swap; 145 bool msh_compressor_is_low_on_space; 146 bool msh_compressor_is_thrashing; 147 bool msh_compressed_pages_nearing_limit; 148 bool msh_filecache_is_thrashing; 149 bool msh_phantom_cache_pressure; 150 bool msh_swappable_compressor_segments_over_limit; 151 bool msh_swapin_queue_over_limit; 152 bool msh_swap_low_on_space; 153 bool msh_swap_out_of_space; 154 bool msh_pageout_starved; 155 #endif /* CONFIG_JETSAM */ 156 bool msh_zone_map_is_exhausted; 157 } memorystatus_system_health_t; 158 159 void memorystatus_log_system_health(const memorystatus_system_health_t *health); 160 bool memorystatus_is_system_healthy(const memorystatus_system_health_t *status); 161 /* Picks a kill cause given an unhealthy system status */ 162 uint32_t memorystatus_pick_kill_cause(const memorystatus_system_health_t *status); 163 164 /* 165 * Agressive jetsam tunables 166 */ 167 #define kJetsamAgingPolicyNone (0) 168 #define kJetsamAgingPolicyLegacy (1) 169 #define kJetsamAgingPolicySysProcsReclaimedFirst (2) 170 #define kJetsamAgingPolicyAppsReclaimedFirst (3) 171 #define kJetsamAgingPolicyMax kJetsamAgingPolicyAppsReclaimedFirst 172 extern boolean_t memorystatus_jld_enabled; /* Enable jetsam loop detection */ 173 extern uint32_t memorystatus_jld_eval_period_msecs; /* Init pass sets this based on device memory size */ 174 extern int memorystatus_jld_eval_aggressive_count; /* Raise the priority max after 'n' aggressive loops */ 175 extern int memorystatus_jld_eval_aggressive_priority_band_max; /* Kill aggressively up through this band */ 176 extern int memorystatus_jld_max_kill_loops; /* How many times should we try and kill up to the target band */ 177 extern unsigned int memorystatus_sysproc_aging_aggr_pages; /* Aggressive jetsam pages threshold for sysproc aging policy */ 178 extern int jld_eval_aggressive_count; 179 extern int32_t jld_priority_band_max; 180 extern uint64_t jld_timestamp_msecs; 181 extern int jld_idle_kill_candidates; 182 183 184 /* 185 * VM globals read by the memorystatus subsystem 186 */ 187 extern unsigned int vm_page_free_count; 188 extern unsigned int vm_page_active_count; 189 extern unsigned int vm_page_inactive_count; 190 extern unsigned int vm_page_throttled_count; 191 extern unsigned int vm_page_purgeable_count; 192 extern unsigned int vm_page_wire_count; 193 extern unsigned int vm_page_speculative_count; 194 extern uint32_t c_late_swapout_count, c_late_swappedin_count; 195 extern uint32_t c_seg_allocsize; 196 extern bool vm_swapout_thread_running; 197 extern _Atomic bool vm_swapout_wake_pending; 198 #define VM_PAGE_DONATE_DISABLED 0 199 #define VM_PAGE_DONATE_ENABLED 1 200 extern uint32_t vm_page_donate_mode; 201 void vm_swapout_thread(void); 202 void vm_compressor_process_special_swapped_in_segments(void); 203 204 #if CONFIG_JETSAM 205 #define MEMORYSTATUS_LOG_AVAILABLE_PAGES memorystatus_available_pages 206 #else /* CONFIG_JETSAM */ 207 #define MEMORYSTATUS_LOG_AVAILABLE_PAGES (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count) 208 #endif /* CONFIG_JETSAM */ 209 210 bool memorystatus_avail_pages_below_pressure(void); 211 bool memorystatus_avail_pages_below_critical(void); 212 #if CONFIG_JETSAM 213 bool memorystatus_swap_over_trigger(uint64_t adjustment_factor); 214 bool memorystatus_swapin_over_trigger(void); 215 #endif /* CONFIG_JETSAM */ 216 217 /* Does cause indicate vm or fc thrashing? */ 218 bool is_reason_thrashing(unsigned cause); 219 /* Is the zone map almost full? */ 220 bool is_reason_zone_map_exhaustion(unsigned cause); 221 222 memorystatus_action_t memorystatus_pick_action(struct jetsam_thread_state *jetsam_thread, 223 uint32_t *kill_cause, bool highwater_remaining, 224 bool suspended_swappable_apps_remaining, 225 bool swappable_apps_remaining, int *jld_idle_kills); 226 227 #define MEMSTAT_PERCENT_TOTAL_PAGES(p) (p * atop_64(max_mem) / 100) 228 229 #pragma mark Logging Utilities 230 231 __enum_decl(memorystatus_log_level_t, unsigned int, { 232 MEMORYSTATUS_LOG_LEVEL_DEFAULT = 0, 233 MEMORYSTATUS_LOG_LEVEL_INFO = 1, 234 MEMORYSTATUS_LOG_LEVEL_DEBUG = 2, 235 }); 236 237 extern os_log_t memorystatus_log_handle; 238 extern memorystatus_log_level_t memorystatus_log_level; 239 240 /* 241 * NB: Critical memorystatus logs (e.g. jetsam kills) are load-bearing for OS 242 * performance testing infrastructure. Be careful when modifying the log-level for 243 * important system events. 244 * 245 * Memorystatus logs are interpreted by a wide audience. To avoid logging information 246 * that could lead to false diagnoses, INFO and DEBUG messages are only logged if the 247 * system has been configured to do so via `kern.memorystatus_log_level` (sysctl) or 248 * `memorystatus_log_level` (boot-arg). 249 * 250 * os_log supports a mechanism for configuring these properties dynamically; however, 251 * this mechanism is currently unsupported in XNU. 252 * 253 * TODO (JC) Deprecate sysctl/boot-arg and move to subsystem preferences pending: 254 * - rdar://27006343 (Custom kernel log handles) 255 * - rdar://80958044 (Kernel Logging Configuration) 256 */ 257 #define _memorystatus_log_with_type(type, format, ...) os_log_with_type(memorystatus_log_handle, type, format, ##__VA_ARGS__) 258 #define memorystatus_log(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_DEFAULT, format, ##__VA_ARGS__) 259 #define memorystatus_log_info(format, ...) if (memorystatus_log_level >= MEMORYSTATUS_LOG_LEVEL_INFO) { _memorystatus_log_with_type(OS_LOG_TYPE_INFO, format, ##__VA_ARGS__); } 260 #define memorystatus_log_debug(format, ...) if (memorystatus_log_level >= MEMORYSTATUS_LOG_LEVEL_DEBUG) { _memorystatus_log_with_type(OS_LOG_TYPE_DEBUG, format, ##__VA_ARGS__); } 261 #define memorystatus_log_error(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_ERROR, format, ##__VA_ARGS__) 262 #define memorystatus_log_fault(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_FAULT, format, ##__VA_ARGS__) 263 264 #pragma mark Freezer 265 #if CONFIG_FREEZE 266 /* 267 * Freezer data types 268 */ 269 270 /* An ordered list of freeze or demotion candidates */ 271 struct memorystatus_freezer_candidate_list { 272 memorystatus_properties_freeze_entry_v1 *mfcl_list; 273 size_t mfcl_length; 274 }; 275 276 struct memorystatus_freeze_list_iterator { 277 bool refreeze_only; 278 proc_t last_p; 279 size_t global_freeze_list_index; 280 }; 281 282 /* 283 * Freezer globals 284 */ 285 extern struct memorystatus_freezer_stats_t memorystatus_freezer_stats; 286 extern int memorystatus_freezer_use_ordered_list; 287 extern struct memorystatus_freezer_candidate_list memorystatus_global_freeze_list; 288 extern struct memorystatus_freezer_candidate_list memorystatus_global_demote_list; 289 extern uint64_t memorystatus_freezer_thread_next_run_ts; 290 bool memorystatus_is_process_eligible_for_freeze(proc_t p); 291 bool memorystatus_freeze_proc_is_refreeze_eligible(proc_t p); 292 293 proc_t memorystatus_freezer_candidate_list_get_proc( 294 struct memorystatus_freezer_candidate_list *list, 295 size_t index, 296 uint64_t *pid_mismatch_counter); 297 /* 298 * Returns the leader of the p's jetsam coalition 299 * and the role of p in that coalition. 300 */ 301 proc_t memorystatus_get_coalition_leader_and_role(proc_t p, int *role_in_coalition); 302 bool memorystatus_freeze_process_is_recommended(const proc_t p); 303 304 /* 305 * Ordered iterator over all freeze candidates. 306 * The iterator should initially be zeroed out by the caller and 307 * can be zeroed out whenever the caller wishes to start from the beginning 308 * of the list again. 309 * Returns PROC_NULL when all candidates have been iterated over. 310 */ 311 proc_t memorystatus_freeze_pick_process(struct memorystatus_freeze_list_iterator *iterator); 312 313 /* 314 * Returns the number of processes that the freezer thread should try to freeze 315 * on this wakeup. 316 */ 317 size_t memorystatus_pick_freeze_count_for_wakeup(void); 318 319 /* 320 * Configure the freezer for app-based swap mode. 321 * Should be called at boot. 322 */ 323 void memorystatus_freeze_configure_for_swap(void); 324 /* 325 * Undo memorystatus_freeze_configure_for_swap 326 */ 327 void memorystatus_freeze_disable_swap(void); 328 #endif /* CONFIG_FREEZE */ 329 330 #endif /* BSD_KERNEL_PRIVATE */ 331 332 #endif /* _KERN_MEMORYSTATUS_INTERNAL_H_ */ 333