1 /*
2 * Copyright (c) 2006-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 *
28 */
29
30 #include <sys/kern_event.h>
31 #include <kern/sched_prim.h>
32 #include <kern/assert.h>
33 #include <kern/debug.h>
34 #include <kern/locks.h>
35 #include <kern/task.h>
36 #include <kern/thread.h>
37 #include <kern/thread_call.h>
38 #include <kern/host.h>
39 #include <kern/policy_internal.h>
40 #include <kern/thread_group.h>
41
42 #include <IOKit/IOBSD.h>
43
44 #include <libkern/libkern.h>
45 #include <libkern/coreanalytics/coreanalytics.h>
46 #include <mach/coalition.h>
47 #include <mach/clock_types.h>
48 #include <mach/mach_time.h>
49 #include <mach/task.h>
50 #include <mach/host_priv.h>
51 #include <mach/mach_host.h>
52 #include <os/log.h>
53 #include <pexpert/pexpert.h>
54 #include <sys/coalition.h>
55 #include <sys/kern_event.h>
56 #include <sys/proc.h>
57 #include <sys/proc_info.h>
58 #include <sys/reason.h>
59 #include <sys/signal.h>
60 #include <sys/signalvar.h>
61 #include <sys/sysctl.h>
62 #include <sys/sysproto.h>
63 #include <sys/time.h>
64 #include <sys/wait.h>
65 #include <sys/tree.h>
66 #include <sys/priv.h>
67 #include <vm/vm_pageout_xnu.h>
68 #include <vm/vm_protos.h>
69 #include <vm/vm_purgeable_xnu.h>
70 #include <mach/machine/sdt.h>
71 #include <libkern/section_keywords.h>
72 #include <stdatomic.h>
73
74 #if CONFIG_FREEZE
75 #include <vm/vm_map.h>
76 #endif /* CONFIG_FREEZE */
77
78 #include <kern/kern_memorystatus_internal.h>
79 #include <sys/kern_memorystatus.h>
80 #include <sys/kern_memorystatus_notify.h>
81 #include <sys/kern_memorystatus_xnu.h>
82
83 /*
84 * Memorystatus klist structures
85 */
86 struct klist memorystatus_klist;
87 static lck_mtx_t memorystatus_klist_mutex;
88 static void memorystatus_klist_lock(void);
89 static void memorystatus_klist_unlock(void);
90
91 /*
92 * Memorystatus kevent filter routines
93 */
94 static int filt_memorystatusattach(struct knote *kn, struct kevent_qos_s *kev);
95 static void filt_memorystatusdetach(struct knote *kn);
96 static int filt_memorystatus(struct knote *kn, long hint);
97 static int filt_memorystatustouch(struct knote *kn, struct kevent_qos_s *kev);
98 static int filt_memorystatusprocess(struct knote *kn, struct kevent_qos_s *kev);
99
100 SECURITY_READ_ONLY_EARLY(struct filterops) memorystatus_filtops = {
101 .f_attach = filt_memorystatusattach,
102 .f_detach = filt_memorystatusdetach,
103 .f_event = filt_memorystatus,
104 .f_touch = filt_memorystatustouch,
105 .f_process = filt_memorystatusprocess,
106 };
107
108 /*
109 * Memorystatus notification events
110 */
111 enum {
112 kMemorystatusNoPressure = 0x1,
113 kMemorystatusPressure = 0x2,
114 kMemorystatusLowSwap = 0x4,
115 kMemorystatusProcLimitWarn = 0x8,
116 kMemorystatusProcLimitCritical = 0x10
117 };
118
119 #define INTER_NOTIFICATION_DELAY (250000) /* .25 second */
120 #define VM_PRESSURE_DECREASED_SMOOTHING_PERIOD 5000 /* milliseconds */
121 #define WARNING_NOTIFICATION_RESTING_PERIOD 25 /* seconds */
122 #define CRITICAL_NOTIFICATION_RESTING_PERIOD 25 /* seconds */
123
124 /*
125 * Memorystatus notification helper routines
126 */
127 static vm_pressure_level_t convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t);
128 static boolean_t is_knote_registered_modify_task_pressure_bits(struct knote*, int, task_t, vm_pressure_level_t, vm_pressure_level_t);
129 static void memorystatus_klist_reset_all_for_level(vm_pressure_level_t pressure_level_to_clear);
130 static struct knote *vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int level, boolean_t target_foreground_process, uint64_t *next_telemetry_update);
131 static void vm_dispatch_memory_pressure(void);
132 kern_return_t memorystatus_update_vm_pressure(boolean_t target_foreground_process);
133
134 #if VM_PRESSURE_EVENTS
135
136 /*
137 * This value is the threshold that a process must meet to be considered for scavenging.
138 */
139 #if XNU_TARGET_OS_OSX
140 #define VM_PRESSURE_MINIMUM_RSIZE 10 /* MB */
141 #else /* XNU_TARGET_OS_OSX */
142 #define VM_PRESSURE_MINIMUM_RSIZE 6 /* MB */
143 #endif /* XNU_TARGET_OS_OSX */
144
145 static TUNABLE_DEV_WRITEABLE(uint32_t, vm_pressure_task_footprint_min, "vm_pressure_notify_min_footprint_mb", VM_PRESSURE_MINIMUM_RSIZE);
146
147 #if DEVELOPMENT || DEBUG
148 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_vm_pressure_task_footprint_min, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_pressure_task_footprint_min, 0, "");
149 #endif /* DEVELOPMENT || DEBUG */
150
151 vm_pressure_level_t memorystatus_vm_pressure_level = kVMPressureNormal;
152
153 /*
154 * We use this flag to signal if we have any HWM offenders
155 * on the system. This way we can reduce the number of wakeups
156 * of the memorystatus_thread when the system is between the
157 * "pressure" and "critical" threshold.
158 *
159 * The (re-)setting of this variable is done without any locks
160 * or synchronization simply because it is not possible (currently)
161 * to keep track of HWM offenders that drop down below their memory
162 * limit and/or exit. So, we choose to burn a couple of wasted wakeups
163 * by allowing the unguarded modification of this variable.
164 *
165 * TODO: this should be a count of number of hwm candidates
166 */
167 _Atomic bool memorystatus_hwm_candidates = false;
168
169 #endif /* VM_PRESSURE_EVENTS */
170
171 uint32_t memorystatus_jetsam_fg_band_waiters = 0;
172 uint32_t memorystatus_jetsam_bg_band_waiters = 0;
173 static uint64_t memorystatus_jetsam_fg_band_timestamp_ns = 0; /* nanosec */
174 static uint64_t memorystatus_jetsam_bg_band_timestamp_ns = 0; /* nanosec */
175 static uint64_t memorystatus_jetsam_notification_delay_ns = 5ull * 1000 * 1000 * 1000; /* nanosec */
176
177 #if DEVELOPMENT || DEBUG
178 SYSCTL_QUAD(_kern, OID_AUTO, memorystatus_jetsam_notification_delay_ns, CTLFLAG_RW | CTLFLAG_LOCKED,
179 &memorystatus_jetsam_notification_delay_ns, "");
180 #endif
181
182 static int
filt_memorystatusattach(struct knote * kn,__unused struct kevent_qos_s * kev)183 filt_memorystatusattach(struct knote *kn, __unused struct kevent_qos_s *kev)
184 {
185 int error;
186
187 kn->kn_flags |= EV_CLEAR; /* automatically set */
188 kn->kn_sdata = 0; /* incoming data is ignored */
189 memset(&kn->kn_ext, 0, sizeof(kn->kn_ext));
190
191 error = memorystatus_knote_register(kn);
192 if (error) {
193 knote_set_error(kn, error);
194 }
195 return 0;
196 }
197
198 static void
filt_memorystatusdetach(struct knote * kn)199 filt_memorystatusdetach(struct knote *kn)
200 {
201 memorystatus_knote_unregister(kn);
202 }
203
204 static int
filt_memorystatus(struct knote * kn __unused,long hint)205 filt_memorystatus(struct knote *kn __unused, long hint)
206 {
207 if (hint) {
208 switch (hint) {
209 case kMemorystatusNoPressure:
210 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
211 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
212 }
213 break;
214 case kMemorystatusPressure:
215 if (memorystatus_vm_pressure_level == kVMPressureWarning || memorystatus_vm_pressure_level == kVMPressureUrgent) {
216 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
217 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
218 }
219 } else if (memorystatus_vm_pressure_level == kVMPressureCritical) {
220 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
221 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
222 }
223 }
224 break;
225 case kMemorystatusLowSwap:
226 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_LOW_SWAP) {
227 kn->kn_fflags = NOTE_MEMORYSTATUS_LOW_SWAP;
228 }
229 break;
230
231 case kMemorystatusProcLimitWarn:
232 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
233 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
234 }
235 break;
236
237 case kMemorystatusProcLimitCritical:
238 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
239 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
240 }
241 break;
242
243 default:
244 break;
245 }
246 }
247
248 #if 0
249 if (kn->kn_fflags != 0) {
250 proc_t knote_proc = knote_get_kq(kn)->kq_p;
251 pid_t knote_pid = proc_getpid(knote_proc);
252
253 printf("filt_memorystatus: sending kn 0x%lx (event 0x%x) for pid (%d)\n",
254 (unsigned long)kn, kn->kn_fflags, knote_pid);
255 }
256 #endif
257
258 return kn->kn_fflags != 0;
259 }
260
261 static int
filt_memorystatustouch(struct knote * kn,struct kevent_qos_s * kev)262 filt_memorystatustouch(struct knote *kn, struct kevent_qos_s *kev)
263 {
264 int res;
265 int prev_kn_sfflags = 0;
266
267 memorystatus_klist_lock();
268
269 /*
270 * copy in new kevent settings
271 * (saving the "desired" data and fflags).
272 */
273
274 prev_kn_sfflags = kn->kn_sfflags;
275 kn->kn_sfflags = (kev->fflags & EVFILT_MEMORYSTATUS_ALL_MASK);
276
277 #if XNU_TARGET_OS_OSX
278 /*
279 * Only on desktop do we restrict notifications to
280 * one per active/inactive state (soft limits only).
281 */
282 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
283 /*
284 * Is there previous state to preserve?
285 */
286 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
287 /*
288 * This knote was previously interested in proc_limit_warn,
289 * so yes, preserve previous state.
290 */
291 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE) {
292 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
293 }
294 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE) {
295 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
296 }
297 } else {
298 /*
299 * This knote was not previously interested in proc_limit_warn,
300 * but it is now. Set both states.
301 */
302 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
303 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
304 }
305 }
306
307 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
308 /*
309 * Is there previous state to preserve?
310 */
311 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
312 /*
313 * This knote was previously interested in proc_limit_critical,
314 * so yes, preserve previous state.
315 */
316 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE) {
317 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
318 }
319 if (prev_kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE) {
320 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
321 }
322 } else {
323 /*
324 * This knote was not previously interested in proc_limit_critical,
325 * but it is now. Set both states.
326 */
327 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
328 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
329 }
330 }
331 #endif /* XNU_TARGET_OS_OSX */
332
333 /*
334 * reset the output flags based on a
335 * combination of the old events and
336 * the new desired event list.
337 */
338 //kn->kn_fflags &= kn->kn_sfflags;
339
340 res = (kn->kn_fflags != 0);
341
342 memorystatus_klist_unlock();
343
344 return res;
345 }
346
347 static int
filt_memorystatusprocess(struct knote * kn,struct kevent_qos_s * kev)348 filt_memorystatusprocess(struct knote *kn, struct kevent_qos_s *kev)
349 {
350 int res = 0;
351
352 memorystatus_klist_lock();
353 if (kn->kn_fflags) {
354 knote_fill_kevent(kn, kev, 0);
355 res = 1;
356 }
357 memorystatus_klist_unlock();
358
359 return res;
360 }
361
362 static void
memorystatus_klist_lock(void)363 memorystatus_klist_lock(void)
364 {
365 lck_mtx_lock(&memorystatus_klist_mutex);
366 }
367
368 static void
memorystatus_klist_unlock(void)369 memorystatus_klist_unlock(void)
370 {
371 lck_mtx_unlock(&memorystatus_klist_mutex);
372 }
373
374 void
memorystatus_kevent_init(lck_grp_t * grp,lck_attr_t * attr)375 memorystatus_kevent_init(lck_grp_t *grp, lck_attr_t *attr)
376 {
377 lck_mtx_init(&memorystatus_klist_mutex, grp, attr);
378 klist_init(&memorystatus_klist);
379 }
380
381 int
memorystatus_knote_register(struct knote * kn)382 memorystatus_knote_register(struct knote *kn)
383 {
384 int error = 0;
385
386 memorystatus_klist_lock();
387
388 /*
389 * Support only userspace visible flags.
390 */
391 if ((kn->kn_sfflags & EVFILT_MEMORYSTATUS_ALL_MASK) == (unsigned int) kn->kn_sfflags) {
392 #if XNU_TARGET_OS_OSX
393 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
394 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
395 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
396 }
397
398 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
399 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
400 kn->kn_sfflags |= NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
401 }
402 #endif /* XNU_TARGET_OS_OSX */
403
404 KNOTE_ATTACH(&memorystatus_klist, kn);
405 } else {
406 error = ENOTSUP;
407 }
408
409 memorystatus_klist_unlock();
410
411 return error;
412 }
413
414 void
memorystatus_knote_unregister(struct knote * kn __unused)415 memorystatus_knote_unregister(struct knote *kn __unused)
416 {
417 memorystatus_klist_lock();
418 KNOTE_DETACH(&memorystatus_klist, kn);
419 memorystatus_klist_unlock();
420 }
421
422 #if VM_PRESSURE_EVENTS
423
424 static thread_call_t sustained_pressure_handler_thread_call;
425 /* Count the number of sustained pressure kills we've done since boot. */
426 uint64_t memorystatus_kill_on_sustained_pressure_count = 0;
427 uint64_t memorystatus_kill_on_sustained_pressure_window_s = 60 * 10; /* 10 Minutes */
428 uint64_t memorystatus_kill_on_sustained_pressure_delay_ms = 500; /* .5 seconds */
429
430 SYSCTL_QUAD(_kern_memorystatus, OID_AUTO, kill_on_sustained_pressure_count, CTLFLAG_RD | CTLFLAG_LOCKED, &memorystatus_kill_on_sustained_pressure_count, "");
431 SYSCTL_QUAD(_kern_memorystatus, OID_AUTO, kill_on_sustained_pressure_window_s, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_kill_on_sustained_pressure_window_s, "");
432 SYSCTL_QUAD(_kern_memorystatus, OID_AUTO, kill_on_sustained_pressure_delay_ms, CTLFLAG_RW | CTLFLAG_LOCKED, &memorystatus_kill_on_sustained_pressure_delay_ms, "");
433
434 static void sustained_pressure_handler(void*, void*);
435
436 static thread_call_t memorystatus_notify_update_telemetry_thread_call;
437 static void update_footprints_for_telemetry(void*, void*);
438
439 void
memorystatus_notify_init()440 memorystatus_notify_init()
441 {
442 sustained_pressure_handler_thread_call = thread_call_allocate_with_options(sustained_pressure_handler, NULL, THREAD_CALL_PRIORITY_KERNEL_HIGH, THREAD_CALL_OPTIONS_ONCE);
443 memorystatus_notify_update_telemetry_thread_call = thread_call_allocate_with_options(update_footprints_for_telemetry, NULL, THREAD_CALL_PRIORITY_USER, THREAD_CALL_OPTIONS_ONCE);
444 }
445
446 #if CONFIG_MEMORYSTATUS
447
448 inline int
memorystatus_send_note(int event_code,void * data,uint32_t data_length)449 memorystatus_send_note(int event_code, void *data, uint32_t data_length)
450 {
451 int ret;
452 struct kev_msg ev_msg;
453
454 ev_msg.vendor_code = KEV_VENDOR_APPLE;
455 ev_msg.kev_class = KEV_SYSTEM_CLASS;
456 ev_msg.kev_subclass = KEV_MEMORYSTATUS_SUBCLASS;
457
458 ev_msg.event_code = event_code;
459
460 ev_msg.dv[0].data_length = data_length;
461 ev_msg.dv[0].data_ptr = data;
462 ev_msg.dv[1].data_length = 0;
463
464 ret = kev_post_msg(&ev_msg);
465 if (ret) {
466 memorystatus_log_error("%s: kev_post_msg() failed, err %d\n", __func__, ret);
467 }
468
469 return ret;
470 }
471
472 boolean_t
memorystatus_warn_process(const proc_t p,__unused boolean_t is_active,__unused boolean_t is_fatal,boolean_t limit_exceeded)473 memorystatus_warn_process(const proc_t p, __unused boolean_t is_active, __unused boolean_t is_fatal, boolean_t limit_exceeded)
474 {
475 /*
476 * This function doesn't take a reference to p or lock it. So it better be the current process.
477 */
478 assert(p == current_proc());
479 pid_t pid = proc_getpid(p);
480 boolean_t ret = FALSE;
481 boolean_t found_knote = FALSE;
482 struct knote *kn = NULL;
483 int send_knote_count = 0;
484 uint32_t platform;
485 platform = proc_platform(p);
486
487 /*
488 * See comment in sysctl_memorystatus_vm_pressure_send.
489 */
490
491 memorystatus_klist_lock();
492
493 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
494 proc_t knote_proc = knote_get_kq(kn)->kq_p;
495 pid_t knote_pid = proc_getpid(knote_proc);
496
497 if (knote_pid == pid) {
498 /*
499 * By setting the "fflags" here, we are forcing
500 * a process to deal with the case where it's
501 * bumping up into its memory limits. If we don't
502 * do this here, we will end up depending on the
503 * system pressure snapshot evaluation in
504 * filt_memorystatus().
505 */
506
507 /*
508 * The type of notification and the frequency are different between
509 * embedded and desktop.
510 *
511 * Embedded processes register for global pressure notifications
512 * (NOTE_MEMORYSTATUS_PRESSURE_WARN | NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) via UIKit
513 * (see applicationDidReceiveMemoryWarning in UIKit). We'll warn them here if
514 * they are near there memory limit. filt_memorystatus() will warn them based
515 * on the system pressure level.
516 *
517 * On desktop, (NOTE_MEMORYSTATUS_PRESSURE_WARN | NOTE_MEMORYSTATUS_PRESSURE_CRITICAL)
518 * are only expected to fire for system level warnings. Desktop procesess
519 * register for NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
520 * if they want to be warned when they approach their limit
521 * and for NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL to be warned when they
522 * exceed their limit.
523 *
524 * On embedded we continuously warn processes that are approaching their
525 * memory limit. However on desktop, we only send one warning while
526 * the process is active/inactive if the limit is soft..
527 *
528 */
529 if (platform == PLATFORM_MACOS || platform == PLATFORM_MACCATALYST || platform == PLATFORM_DRIVERKIT) {
530 if (!limit_exceeded) {
531 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
532 found_knote = TRUE;
533 if (!is_fatal) {
534 /*
535 * Restrict proc_limit_warn notifications when
536 * non-fatal (soft) limit is at play.
537 */
538 if (is_active) {
539 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE) {
540 /*
541 * Mark this knote for delivery.
542 */
543 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
544 /*
545 * And suppress it from future notifications.
546 */
547 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_ACTIVE;
548 send_knote_count++;
549 }
550 } else {
551 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE) {
552 /*
553 * Mark this knote for delivery.
554 */
555 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
556 /*
557 * And suppress it from future notifications.
558 */
559 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_WARN_INACTIVE;
560 send_knote_count++;
561 }
562 }
563 } else {
564 /*
565 * No restriction on proc_limit_warn notifications when
566 * fatal (hard) limit is at play.
567 */
568 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
569 send_knote_count++;
570 }
571 }
572 } else {
573 /*
574 * Send this notification when a process has exceeded a soft limit,
575 */
576
577 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
578 found_knote = TRUE;
579 if (!is_fatal) {
580 /*
581 * Restrict critical notifications for soft limits.
582 */
583
584 if (is_active) {
585 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE) {
586 /*
587 * Suppress future proc_limit_critical notifications
588 * for the active soft limit.
589 */
590 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_ACTIVE;
591 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
592 send_knote_count++;
593 }
594 } else {
595 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE) {
596 /*
597 * Suppress future proc_limit_critical_notifications
598 * for the inactive soft limit.
599 */
600 kn->kn_sfflags &= ~NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL_INACTIVE;
601 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
602 send_knote_count++;
603 }
604 }
605 } else {
606 /*
607 * We should never be trying to send a critical notification for
608 * a hard limit... the process would be killed before it could be
609 * received.
610 */
611 panic("Caught sending pid %d a critical warning for a fatal limit.", pid);
612 }
613 }
614 }
615 } else {
616 if (!limit_exceeded) {
617 /*
618 * Intentionally set either the unambiguous limit warning,
619 * the system-wide critical or the system-wide warning
620 * notification bit.
621 */
622
623 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) {
624 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_WARN;
625 found_knote = TRUE;
626 send_knote_count++;
627 } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
628 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
629 found_knote = TRUE;
630 send_knote_count++;
631 } else if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_WARN) {
632 kn->kn_fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
633 found_knote = TRUE;
634 send_knote_count++;
635 }
636 } else {
637 /*
638 * Send this notification when a process has exceeded a soft limit.
639 */
640 if (kn->kn_sfflags & NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) {
641 kn->kn_fflags = NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL;
642 found_knote = TRUE;
643 send_knote_count++;
644 }
645 }
646 }
647 }
648 }
649
650 if (found_knote) {
651 if (send_knote_count > 0) {
652 KNOTE(&memorystatus_klist, 0);
653 }
654 ret = TRUE;
655 }
656
657 memorystatus_klist_unlock();
658
659 return ret;
660 }
661
662 /*
663 * Can only be set by the current task on itself.
664 */
665 int
memorystatus_low_mem_privileged_listener(uint32_t op_flags)666 memorystatus_low_mem_privileged_listener(uint32_t op_flags)
667 {
668 boolean_t set_privilege = FALSE;
669 /*
670 * Need an entitlement check here?
671 */
672 if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_ENABLE) {
673 set_privilege = TRUE;
674 } else if (op_flags == MEMORYSTATUS_CMD_PRIVILEGED_LISTENER_DISABLE) {
675 set_privilege = FALSE;
676 } else {
677 return EINVAL;
678 }
679
680 return task_low_mem_privileged_listener(current_task(), set_privilege, NULL);
681 }
682
683 int
memorystatus_send_pressure_note(pid_t pid)684 memorystatus_send_pressure_note(pid_t pid)
685 {
686 memorystatus_log_debug("memorystatus_send_pressure_note(): pid %d\n", pid);
687 return memorystatus_send_note(kMemorystatusPressureNote, &pid, sizeof(pid));
688 }
689
690 boolean_t
memorystatus_is_foreground_locked(proc_t p)691 memorystatus_is_foreground_locked(proc_t p)
692 {
693 return (p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND) ||
694 (p->p_memstat_effectivepriority == JETSAM_PRIORITY_FOREGROUND_SUPPORT);
695 }
696
697 /*
698 * This is meant for stackshot and kperf -- it does not take the proc_list_lock
699 * to access the p_memstat_dirty field.
700 */
701 void
memorystatus_proc_flags_unsafe(void * v,boolean_t * is_dirty,boolean_t * is_dirty_tracked,boolean_t * allow_idle_exit,boolean_t * is_active,boolean_t * is_managed,boolean_t * has_assertion)702 memorystatus_proc_flags_unsafe(void * v, boolean_t *is_dirty, boolean_t *is_dirty_tracked, boolean_t *allow_idle_exit, boolean_t *is_active, boolean_t *is_managed, boolean_t *has_assertion)
703 {
704 if (!v) {
705 *is_dirty = FALSE;
706 *is_dirty_tracked = FALSE;
707 *allow_idle_exit = FALSE;
708 *is_active = FALSE;
709 *is_managed = FALSE;
710 *has_assertion = FALSE;
711 } else {
712 proc_t p = (proc_t)v;
713 *is_dirty = (p->p_memstat_dirty & P_DIRTY_IS_DIRTY) != 0;
714 *is_dirty_tracked = (p->p_memstat_dirty & P_DIRTY_TRACK) != 0;
715 *allow_idle_exit = (p->p_memstat_dirty & P_DIRTY_ALLOW_IDLE_EXIT) != 0;
716 *is_active = (p->p_memstat_memlimit == p->p_memstat_memlimit_active);
717 *is_managed = (p->p_memstat_state & P_MEMSTAT_MANAGED) != 0;
718 *has_assertion = (p->p_memstat_state & P_MEMSTAT_PRIORITY_ASSERTION) != 0;
719 }
720 }
721
722 boolean_t
memorystatus_bg_pressure_eligible(proc_t p)723 memorystatus_bg_pressure_eligible(proc_t p)
724 {
725 boolean_t eligible = FALSE;
726
727 proc_list_lock();
728
729 memorystatus_log_debug("memorystatus_bg_pressure_eligible: pid %d, state 0x%x\n", proc_getpid(p), p->p_memstat_state);
730
731 /* Foreground processes have already been dealt with at this point, so just test for eligibility */
732 if (!(p->p_memstat_state & (P_MEMSTAT_TERMINATED | P_MEMSTAT_LOCKED | P_MEMSTAT_SUSPENDED | P_MEMSTAT_FROZEN))) {
733 eligible = TRUE;
734 }
735
736 if (p->p_memstat_effectivepriority < JETSAM_PRIORITY_BACKGROUND_OPPORTUNISTIC) {
737 /*
738 * IDLE and IDLE_DEFERRED bands contain processes
739 * that have dropped memory to be under their inactive
740 * memory limits. And so they can't really give back
741 * anything.
742 */
743 eligible = FALSE;
744 }
745
746 proc_list_unlock();
747
748 return eligible;
749 }
750
751 void
memorystatus_send_low_swap_note(void)752 memorystatus_send_low_swap_note(void)
753 {
754 struct knote *kn = NULL;
755
756 memorystatus_klist_lock();
757 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
758 /* We call is_knote_registered_modify_task_pressure_bits to check if the sfflags for the
759 * current note contain NOTE_MEMORYSTATUS_LOW_SWAP. Once we find one note in the memorystatus_klist
760 * that has the NOTE_MEMORYSTATUS_LOW_SWAP flags in its sfflags set, we call KNOTE with
761 * kMemoryStatusLowSwap as the hint to process and update all knotes on the memorystatus_klist accordingly. */
762 if (is_knote_registered_modify_task_pressure_bits(kn, NOTE_MEMORYSTATUS_LOW_SWAP, NULL, 0, 0) == TRUE) {
763 KNOTE(&memorystatus_klist, kMemorystatusLowSwap);
764 break;
765 }
766 }
767
768 memorystatus_klist_unlock();
769 }
770
771 #endif /* CONFIG_MEMORYSTATUS */
772
773 /*
774 * Notification telemetry
775 */
776 CA_EVENT(memorystatus_pressure_interval,
777 CA_INT, num_processes_registered,
778 CA_INT, num_notifications_sent,
779 CA_INT, max_level,
780 CA_INT, num_transitions,
781 CA_INT, num_kills,
782 CA_INT, duration);
783
784 /* Separate struct for tracking so that we have aligned members for atomics */
785 struct memstat_cur_interval {
786 int64_t num_procs;
787 int64_t num_notifs;
788 int64_t num_transitions;
789 uint64_t start_mt;
790 _Atomic uint32_t num_kills;
791 vm_pressure_level_t max_level;
792 } memstat_cur_interval;
793
794 CA_EVENT(memorystatus_proc_notification,
795 CA_INT, footprint_before_notification,
796 CA_INT, footprint_1_min_after_first_warning,
797 CA_INT, footprint_5_min_after_first_warning,
798 CA_INT, footprint_20_min_after_first_warning,
799 CA_INT, footprint_1_min_after_first_critical,
800 CA_INT, footprint_5_min_after_first_critical,
801 CA_INT, footprint_20_min_after_first_critical,
802 CA_INT, order_within_list,
803 CA_INT, num_notifications_sent,
804 CA_INT, time_between_warning_and_critical,
805 CA_STATIC_STRING(CA_PROCNAME_LEN), proc_name);
806
807 /* The send timestamps for the first notifications are stored in the knote's kn_sdata field */
808 #define KNOTE_SEND_TIMESTAMP_WARNING_INDEX 0
809 #define KNOTE_SEND_TIMESTAMP_CRITICAL_INDEX 1
810
811 /* The footprint history for this task is stored in the knote's kn_ext array. */
812 struct knote_footprint_history {
813 uint32_t kfh_starting_footprint;
814 uint32_t kfh_footprint_after_warn_1; /* 1 minute after first warning notification */
815 uint32_t kfh_footprint_after_warn_5; /* 5 minutes after first warning notification */
816 uint32_t kfh_footprint_after_warn_20; /* 20 minutes after first warning notification */
817 uint32_t kfh_footprint_after_critical_1; /* 1 minute after first critical notification */
818 uint32_t kfh_footprint_after_critical_5; /* 5 minutes after first critical notification */
819 uint32_t kfh_footprint_after_critical_20; /* 20 minutes after first critical notification */
820 uint16_t kfh_num_notifications;
821 uint16_t kfh_notification_order;
822 } __attribute__((packed));
823
824
825 static_assert(sizeof(struct knote_footprint_history) <= sizeof(uint64_t) * 4, "footprint history fits in knote extensions");
826
827 static void
mark_knote_send_time(struct knote * kn,task_t task,int knote_pressure_level,uint16_t order_within_list)828 mark_knote_send_time(struct knote *kn, task_t task, int knote_pressure_level, uint16_t order_within_list)
829 {
830 uint32_t *timestamps;
831 uint32_t index;
832 uint64_t curr_ts, curr_ts_seconds;
833 struct knote_footprint_history *footprint_history = (struct knote_footprint_history *)kn->kn_ext;
834 if (knote_pressure_level != NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
835 timestamps = (uint32_t *)&(kn->kn_sdata);
836 index = knote_pressure_level == NOTE_MEMORYSTATUS_PRESSURE_WARN ?
837 KNOTE_SEND_TIMESTAMP_WARNING_INDEX : KNOTE_SEND_TIMESTAMP_CRITICAL_INDEX;
838 if (timestamps[index] == 0) {
839 /* First notification for this level since pressure elevated from normal. */
840 curr_ts = mach_absolute_time();
841 curr_ts_seconds = 0;
842 absolutetime_to_nanoseconds(curr_ts, &curr_ts_seconds);
843 curr_ts_seconds /= NSEC_PER_SEC;
844
845 timestamps[index] = (uint32_t)MIN(UINT32_MAX, curr_ts_seconds);
846
847 /* Record task initial footprint */
848 if (timestamps[index == KNOTE_SEND_TIMESTAMP_WARNING_INDEX ? KNOTE_SEND_TIMESTAMP_CRITICAL_INDEX : KNOTE_SEND_TIMESTAMP_WARNING_INDEX] == 0) {
849 /*
850 * First notification at any level since pressure elevated from normal.
851 * Record the footprint and our order in the notification list.
852 */
853 footprint_history->kfh_starting_footprint = (uint32_t) MIN(UINT32_MAX, get_task_phys_footprint(task) / (2UL << 20));
854 footprint_history->kfh_notification_order = order_within_list;
855 }
856 }
857 }
858 footprint_history->kfh_num_notifications++;
859 }
860
861 /*
862 * Records the current footprint for this task in the knote telemetry.
863 *
864 * Returns the soonest absolutetime when this footprint history should be updated again.
865 */
866 static uint64_t
update_knote_footprint_history(struct knote * kn,task_t task,uint64_t curr_ts)867 update_knote_footprint_history(struct knote *kn, task_t task, uint64_t curr_ts)
868 {
869 uint32_t *timestamps = (uint32_t *)&(kn->kn_sdata);
870 struct knote_footprint_history *footprint_history = (struct knote_footprint_history *)kn->kn_ext;
871 uint64_t warning_send_time, critical_send_time, minutes_since_warning = UINT64_MAX, minutes_since_critical = UINT64_MAX;
872 warning_send_time = timestamps[KNOTE_SEND_TIMESTAMP_WARNING_INDEX];
873 critical_send_time = timestamps[KNOTE_SEND_TIMESTAMP_CRITICAL_INDEX];
874 uint32_t task_phys_footprint_mb = (uint32_t) MIN(UINT32_MAX, get_task_phys_footprint(task) / (2UL << 20));
875 uint64_t next_run = UINT64_MAX, absolutetime_in_minute = 0, minutes_since_last_notification = 0, curr_ts_s;
876 absolutetime_to_nanoseconds(curr_ts, &curr_ts_s);
877 nanoseconds_to_absolutetime(60 * NSEC_PER_SEC, &absolutetime_in_minute);
878 curr_ts_s /= NSEC_PER_SEC;
879
880 if (warning_send_time != 0) {
881 /* This task received a warning notification. */
882 minutes_since_warning = (curr_ts_s - warning_send_time) / 60;
883 if (footprint_history->kfh_footprint_after_warn_1 == 0 && minutes_since_warning >= 1) {
884 footprint_history->kfh_footprint_after_warn_1 = task_phys_footprint_mb;
885 }
886 if (footprint_history->kfh_footprint_after_warn_5 == 0 && minutes_since_warning >= 5) {
887 footprint_history->kfh_footprint_after_warn_5 = task_phys_footprint_mb;
888 }
889 if (footprint_history->kfh_footprint_after_warn_20 == 0 && minutes_since_warning >= 20) {
890 footprint_history->kfh_footprint_after_warn_20 = task_phys_footprint_mb;
891 }
892 }
893 if (critical_send_time != 0) {
894 /* This task received a critical notification. */
895 minutes_since_critical = (curr_ts_s - critical_send_time) / 60;
896 if (footprint_history->kfh_footprint_after_critical_1 == 0 && minutes_since_critical >= 1) {
897 footprint_history->kfh_footprint_after_critical_1 = task_phys_footprint_mb;
898 }
899 if (footprint_history->kfh_footprint_after_critical_5 == 0 && minutes_since_critical >= 5) {
900 footprint_history->kfh_footprint_after_critical_5 = task_phys_footprint_mb;
901 }
902 if (footprint_history->kfh_footprint_after_critical_20 == 0 && minutes_since_critical >= 20) {
903 footprint_history->kfh_footprint_after_critical_20 = task_phys_footprint_mb;
904 }
905 }
906
907 minutes_since_last_notification = MIN(minutes_since_warning, minutes_since_critical);
908 if (minutes_since_last_notification < 20) {
909 if (minutes_since_last_notification < 5) {
910 if (minutes_since_last_notification < 1) {
911 next_run = curr_ts + absolutetime_in_minute;
912 } else {
913 next_run = curr_ts + (absolutetime_in_minute * 5);
914 }
915 } else {
916 next_run = curr_ts + (absolutetime_in_minute * 20);
917 }
918 }
919
920 return next_run;
921 }
922
923 extern char *proc_name_address(void *p);
924
925 /*
926 * Send pressure interval telemetry.
927 */
928 static void
memorystatus_pressure_interval_send(void)929 memorystatus_pressure_interval_send(void)
930 {
931 uint64_t duration_nanoseconds;
932 CA_EVENT_TYPE(memorystatus_pressure_interval) * evt_data;
933
934 /*
935 * Drop the event rather than block for memory. We should be in a normal pressure level now,
936 * but we don't want to end up blocked in page_wait if there's a sudden spike in pressure.
937 */
938 ca_event_t event_wrapper = CA_EVENT_ALLOCATE_FLAGS(memorystatus_pressure_interval, Z_NOWAIT);
939 if (event_wrapper) {
940 absolutetime_to_nanoseconds(
941 mach_absolute_time() - memstat_cur_interval.start_mt,
942 &duration_nanoseconds);
943
944 evt_data = event_wrapper->data;
945 evt_data->num_processes_registered = memstat_cur_interval.num_procs;
946 evt_data->num_notifications_sent = memstat_cur_interval.num_notifs;
947 evt_data->max_level = memstat_cur_interval.max_level;
948 evt_data->num_transitions = memstat_cur_interval.num_transitions;
949 evt_data->num_kills = os_atomic_load(&memstat_cur_interval.num_kills, relaxed);
950 evt_data->duration = duration_nanoseconds / NSEC_PER_SEC;
951
952 CA_EVENT_SEND(event_wrapper);
953 } else {
954 memorystatus_log_error("memorystatus: Dropping interval telemetry event\n");
955 }
956 }
957
958 /*
959 * Attempt to send the per-proc telemetry events.
960 * Clears the footprint histories on the knotes.
961 */
962 static void
memorystatus_pressure_proc_telemetry_send(void)963 memorystatus_pressure_proc_telemetry_send(void)
964 {
965 struct knote *kn = NULL;
966 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
967 proc_t p = PROC_NULL;
968 struct knote_footprint_history *footprint_history = (struct knote_footprint_history *)kn->kn_ext;
969 uint32_t *timestamps = (uint32_t *)&(kn->kn_sdata);
970 uint32_t warning_send_time = timestamps[KNOTE_SEND_TIMESTAMP_WARNING_INDEX];
971 uint32_t critical_send_time = timestamps[KNOTE_SEND_TIMESTAMP_CRITICAL_INDEX];
972 CA_EVENT_TYPE(memorystatus_proc_notification) * event = NULL;
973 if (warning_send_time != 0 || critical_send_time != 0) {
974 /*
975 * Drop the event rather than block for memory. We should be in a normal pressure level now,
976 * but we don't want to end up blocked in page_wait if there's a sudden spike in pressure.
977 */
978 ca_event_t event_wrapper = CA_EVENT_ALLOCATE_FLAGS(memorystatus_proc_notification, Z_NOWAIT | Z_ZERO);
979 if (event_wrapper) {
980 event = event_wrapper->data;
981
982 event->footprint_before_notification = footprint_history->kfh_starting_footprint;
983 event->footprint_1_min_after_first_warning = footprint_history->kfh_footprint_after_warn_1;
984 event->footprint_5_min_after_first_warning = footprint_history->kfh_footprint_after_warn_5;
985 event->footprint_20_min_after_first_warning = footprint_history->kfh_footprint_after_warn_20;
986 event->footprint_1_min_after_first_critical = footprint_history->kfh_footprint_after_critical_1;
987 event->footprint_5_min_after_first_critical = footprint_history->kfh_footprint_after_critical_5;
988 event->footprint_20_min_after_first_critical = footprint_history->kfh_footprint_after_critical_20;
989 event->num_notifications_sent = footprint_history->kfh_num_notifications;
990 if (warning_send_time != 0 && critical_send_time != 0) {
991 event->time_between_warning_and_critical = (critical_send_time - warning_send_time) / 60; // Minutes
992 }
993 event->order_within_list = footprint_history->kfh_notification_order;
994
995 p = proc_ref(knote_get_kq(kn)->kq_p, false);
996 if (p == NULL) {
997 CA_EVENT_DEALLOCATE(event_wrapper);
998 continue;
999 }
1000 strlcpy(event->proc_name, proc_name_address(p), sizeof(event->proc_name));
1001
1002 proc_rele(p);
1003 CA_EVENT_SEND(event_wrapper);
1004 }
1005 }
1006 memset(footprint_history, 0, sizeof(*footprint_history));
1007 timestamps[KNOTE_SEND_TIMESTAMP_WARNING_INDEX] = 0;
1008 timestamps[KNOTE_SEND_TIMESTAMP_CRITICAL_INDEX] = 0;
1009 }
1010 }
1011
1012 /*
1013 * kn_max - knote
1014 *
1015 * knote_pressure_level - to check if the knote is registered for this notification level.
1016 *
1017 * task - task whose bits we'll be modifying
1018 *
1019 * pressure_level_to_clear - if the task has been notified of this past level, clear that notification bit so that if/when we revert to that level, the task will be notified again.
1020 *
1021 * pressure_level_to_set - the task is about to be notified of this new level. Update the task's bit notification information appropriately.
1022 *
1023 */
1024
1025 static boolean_t
is_knote_registered_modify_task_pressure_bits(struct knote * kn_max,int knote_pressure_level,task_t task,vm_pressure_level_t pressure_level_to_clear,vm_pressure_level_t pressure_level_to_set)1026 is_knote_registered_modify_task_pressure_bits(struct knote *kn_max, int knote_pressure_level, task_t task, vm_pressure_level_t pressure_level_to_clear, vm_pressure_level_t pressure_level_to_set)
1027 {
1028 if (kn_max->kn_sfflags & knote_pressure_level) {
1029 if (pressure_level_to_clear && task_has_been_notified(task, pressure_level_to_clear) == TRUE) {
1030 task_clear_has_been_notified(task, pressure_level_to_clear);
1031 }
1032
1033 task_mark_has_been_notified(task, pressure_level_to_set);
1034 return TRUE;
1035 }
1036
1037 return FALSE;
1038 }
1039
1040 static void
memorystatus_klist_reset_all_for_level(vm_pressure_level_t pressure_level_to_clear)1041 memorystatus_klist_reset_all_for_level(vm_pressure_level_t pressure_level_to_clear)
1042 {
1043 struct knote *kn = NULL;
1044
1045 memorystatus_klist_lock();
1046
1047 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
1048 proc_t p = knote_get_kq(kn)->kq_p;
1049
1050 if (p == proc_ref(p, false)) {
1051 task_clear_has_been_notified(proc_task(p), pressure_level_to_clear);
1052 proc_rele(p);
1053 }
1054 }
1055
1056 memorystatus_klist_unlock();
1057 }
1058
1059 /*
1060 * Used by the vm_pressure_thread which is
1061 * signalled from within vm_pageout_scan().
1062 */
1063
1064 void
consider_vm_pressure_events(void)1065 consider_vm_pressure_events(void)
1066 {
1067 vm_dispatch_memory_pressure();
1068 }
1069
1070 static void
vm_dispatch_memory_pressure(void)1071 vm_dispatch_memory_pressure(void)
1072 {
1073 memorystatus_update_vm_pressure(FALSE);
1074 }
1075
1076 static struct knote *
vm_pressure_select_optimal_candidate_to_notify(struct klist * candidate_list,int level,boolean_t target_foreground_process,uint64_t * next_telemetry_update)1077 vm_pressure_select_optimal_candidate_to_notify(struct klist *candidate_list, int level, boolean_t target_foreground_process, uint64_t *next_telemetry_update)
1078 {
1079 struct knote *kn = NULL, *kn_max = NULL;
1080 uint64_t resident_max = 0;/* MB */
1081 int selected_task_importance = 0;
1082 static int pressure_snapshot = -1;
1083 boolean_t pressure_increase = FALSE;
1084 uint64_t curr_ts = mach_absolute_time();
1085 *next_telemetry_update = UINT64_MAX;
1086
1087 if (pressure_snapshot == -1) {
1088 /*
1089 * Initial snapshot.
1090 */
1091 pressure_snapshot = level;
1092 pressure_increase = TRUE;
1093 } else {
1094 if (level && (level >= pressure_snapshot)) {
1095 pressure_increase = TRUE;
1096 } else {
1097 pressure_increase = FALSE;
1098 }
1099
1100 pressure_snapshot = level;
1101 }
1102
1103 if (pressure_increase == TRUE) {
1104 /*
1105 * We'll start by considering the largest
1106 * unimportant task in our list.
1107 */
1108 selected_task_importance = INT_MAX;
1109 } else {
1110 /*
1111 * We'll start by considering the largest
1112 * important task in our list.
1113 */
1114 selected_task_importance = 0;
1115 }
1116
1117 SLIST_FOREACH(kn, candidate_list, kn_selnext) {
1118 uint64_t resident_size = 0;/* MB */
1119 proc_t p = PROC_NULL;
1120 struct task* t = TASK_NULL;
1121 int curr_task_importance = 0;
1122 uint64_t telemetry_update = 0;
1123 boolean_t consider_knote = FALSE;
1124 boolean_t privileged_listener = FALSE;
1125
1126 p = proc_ref(knote_get_kq(kn)->kq_p, false);
1127 if (p == PROC_NULL) {
1128 continue;
1129 }
1130
1131 #if CONFIG_MEMORYSTATUS
1132 if (target_foreground_process == TRUE && !memorystatus_is_foreground_locked(p)) {
1133 /*
1134 * Skip process not marked foreground.
1135 */
1136 proc_rele(p);
1137 continue;
1138 }
1139 #endif /* CONFIG_MEMORYSTATUS */
1140
1141 t = (struct task *)(proc_task(p));
1142 telemetry_update = update_knote_footprint_history(kn, t, curr_ts);
1143 *next_telemetry_update = MIN(*next_telemetry_update, telemetry_update);
1144
1145 vm_pressure_level_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(level);
1146
1147 if ((kn->kn_sfflags & dispatch_level) == 0) {
1148 proc_rele(p);
1149 continue;
1150 }
1151
1152 #if CONFIG_MEMORYSTATUS
1153 if (target_foreground_process == FALSE && !memorystatus_bg_pressure_eligible(p)) {
1154 VM_PRESSURE_DEBUG(1, "[vm_pressure] skipping process %d\n", proc_getpid(p));
1155 proc_rele(p);
1156 continue;
1157 }
1158 #endif /* CONFIG_MEMORYSTATUS */
1159
1160 #if XNU_TARGET_OS_OSX
1161 curr_task_importance = task_importance_estimate(t);
1162 #else /* XNU_TARGET_OS_OSX */
1163 curr_task_importance = p->p_memstat_effectivepriority;
1164 #endif /* XNU_TARGET_OS_OSX */
1165
1166 /*
1167 * Privileged listeners are only considered in the multi-level pressure scheme
1168 * AND only if the pressure is increasing.
1169 */
1170 if (level > 0) {
1171 if (task_has_been_notified(t, level) == FALSE) {
1172 /*
1173 * Is this a privileged listener?
1174 */
1175 if (task_low_mem_privileged_listener(t, FALSE, &privileged_listener) == 0) {
1176 if (privileged_listener) {
1177 kn_max = kn;
1178 proc_rele(p);
1179 goto done_scanning;
1180 }
1181 }
1182 } else {
1183 proc_rele(p);
1184 continue;
1185 }
1186 } else if (level == 0) {
1187 /*
1188 * Task wasn't notified when the pressure was increasing and so
1189 * no need to notify it that the pressure is decreasing.
1190 */
1191 if ((task_has_been_notified(t, kVMPressureWarning) == FALSE) && (task_has_been_notified(t, kVMPressureCritical) == FALSE)) {
1192 proc_rele(p);
1193 continue;
1194 }
1195 }
1196
1197 /*
1198 * We don't want a small process to block large processes from
1199 * being notified again. <rdar://problem/7955532>
1200 */
1201 resident_size = (get_task_phys_footprint(t)) / (1024 * 1024ULL); /* MB */
1202
1203 if (resident_size >= vm_pressure_task_footprint_min) {
1204 if (level > 0) {
1205 /*
1206 * Warning or Critical Pressure.
1207 */
1208 if (pressure_increase) {
1209 if ((curr_task_importance < selected_task_importance) ||
1210 ((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
1211 /*
1212 * We have found a candidate process which is:
1213 * a) at a lower importance than the current selected process
1214 * OR
1215 * b) has importance equal to that of the current selected process but is larger
1216 */
1217
1218 consider_knote = TRUE;
1219 }
1220 } else {
1221 if ((curr_task_importance > selected_task_importance) ||
1222 ((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
1223 /*
1224 * We have found a candidate process which is:
1225 * a) at a higher importance than the current selected process
1226 * OR
1227 * b) has importance equal to that of the current selected process but is larger
1228 */
1229
1230 consider_knote = TRUE;
1231 }
1232 }
1233 } else if (level == 0) {
1234 /*
1235 * Pressure back to normal.
1236 */
1237 if ((curr_task_importance > selected_task_importance) ||
1238 ((curr_task_importance == selected_task_importance) && (resident_size > resident_max))) {
1239 consider_knote = TRUE;
1240 }
1241 }
1242
1243 if (consider_knote) {
1244 resident_max = resident_size;
1245 kn_max = kn;
1246 selected_task_importance = curr_task_importance;
1247 consider_knote = FALSE; /* reset for the next candidate */
1248 }
1249 } else {
1250 /* There was no candidate with enough resident memory to scavenge */
1251 VM_PRESSURE_DEBUG(0, "[vm_pressure] threshold failed for pid %d with %llu resident...\n", proc_getpid(p), resident_size);
1252 }
1253 proc_rele(p);
1254 }
1255
1256 done_scanning:
1257 if (kn_max) {
1258 VM_DEBUG_CONSTANT_EVENT(vm_pressure_event, DBG_VM_PRESSURE_EVENT, DBG_FUNC_NONE, proc_getpid(knote_get_kq(kn_max)->kq_p), resident_max, 0, 0);
1259 VM_PRESSURE_DEBUG(1, "[vm_pressure] sending event to pid %d with %llu resident\n", proc_getpid(knote_get_kq(kn_max)->kq_p), resident_max);
1260 }
1261
1262 return kn_max;
1263 }
1264
1265 /*
1266 * To avoid notification storms in a system with sawtooth behavior of pressure levels eg:
1267 * Normal -> warning (notify clients) -> critical (notify) -> warning (notify) -> critical (notify) -> warning (notify)...
1268 *
1269 * We have 'resting' periods: WARNING_NOTIFICATION_RESTING_PERIOD and CRITICAL_NOTIFICATION_RESTING_PERIOD
1270 *
1271 * So it would look like:-
1272 * Normal -> warning (notify) -> critical (notify) -> warning (notify if it has been RestPeriod since last warning) -> critical (notify if it has been RestPeriod since last critical) -> ...
1273 *
1274 * That's what these 2 timestamps below signify.
1275 */
1276
1277 uint64_t next_warning_notification_sent_at_ts = 0;
1278 uint64_t next_critical_notification_sent_at_ts = 0;
1279
1280 boolean_t memorystatus_manual_testing_on = FALSE;
1281 vm_pressure_level_t memorystatus_manual_testing_level = kVMPressureNormal;
1282
1283 TUNABLE_DEV_WRITEABLE(unsigned int, memstat_sustained_pressure_max_pri, "memstat_sustained_pressure_max_pri", JETSAM_PRIORITY_IDLE);
1284 #if DEVELOPMENT || DEBUG
1285 SYSCTL_UINT(_kern_memorystatus, OID_AUTO, sustained_pressure_max_pri, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &memstat_sustained_pressure_max_pri, 0, "");
1286 #endif /* DEVELOPMENT || DEBUG */
1287
1288 #if CONFIG_JETSAM
1289 #define MEMSTAT_PRESSURE_CONFIG_DEFAULT (MEMSTAT_WARNING_KILL_SUSTAINED)
1290 #else
1291 #define MEMSTAT_PRESSURE_CONFIG_DEFAULT (MEMSTAT_WARNING_KILL_IDLE_THROTTLED | MEMSTAT_CRITICAL_PURGE_CACHES)
1292 #endif
1293
1294 TUNABLE_WRITEABLE(memstat_pressure_options_t, memstat_pressure_config,
1295 "memorystatus_pressure_config", MEMSTAT_PRESSURE_CONFIG_DEFAULT);
1296 EXPERIMENT_FACTOR_UINT(memorystatus_pressure_config, &memstat_pressure_config,
1297 0, MEMSTAT_PRESSURE_CONFIG_MAX,
1298 "Which actions to take in response to rising VM pressure");
1299 #if DEVELOPMENT || DEBUG
1300 SYSCTL_UINT(_kern_memorystatus, OID_AUTO, pressure_config,
1301 CTLFLAG_RW | CTLFLAG_LOCKED, &memstat_pressure_config, 0,
1302 "How to respond to VM pressure");
1303
1304 static int
1305 sysctl_memstat_should_kill_sustained SYSCTL_HANDLER_ARGS
1306 {
1307 int old = !!(memstat_pressure_config & MEMSTAT_WARNING_KILL_SUSTAINED);
1308 int new, changed;
1309
1310 int ret = sysctl_io_number(req, old, sizeof(old), &new, &changed);
1311
1312 if (changed) {
1313 if (new) {
1314 memstat_pressure_config |= MEMSTAT_WARNING_KILL_SUSTAINED;
1315 } else {
1316 memstat_pressure_config &= ~MEMSTAT_WARNING_KILL_SUSTAINED;
1317 }
1318 }
1319 return ret;
1320 }
1321
1322 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_should_kill_on_sustained_pressure,
1323 CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0, sysctl_memstat_should_kill_sustained, "IU",
1324 "Whether to kill idle processes under sustained pressure");
1325 #endif
1326
1327 /*
1328 * TODO(jason): The memorystatus thread should be responsible for this
1329 * It can just check how long the pressure level has been at warning and the timestamp
1330 * of the last sustained pressure kill.
1331 */
1332 static void
sustained_pressure_handler(void * arg0 __unused,void * arg1 __unused)1333 sustained_pressure_handler(void* arg0 __unused, void* arg1 __unused)
1334 {
1335 int max_kills = 0, kill_count = 0;
1336 /*
1337 * Pressure has been elevated for too long.
1338 * We don't want to leave the system in this state as it can delay background
1339 * work indefinitely & drain battery.
1340 *
1341 * Try to return the system to normal via jetsam.
1342 * We'll run through the idle band up to 2 times.
1343 * If the pressure hasn't been relieved by then, the problem is memory
1344 * consumption in a higher band and this churn is probably doing more harm than good.
1345 */
1346 max_kills = memstat_get_proccnt_upto_priority(memstat_sustained_pressure_max_pri) * 2;
1347 memorystatus_log("memorystatus: Pressure level has been elevated for too long. killing up to %d idle processes\n", max_kills);
1348 while (memorystatus_vm_pressure_level != kVMPressureNormal && kill_count < max_kills) {
1349 bool killed = memorystatus_kill_on_sustained_pressure();
1350 if (killed) {
1351 /*
1352 * Pause before our next kill & see if pressure reduces.
1353 */
1354 delay((int)(memorystatus_kill_on_sustained_pressure_delay_ms * NSEC_PER_MSEC / NSEC_PER_USEC));
1355 kill_count++;
1356 memorystatus_kill_on_sustained_pressure_count++;
1357 os_atomic_inc(&memstat_cur_interval.num_kills, relaxed);
1358 } else {
1359 /* Nothing left to kill */
1360 break;
1361 }
1362 }
1363 if (memorystatus_vm_pressure_level != kVMPressureNormal) {
1364 memorystatus_log("memorystatus: Killed %d idle processes due to sustained pressure, but device didn't quiesce. Giving up.\n", kill_count);
1365 }
1366 }
1367
1368 /*
1369 * Returns the number of processes registered for notifications at this level.
1370 */
1371 static size_t
memorystatus_klist_length(int level)1372 memorystatus_klist_length(int level)
1373 {
1374 LCK_MTX_ASSERT(&memorystatus_klist_mutex, LCK_MTX_ASSERT_OWNED);
1375 struct knote *kn;
1376 size_t count = 0;
1377 int knote_pressure_level = convert_internal_pressure_level_to_dispatch_level(level);
1378 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
1379 if (kn->kn_sfflags & knote_pressure_level) {
1380 count++;
1381 }
1382 }
1383 return count;
1384 }
1385
1386 /*
1387 * Starts a pressure interval, setting up tracking for it
1388 */
1389 static void
memstat_pressure_interval_start(uint64_t curr_ts)1390 memstat_pressure_interval_start(uint64_t curr_ts)
1391 {
1392 LCK_MTX_ASSERT(&memorystatus_klist_mutex, LCK_MTX_ASSERT_OWNED);
1393 memstat_cur_interval.num_procs = 0;
1394 memstat_cur_interval.num_notifs = 0;
1395 memstat_cur_interval.num_transitions = 0;
1396 memstat_cur_interval.start_mt = curr_ts;
1397 os_atomic_store(&memstat_cur_interval.num_kills, 0, relaxed);
1398 memstat_cur_interval.max_level = kVMPressureNormal;
1399 }
1400
1401 /*
1402 * Ends a pressure interval, sending all telemetry associated with it
1403 */
1404 static void
memstat_pressure_interval_end(void)1405 memstat_pressure_interval_end(void)
1406 {
1407 LCK_MTX_ASSERT(&memorystatus_klist_mutex, LCK_MTX_ASSERT_OWNED);
1408 memorystatus_pressure_interval_send();
1409 memorystatus_pressure_proc_telemetry_send();
1410 }
1411
1412 /*
1413 * Updates the pressure interval when the pressure level changes
1414 */
1415 static void
memstat_pressure_interval_update(vm_pressure_level_t new_level)1416 memstat_pressure_interval_update(vm_pressure_level_t new_level)
1417 {
1418 LCK_MTX_ASSERT(&memorystatus_klist_mutex, LCK_MTX_ASSERT_OWNED);
1419 memstat_cur_interval.num_transitions++;
1420 if (new_level <= memstat_cur_interval.max_level) {
1421 return;
1422 }
1423 memstat_cur_interval.num_procs = memorystatus_klist_length(new_level);
1424 memstat_cur_interval.max_level = new_level;
1425 }
1426
1427
1428 /*
1429 * Updates the footprint telemetry for procs that have received notifications.
1430 */
1431 static void
update_footprints_for_telemetry(void * arg0 __unused,void * arg1 __unused)1432 update_footprints_for_telemetry(void* arg0 __unused, void* arg1 __unused)
1433 {
1434 uint64_t curr_ts = mach_absolute_time(), next_telemetry_update = UINT64_MAX;
1435 struct knote *kn;
1436
1437 memorystatus_klist_lock();
1438 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
1439 proc_t p = PROC_NULL;
1440 struct task* t = TASK_NULL;
1441 uint64_t telemetry_update;
1442
1443 p = proc_ref(knote_get_kq(kn)->kq_p, false);
1444 if (p == PROC_NULL) {
1445 continue;
1446 }
1447 t = (struct task *)(proc_task(p));
1448 proc_rele(p);
1449 p = PROC_NULL;
1450 telemetry_update = update_knote_footprint_history(kn, t, curr_ts);
1451 next_telemetry_update = MIN(next_telemetry_update, telemetry_update);
1452 }
1453 memorystatus_klist_unlock();
1454 if (next_telemetry_update != UINT64_MAX) {
1455 uint64_t next_update_seconds;
1456 absolutetime_to_nanoseconds(next_telemetry_update, &next_update_seconds);
1457 next_update_seconds /= NSEC_PER_SEC;
1458 thread_call_enter_delayed(memorystatus_notify_update_telemetry_thread_call, next_telemetry_update);
1459 }
1460 }
1461
1462 kern_return_t
memorystatus_update_vm_pressure(boolean_t target_foreground_process)1463 memorystatus_update_vm_pressure(boolean_t target_foreground_process)
1464 {
1465 struct knote *kn_max = NULL;
1466 struct knote *kn_cur = NULL, *kn_temp = NULL;/* for safe list traversal */
1467 pid_t target_pid = -1;
1468 struct klist dispatch_klist = { NULL };
1469 proc_t target_proc = PROC_NULL;
1470 struct task *task = NULL;
1471 boolean_t found_candidate = FALSE;
1472
1473 static vm_pressure_level_t level_snapshot = kVMPressureNormal;
1474 static vm_pressure_level_t prev_level_snapshot = kVMPressureNormal;
1475 boolean_t smoothing_window_started = FALSE;
1476 struct timeval smoothing_window_start_tstamp = {0, 0};
1477 struct timeval curr_tstamp = {0, 0};
1478 int64_t elapsed_msecs = 0;
1479 uint64_t curr_ts = mach_absolute_time(), next_telemetry_update = UINT64_MAX;
1480
1481
1482 uint64_t logging_now;
1483 absolutetime_to_nanoseconds(curr_ts, &logging_now);
1484 #if !CONFIG_JETSAM
1485 #define MAX_IDLE_KILLS 100 /* limit the number of idle kills allowed */
1486
1487 int idle_kill_counter = 0;
1488
1489 /*
1490 * On desktop we take this opportunity to free up memory pressure
1491 * by immediately killing idle exitable processes. We use a delay
1492 * to avoid overkill. And we impose a max counter as a fail safe
1493 * in case daemons re-launch too fast.
1494 */
1495 while (memstat_pressure_config & MEMSTAT_WARNING_KILL_IDLE_THROTTLED &&
1496 memorystatus_vm_pressure_level != kVMPressureNormal &&
1497 idle_kill_counter < MAX_IDLE_KILLS) {
1498 uint64_t footprint;
1499 if (!memstat_kill_idle_process(kMemorystatusKilledIdleExit, &footprint)) {
1500 /* No idle exitable processes left to kill */
1501 break;
1502 }
1503 idle_kill_counter++;
1504
1505 if (memorystatus_manual_testing_on == TRUE) {
1506 /*
1507 * Skip the delay when testing
1508 * the pressure notification scheme.
1509 */
1510 } else {
1511 delay(1 * USEC_PER_SEC);
1512 }
1513 }
1514 #endif /* !CONFIG_JETSAM */
1515
1516 if (level_snapshot != kVMPressureNormal) {
1517 /*
1518 * Check to see if we are still in the 'resting' period
1519 * after having notified all clients interested in
1520 * a particular pressure level.
1521 */
1522
1523 level_snapshot = memorystatus_vm_pressure_level;
1524
1525 if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) {
1526 if (next_warning_notification_sent_at_ts) {
1527 if (curr_ts < next_warning_notification_sent_at_ts) {
1528 delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */);
1529 return KERN_SUCCESS;
1530 }
1531
1532 next_warning_notification_sent_at_ts = 0;
1533 memorystatus_klist_reset_all_for_level(kVMPressureWarning);
1534 }
1535 } else if (level_snapshot == kVMPressureCritical) {
1536 if (next_critical_notification_sent_at_ts) {
1537 if (curr_ts < next_critical_notification_sent_at_ts) {
1538 delay(INTER_NOTIFICATION_DELAY * 4 /* 1 sec */);
1539 return KERN_SUCCESS;
1540 }
1541 next_critical_notification_sent_at_ts = 0;
1542 memorystatus_klist_reset_all_for_level(kVMPressureCritical);
1543 }
1544 }
1545 }
1546
1547 if (memstat_pressure_config & MEMSTAT_WARNING_KILL_SUSTAINED) {
1548 if (memorystatus_vm_pressure_level == kVMPressureNormal && prev_level_snapshot != kVMPressureNormal) {
1549 memorystatus_log("memorystatus: Pressure has returned to level %d. Cancelling scheduled jetsam\n", memorystatus_vm_pressure_level);
1550 thread_call_cancel(sustained_pressure_handler_thread_call);
1551 } else if (memorystatus_vm_pressure_level != kVMPressureNormal && prev_level_snapshot == kVMPressureNormal) {
1552 /*
1553 * Pressure has increased from normal.
1554 * Hopefully the notifications will relieve it,
1555 * but as a fail-safe we'll trigger jetsam
1556 * after a configurable amount of time.
1557 */
1558 memorystatus_log("memorystatus: Pressure level has increased from %d to %d. Scheduling jetsam.\n", prev_level_snapshot, memorystatus_vm_pressure_level);
1559 uint64_t kill_time;
1560 nanoseconds_to_absolutetime(memorystatus_kill_on_sustained_pressure_window_s * NSEC_PER_SEC, &kill_time);
1561 kill_time += mach_absolute_time();
1562 thread_call_enter_delayed(sustained_pressure_handler_thread_call, kill_time);
1563 }
1564 }
1565
1566 while (1) {
1567 /*
1568 * There is a race window here. But it's not clear
1569 * how much we benefit from having extra synchronization.
1570 */
1571 level_snapshot = memorystatus_vm_pressure_level;
1572
1573 if (prev_level_snapshot > level_snapshot) {
1574 /*
1575 * Pressure decreased? Let's take a little breather
1576 * and see if this condition stays.
1577 */
1578 if (smoothing_window_started == FALSE) {
1579 smoothing_window_started = TRUE;
1580 microuptime(&smoothing_window_start_tstamp);
1581 }
1582
1583 microuptime(&curr_tstamp);
1584 timevalsub(&curr_tstamp, &smoothing_window_start_tstamp);
1585 elapsed_msecs = curr_tstamp.tv_sec * 1000 + curr_tstamp.tv_usec / 1000;
1586
1587 if (elapsed_msecs < VM_PRESSURE_DECREASED_SMOOTHING_PERIOD) {
1588 delay(INTER_NOTIFICATION_DELAY);
1589 continue;
1590 }
1591 }
1592
1593 prev_level_snapshot = level_snapshot;
1594 smoothing_window_started = FALSE;
1595
1596 if (memstat_pressure_config & MEMSTAT_WARNING_KILL_LONG_IDLE &&
1597 level_snapshot >= kVMPressureWarning &&
1598 memstat_get_long_idle_proccnt() > 0) {
1599 /* There are long-idle daemons to kill */
1600 memorystatus_thread_wake();
1601 } else if (level_snapshot == kVMPressureCritical) {
1602 if (memstat_pressure_config & MEMSTAT_CRITICAL_PURGE_CACHES) {
1603 uint64_t now = mach_absolute_time();
1604 uint64_t delta_ns;
1605 absolutetime_to_nanoseconds(now - memstat_last_cache_purge_ts, &delta_ns);
1606 if (delta_ns >= memstat_cache_purge_backoff_ns) {
1607 /* Wake up the jetsam thread to purge caches */
1608 memorystatus_thread_wake();
1609 }
1610 } else if (memstat_pressure_config & MEMSTAT_CRITICAL_KILL_IDLE &&
1611 memstat_get_idle_proccnt() > 0) {
1612 memorystatus_thread_wake();
1613 }
1614 }
1615
1616 memorystatus_klist_lock();
1617
1618 /* Interval tracking & telemetry */
1619 if (prev_level_snapshot != level_snapshot) {
1620 if (level_snapshot == kVMPressureNormal) {
1621 memstat_pressure_interval_end();
1622 } else if (prev_level_snapshot == kVMPressureNormal) {
1623 memstat_pressure_interval_start(curr_ts);
1624 }
1625
1626 memstat_pressure_interval_update(level_snapshot);
1627 }
1628
1629 kn_max = vm_pressure_select_optimal_candidate_to_notify(&memorystatus_klist, level_snapshot, target_foreground_process, &next_telemetry_update);
1630
1631 if (kn_max == NULL) {
1632 memorystatus_klist_unlock();
1633
1634 /*
1635 * No more level-based clients to notify.
1636 *
1637 * Start the 'resting' window within which clients will not be re-notified.
1638 */
1639
1640 if (level_snapshot != kVMPressureNormal) {
1641 if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) {
1642 nanoseconds_to_absolutetime(WARNING_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts);
1643
1644 /* Next warning notification (if nothing changes) won't be sent before...*/
1645 next_warning_notification_sent_at_ts = mach_absolute_time() + curr_ts;
1646 }
1647
1648 if (level_snapshot == kVMPressureCritical) {
1649 nanoseconds_to_absolutetime(CRITICAL_NOTIFICATION_RESTING_PERIOD * NSEC_PER_SEC, &curr_ts);
1650
1651 /* Next critical notification (if nothing changes) won't be sent before...*/
1652 next_critical_notification_sent_at_ts = mach_absolute_time() + curr_ts;
1653 }
1654 }
1655 absolutetime_to_nanoseconds(mach_absolute_time(), &logging_now);
1656 if (next_telemetry_update != UINT64_MAX) {
1657 thread_call_enter_delayed(memorystatus_notify_update_telemetry_thread_call, next_telemetry_update);
1658 } else {
1659 thread_call_cancel(memorystatus_notify_update_telemetry_thread_call);
1660 }
1661 return KERN_FAILURE;
1662 }
1663
1664 target_proc = proc_ref(knote_get_kq(kn_max)->kq_p, false);
1665 if (target_proc == PROC_NULL) {
1666 memorystatus_klist_unlock();
1667 continue;
1668 }
1669
1670 target_pid = proc_getpid(target_proc);
1671
1672 task = (struct task *)(proc_task(target_proc));
1673
1674 if (level_snapshot != kVMPressureNormal) {
1675 if (level_snapshot == kVMPressureWarning || level_snapshot == kVMPressureUrgent) {
1676 if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_WARN, task, 0, kVMPressureWarning) == TRUE) {
1677 found_candidate = TRUE;
1678 }
1679 } else {
1680 if (level_snapshot == kVMPressureCritical) {
1681 if (is_knote_registered_modify_task_pressure_bits(kn_max, NOTE_MEMORYSTATUS_PRESSURE_CRITICAL, task, 0, kVMPressureCritical) == TRUE) {
1682 found_candidate = TRUE;
1683 }
1684 }
1685 }
1686 } else {
1687 if (kn_max->kn_sfflags & NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
1688 task_clear_has_been_notified(task, kVMPressureWarning);
1689 task_clear_has_been_notified(task, kVMPressureCritical);
1690
1691 found_candidate = TRUE;
1692 }
1693 }
1694
1695 if (found_candidate == FALSE) {
1696 proc_rele(target_proc);
1697 memorystatus_klist_unlock();
1698 continue;
1699 }
1700
1701 SLIST_FOREACH_SAFE(kn_cur, &memorystatus_klist, kn_selnext, kn_temp) {
1702 int knote_pressure_level = convert_internal_pressure_level_to_dispatch_level(level_snapshot);
1703
1704 if (is_knote_registered_modify_task_pressure_bits(kn_cur, knote_pressure_level, task, 0, level_snapshot) == TRUE) {
1705 proc_t knote_proc = knote_get_kq(kn_cur)->kq_p;
1706 pid_t knote_pid = proc_getpid(knote_proc);
1707 if (knote_pid == target_pid) {
1708 KNOTE_DETACH(&memorystatus_klist, kn_cur);
1709 KNOTE_ATTACH(&dispatch_klist, kn_cur);
1710 }
1711 }
1712 }
1713
1714 if (level_snapshot != kVMPressureNormal) {
1715 uint16_t num_notifications;
1716 if (os_convert_overflow(memstat_cur_interval.num_notifs, &num_notifications)) {
1717 num_notifications = UINT16_MAX;
1718 }
1719 mark_knote_send_time(kn_max, task,
1720 convert_internal_pressure_level_to_dispatch_level(level_snapshot),
1721 num_notifications);
1722 memstat_cur_interval.num_notifs++;
1723 }
1724
1725 KNOTE(&dispatch_klist, (level_snapshot != kVMPressureNormal) ? kMemorystatusPressure : kMemorystatusNoPressure);
1726
1727 SLIST_FOREACH_SAFE(kn_cur, &dispatch_klist, kn_selnext, kn_temp) {
1728 KNOTE_DETACH(&dispatch_klist, kn_cur);
1729 KNOTE_ATTACH(&memorystatus_klist, kn_cur);
1730 }
1731
1732 memorystatus_klist_unlock();
1733
1734 microuptime(&target_proc->vm_pressure_last_notify_tstamp);
1735 proc_rele(target_proc);
1736
1737 if (memorystatus_manual_testing_on == TRUE && target_foreground_process == TRUE) {
1738 break;
1739 }
1740
1741 if (memorystatus_manual_testing_on == TRUE) {
1742 /*
1743 * Testing out the pressure notification scheme.
1744 * No need for delays etc.
1745 */
1746 } else {
1747 uint32_t sleep_interval = INTER_NOTIFICATION_DELAY;
1748 #if CONFIG_JETSAM
1749
1750 uint32_t critical_threshold = memorystatus_get_critical_page_shortage_threshold();
1751 uint32_t soft_threshold = memorystatus_get_soft_memlimit_page_shortage_threshold();
1752 assert(soft_threshold >= critical_threshold);
1753
1754 uint32_t backoff_threshold = soft_threshold -
1755 ((soft_threshold - critical_threshold) / 2);
1756
1757 if (memorystatus_get_available_page_count() <= backoff_threshold) {
1758 /*
1759 * We are nearing the critcal mark fast and can't afford to wait between
1760 * notifications.
1761 */
1762 sleep_interval = 0;
1763 }
1764 #endif /* CONFIG_JETSAM */
1765
1766 if (sleep_interval) {
1767 delay(sleep_interval);
1768 }
1769 }
1770 }
1771
1772 return KERN_SUCCESS;
1773 }
1774
1775 static uint32_t
convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t internal_pressure_level)1776 convert_internal_pressure_level_to_dispatch_level(vm_pressure_level_t internal_pressure_level)
1777 {
1778 uint32_t dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
1779
1780 switch (internal_pressure_level) {
1781 case kVMPressureNormal:
1782 {
1783 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_NORMAL;
1784 break;
1785 }
1786
1787 case kVMPressureWarning:
1788 case kVMPressureUrgent:
1789 {
1790 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_WARN;
1791 break;
1792 }
1793
1794 case kVMPressureCritical:
1795 {
1796 dispatch_level = NOTE_MEMORYSTATUS_PRESSURE_CRITICAL;
1797 break;
1798 }
1799
1800 default:
1801 break;
1802 }
1803
1804 return dispatch_level;
1805 }
1806
1807 /*
1808 * Issue a wakeup to any threads listening for jetsam pressure via
1809 * `mach_vm_pressure_level_monitor`. Subscribers should respond to these
1810 * notifications by freeing cached memory.
1811 */
1812 void
memorystatus_broadcast_jetsam_pressure(vm_pressure_level_t pressure_level)1813 memorystatus_broadcast_jetsam_pressure(vm_pressure_level_t pressure_level)
1814 {
1815 uint64_t now;
1816 uint32_t *waiters = NULL;
1817 uint64_t *last_notification_ns = NULL;
1818
1819 switch (pressure_level) {
1820 case kVMPressureForegroundJetsam:
1821 waiters = &memorystatus_jetsam_fg_band_waiters;
1822 last_notification_ns = &memorystatus_jetsam_fg_band_timestamp_ns;
1823 break;
1824 case kVMPressureBackgroundJetsam:
1825 waiters = &memorystatus_jetsam_bg_band_waiters;
1826 last_notification_ns = &memorystatus_jetsam_bg_band_timestamp_ns;
1827 break;
1828 default:
1829 panic("Unexpected non-jetsam pressure level %d", pressure_level);
1830 }
1831
1832 lck_mtx_lock(&memorystatus_jetsam_broadcast_lock);
1833 absolutetime_to_nanoseconds(mach_absolute_time(), &now);
1834
1835 if (now - *last_notification_ns < memorystatus_jetsam_notification_delay_ns) {
1836 lck_mtx_unlock(&memorystatus_jetsam_broadcast_lock);
1837 return;
1838 }
1839
1840 if (*waiters > 0) {
1841 memorystatus_log("memorystatus: issuing %s jetsam pressure notification to %d waiters",
1842 pressure_level == kVMPressureForegroundJetsam ?
1843 "foreground" : "background", *waiters);
1844 thread_wakeup((event_t)waiters);
1845 *waiters = 0;
1846 *last_notification_ns = now;
1847 }
1848 lck_mtx_unlock(&memorystatus_jetsam_broadcast_lock);
1849 }
1850
1851 /*
1852 * Memorystatus notification debugging support
1853 */
1854
1855 #if DEVELOPMENT || DEBUG
1856
1857 static int
1858 sysctl_memorystatus_broadcast_jetsam_pressure SYSCTL_HANDLER_ARGS
1859 {
1860 int error = 0;
1861 vm_pressure_level_t pressure_level;
1862
1863 error = SYSCTL_IN(req, &pressure_level, sizeof(pressure_level));
1864 if (error) {
1865 return error;
1866 }
1867
1868 if (pressure_level == kVMPressureForegroundJetsam ||
1869 pressure_level == kVMPressureBackgroundJetsam) {
1870 memorystatus_broadcast_jetsam_pressure(pressure_level);
1871 } else {
1872 return EINVAL;
1873 }
1874
1875 return SYSCTL_OUT(req, &pressure_level, sizeof(pressure_level));
1876 }
1877
1878 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_broadcast_jetsam_pressure,
1879 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MASKED | CTLFLAG_LOCKED,
1880 0, 0, &sysctl_memorystatus_broadcast_jetsam_pressure, "I", "");
1881
1882 #endif /* DEVELOPMENT || DEBUG */
1883
1884 static int
1885 sysctl_memorystatus_vm_pressure_level SYSCTL_HANDLER_ARGS
1886 {
1887 #pragma unused(arg1, arg2, oidp)
1888 #if !XNU_TARGET_OS_OSX
1889 int error = 0;
1890
1891 error = priv_check_cred(kauth_cred_get(), PRIV_VM_PRESSURE, 0);
1892 if (error) {
1893 return error;
1894 }
1895
1896 #endif /* !XNU_TARGET_OS_OSX */
1897 uint32_t dispatch_level = convert_internal_pressure_level_to_dispatch_level(memorystatus_vm_pressure_level);
1898
1899 return SYSCTL_OUT(req, &dispatch_level, sizeof(dispatch_level));
1900 }
1901
1902 #if DEBUG || DEVELOPMENT
1903
1904 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
1905 0, 0, &sysctl_memorystatus_vm_pressure_level, "I", "");
1906
1907 #else /* DEBUG || DEVELOPMENT */
1908
1909 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_level, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED | CTLFLAG_MASKED,
1910 0, 0, &sysctl_memorystatus_vm_pressure_level, "I", "");
1911
1912 #endif /* DEBUG || DEVELOPMENT */
1913
1914 /*
1915 * Trigger levels to test the mechanism.
1916 * Can be used via a sysctl.
1917 */
1918 #define TEST_LOW_MEMORY_TRIGGER_ONE 1
1919 #define TEST_LOW_MEMORY_TRIGGER_ALL 2
1920 #define TEST_PURGEABLE_TRIGGER_ONE 3
1921 #define TEST_PURGEABLE_TRIGGER_ALL 4
1922 #define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE 5
1923 #define TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL 6
1924
1925 static int
1926 sysctl_memorypressure_manual_trigger SYSCTL_HANDLER_ARGS
1927 {
1928 #pragma unused(arg1, arg2)
1929
1930 int level = 0;
1931 int error = 0;
1932 int pressure_level = 0;
1933 int trigger_request = 0;
1934 int force_purge;
1935
1936 error = sysctl_handle_int(oidp, &level, 0, req);
1937 if (error || !req->newptr) {
1938 return error;
1939 }
1940
1941 memorystatus_manual_testing_on = TRUE;
1942
1943 trigger_request = (level >> 16) & 0xFFFF;
1944 pressure_level = (level & 0xFFFF);
1945
1946 if (trigger_request < TEST_LOW_MEMORY_TRIGGER_ONE ||
1947 trigger_request > TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL) {
1948 return EINVAL;
1949 }
1950 switch (pressure_level) {
1951 case NOTE_MEMORYSTATUS_PRESSURE_NORMAL:
1952 case NOTE_MEMORYSTATUS_PRESSURE_WARN:
1953 case NOTE_MEMORYSTATUS_PRESSURE_CRITICAL:
1954 break;
1955 default:
1956 return EINVAL;
1957 }
1958
1959 /*
1960 * The pressure level is being set from user-space.
1961 * And user-space uses the constants in sys/event.h
1962 * So we translate those events to our internal levels here.
1963 */
1964 if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
1965 memorystatus_manual_testing_level = kVMPressureNormal;
1966 force_purge = 0;
1967 } else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_WARN) {
1968 memorystatus_manual_testing_level = kVMPressureWarning;
1969 force_purge = vm_pageout_state.memorystatus_purge_on_warning;
1970 } else if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) {
1971 memorystatus_manual_testing_level = kVMPressureCritical;
1972 force_purge = vm_pageout_state.memorystatus_purge_on_critical;
1973 }
1974
1975 memorystatus_vm_pressure_level = memorystatus_manual_testing_level;
1976
1977 /* purge according to the new pressure level */
1978 switch (trigger_request) {
1979 case TEST_PURGEABLE_TRIGGER_ONE:
1980 case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE:
1981 if (force_purge == 0) {
1982 /* no purging requested */
1983 break;
1984 }
1985 vm_purgeable_object_purge_one_unlocked(force_purge);
1986 break;
1987 case TEST_PURGEABLE_TRIGGER_ALL:
1988 case TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL:
1989 if (force_purge == 0) {
1990 /* no purging requested */
1991 break;
1992 }
1993 while (vm_purgeable_object_purge_one_unlocked(force_purge)) {
1994 ;
1995 }
1996 break;
1997 }
1998
1999 if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ONE) ||
2000 (trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ONE)) {
2001 memorystatus_update_vm_pressure(TRUE);
2002 }
2003
2004 if ((trigger_request == TEST_LOW_MEMORY_TRIGGER_ALL) ||
2005 (trigger_request == TEST_LOW_MEMORY_PURGEABLE_TRIGGER_ALL)) {
2006 while (memorystatus_update_vm_pressure(FALSE) == KERN_SUCCESS) {
2007 continue;
2008 }
2009 }
2010
2011 if (pressure_level == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) {
2012 memorystatus_manual_testing_on = FALSE;
2013 }
2014
2015 return 0;
2016 }
2017
2018 SYSCTL_PROC(_kern, OID_AUTO, memorypressure_manual_trigger, CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED,
2019 0, 0, &sysctl_memorypressure_manual_trigger, "I", "");
2020
2021
2022 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_warning, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_pageout_state.memorystatus_purge_on_warning, 0, "");
2023 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_urgent, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &vm_pageout_state.memorystatus_purge_on_urgent, 0, "");
2024 SYSCTL_INT(_kern, OID_AUTO, memorystatus_purge_on_critical, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &vm_pageout_state.memorystatus_purge_on_critical, 0, "");
2025
2026 extern int vm_pressure_level_transition_threshold;
2027 SYSCTL_INT(_kern, OID_AUTO, vm_pressure_level_transition_threshold, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &vm_pressure_level_transition_threshold, 0, "");
2028
2029 #if DEBUG || DEVELOPMENT
2030 SYSCTL_UINT(_kern, OID_AUTO, memorystatus_vm_pressure_events_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &vm_pressure_events_enabled, 0, "");
2031
2032 #if 0
2033 #if CONFIG_JETSAM && VM_PRESSURE_EVENTS
2034 static boolean_t
2035 memorystatus_issue_pressure_kevent(boolean_t pressured)
2036 {
2037 memorystatus_klist_lock();
2038 KNOTE(&memorystatus_klist, pressured ? kMemorystatusPressure : kMemorystatusNoPressure);
2039 memorystatus_klist_unlock();
2040 return TRUE;
2041 }
2042 #endif /* CONFIG_JETSAM && VM_PRESSURE_EVENTS */
2043 #endif /* 0 */
2044
2045 /*
2046 * This routine is used for targeted notifications regardless of system memory pressure
2047 * and regardless of whether or not the process has already been notified.
2048 * It bypasses and has no effect on the only-one-notification per soft-limit policy.
2049 *
2050 * "memnote" is the current user.
2051 */
2052
2053 static int
2054 sysctl_memorystatus_vm_pressure_send SYSCTL_HANDLER_ARGS
2055 {
2056 #pragma unused(arg1, arg2)
2057 /* Need to be root or have memorystatus entitlement */
2058 if (!kauth_cred_issuser(kauth_cred_get()) && !IOCurrentTaskHasEntitlement(MEMORYSTATUS_ENTITLEMENT)) {
2059 return EPERM;
2060 }
2061
2062 int error = 0, pid = 0;
2063 struct knote *kn = NULL;
2064 boolean_t found_knote = FALSE;
2065 int fflags = 0; /* filter flags for EVFILT_MEMORYSTATUS */
2066 uint64_t value = 0;
2067
2068 error = sysctl_handle_quad(oidp, &value, 0, req);
2069 if (error || !req->newptr) {
2070 return error;
2071 }
2072
2073 /*
2074 * Find the pid in the low 32 bits of value passed in.
2075 */
2076 pid = (int)(value & 0xFFFFFFFF);
2077
2078 /*
2079 * Find notification in the high 32 bits of the value passed in.
2080 */
2081 fflags = (int)((value >> 32) & 0xFFFFFFFF);
2082
2083 /*
2084 * For backwards compatibility, when no notification is
2085 * passed in, default to the NOTE_MEMORYSTATUS_PRESSURE_WARN
2086 */
2087 if (fflags == 0) {
2088 fflags = NOTE_MEMORYSTATUS_PRESSURE_WARN;
2089 // printf("memorystatus_vm_pressure_send: using default notification [0x%x]\n", fflags);
2090 }
2091
2092 /* wake up everybody waiting for kVMPressureForegroundJetsam */
2093 if (fflags == NOTE_MEMORYSTATUS_JETSAM_FG_BAND) {
2094 memorystatus_broadcast_jetsam_pressure(kVMPressureForegroundJetsam);
2095 return error;
2096 }
2097
2098 /*
2099 * See event.h ... fflags for EVFILT_MEMORYSTATUS
2100 */
2101 if (!((fflags == NOTE_MEMORYSTATUS_PRESSURE_NORMAL) ||
2102 (fflags == NOTE_MEMORYSTATUS_PRESSURE_WARN) ||
2103 (fflags == NOTE_MEMORYSTATUS_PRESSURE_CRITICAL) ||
2104 (fflags == NOTE_MEMORYSTATUS_LOW_SWAP) ||
2105 (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) ||
2106 (fflags == NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) ||
2107 (((fflags & NOTE_MEMORYSTATUS_MSL_STATUS) != 0 &&
2108 ((fflags & ~NOTE_MEMORYSTATUS_MSL_STATUS) == 0))))) {
2109 memorystatus_log_error("memorystatus_vm_pressure_send: notification [0x%x] not supported\n", fflags);
2110 error = 1;
2111 return error;
2112 }
2113
2114 /*
2115 * Forcibly send pid a memorystatus notification.
2116 */
2117
2118 memorystatus_klist_lock();
2119
2120 SLIST_FOREACH(kn, &memorystatus_klist, kn_selnext) {
2121 proc_t knote_proc = knote_get_kq(kn)->kq_p;
2122 pid_t knote_pid = proc_getpid(knote_proc);
2123
2124 if (knote_pid == pid) {
2125 /*
2126 * Forcibly send this pid a memorystatus notification.
2127 */
2128 kn->kn_fflags = fflags;
2129 found_knote = TRUE;
2130 }
2131 }
2132
2133 if (found_knote) {
2134 KNOTE(&memorystatus_klist, 0);
2135 memorystatus_log_debug("memorystatus_vm_pressure_send: (value 0x%llx) notification [0x%x] sent to process [%d]\n", value, fflags, pid);
2136 error = 0;
2137 } else {
2138 memorystatus_log_error("memorystatus_vm_pressure_send: (value 0x%llx) notification [0x%x] not sent to process [%d] (none registered?)\n", value, fflags, pid);
2139 error = 1;
2140 }
2141
2142 memorystatus_klist_unlock();
2143
2144 return error;
2145 }
2146
2147 SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_pressure_send, CTLTYPE_QUAD | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_MASKED | CTLFLAG_ANYBODY,
2148 0, 0, &sysctl_memorystatus_vm_pressure_send, "Q", "");
2149
2150 #endif /* DEBUG || DEVELOPMENT */
2151
2152 #endif /* VM_PRESSURE_EVENTS */
2153