1 /*
2 * Copyright (c) 2010-2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31
32 #include <kern/kern_types.h>
33 #include <kern/ledger.h>
34 #include <kern/kalloc.h>
35 #include <kern/task.h>
36 #include <kern/thread.h>
37 #include <kern/coalition.h>
38
39 #include <kern/processor.h>
40 #include <kern/machine.h>
41 #include <kern/queue.h>
42 #include <kern/policy_internal.h>
43
44 #include <sys/errno.h>
45
46 #include <libkern/OSAtomic.h>
47 #include <mach/mach_types.h>
48 #include <os/overflow.h>
49
50 #include <vm/pmap.h>
51
52 /*
53 * Ledger entry flags. Bits in second nibble (masked by 0xF0) are used for
54 * ledger actions (LEDGER_ACTION_BLOCK, etc).
55 */
56 #define LF_ENTRY_ACTIVE 0x0001 /* entry is active if set */
57 #define LF_WAKE_NEEDED 0x0100 /* one or more threads are asleep */
58 #define LF_WAKE_INPROGRESS 0x0200 /* the wait queue is being processed */
59 #define LF_REFILL_SCHEDULED 0x0400 /* a refill timer has been set */
60 #define LF_REFILL_INPROGRESS 0x0800 /* the ledger is being refilled */
61 #define LF_CALLED_BACK 0x1000 /* callback was called for balance in deficit */
62 #define LF_WARNED 0x2000 /* callback was called for balance warning */
63 #define LF_TRACKING_MAX 0x4000 /* track max balance. Exclusive w.r.t refill */
64 #define LF_PANIC_ON_NEGATIVE 0x8000 /* panic if it goes negative */
65 #define LF_TRACK_CREDIT_ONLY 0x10000 /* only update "credit" */
66 #define LF_DIAG_WARNED 0x20000 /* callback was called for balance diag */
67 #define LF_DIAG_DISABLED 0x40000 /* diagnostics threshold are disabled at the moment */
68 #define LF_IS_COUNTER 0x80000 /* entry uses a scalable counter */
69
70
71 /*
72 * Ledger entry IDs are actually a tuple of (size, offset).
73 * For backwards compatibility, they're stored in an int.
74 * Size is stored in the upper 16 bits, and offset is stored in the lower 16 bits.
75 *
76 * Use the ENTRY_ID_SIZE and ENTRY_ID_OFFSET macros to extract size and offset.
77 */
78 #define ENTRY_ID_SIZE_SHIFT 16
79 #define ENTRY_ID_OFFSET_MASK ((1 << ENTRY_ID_SIZE_SHIFT) - 1)
80 #define ENTRY_ID_OFFSET(x) ((x) & (ENTRY_ID_OFFSET_MASK))
81 #define ENTRY_ID_SIZE_MASK (ENTRY_ID_OFFSET_MASK << ENTRY_ID_SIZE_SHIFT)
82 #define ENTRY_ID_SIZE(x) ((((uint32_t) (x)) & (ENTRY_ID_SIZE_MASK)) >> ENTRY_ID_SIZE_SHIFT)
83 _Static_assert(((sizeof(struct ledger_entry_small) << ENTRY_ID_SIZE_SHIFT) | (UINT16_MAX / sizeof(struct ledger_entry_small))) > 0, "Valid ledger index < 0");
84 _Static_assert(((sizeof(struct ledger_entry) << ENTRY_ID_SIZE_SHIFT) | (UINT16_MAX / sizeof(struct ledger_entry_small))) > 0, "Valid ledger index < 0");
85 _Static_assert(sizeof(int) * 8 >= ENTRY_ID_SIZE_SHIFT * 2, "Ledger indices don't fit in an int.");
86 #define MAX_LEDGER_ENTRIES (UINT16_MAX / sizeof(struct ledger_entry_small))
87
88 #define LEDGER_DIAG_MEM_THRESHOLD_SHIFT 20
89 #define LEDGER_DIAG_MEM_AMOUNT_TO_THRESHOLD(X) ((X) >> (LEDGER_DIAG_MEM_THRESHOLD_SHIFT))
90 #define LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(X) (((ledger_amount_t)(X)) << (LEDGER_DIAG_MEM_THRESHOLD_SHIFT))
91
92 /* These features can fit in a small ledger entry. All others require a full size ledger entry */
93 #define LEDGER_ENTRY_SMALL_FLAGS (LEDGER_ENTRY_ALLOW_PANIC_ON_NEGATIVE | LEDGER_ENTRY_ALLOW_INACTIVE)
94
95 /*
96 * struct ledger_entry_info is available to user space and used in ledger() syscall.
97 * Changing its size would cause memory corruption. See rdar://132747700
98 */
99 static_assert(sizeof(struct ledger_entry_info) == (6 * sizeof(int64_t)));
100 static_assert(sizeof(struct ledger_entry_info_v2) == (11 * sizeof(int64_t)));
101
102 /*
103 * Make sure ledger_entry_small and ledger_entry_counter are the same size.
104 */
105 static_assert(sizeof(struct ledger_entry_small) == sizeof(struct ledger_entry_counter));
106
107 /* Turn on to debug invalid ledger accesses */
108 #if MACH_ASSERT
109 #define PANIC_ON_INVALID_LEDGER_ACCESS 1
110 #endif /* MACH_ASSERT */
111
112 static inline volatile uint32_t *
get_entry_flags(ledger_t l,int index)113 get_entry_flags(ledger_t l, int index)
114 {
115 assert(l != NULL);
116
117 uint16_t size, offset;
118 size = ENTRY_ID_SIZE(index);
119 offset = ENTRY_ID_OFFSET(index);
120 struct ledger_entry_small *les = &l->l_entries[offset];
121 if (size == sizeof(struct ledger_entry)) {
122 return &((struct ledger_entry *)les)->le_flags;
123 } else if (size == sizeof(struct ledger_entry_small)) {
124 return &les->les_flags;
125 } else {
126 panic("Unknown ledger entry size! ledger=%p, index=0x%x, entry_size=%d\n", l, index, size);
127 }
128 }
129
130 #if PANIC_ON_INVALID_LEDGER_ACCESS
131 #define INVALID_LEDGER_ACCESS(l, e) if ((e) != -1) panic("Invalid ledger access: ledger=%p, entry=0x%x, entry_size=0x%x, entry_offset=0x%x\n", \
132 (l), (e), (ENTRY_ID_SIZE((e))), ENTRY_ID_OFFSET((e)));
133 #else
134 #define INVALID_LEDGER_ACCESS(l, e)
135 #endif /* PANIC_ON_INVALID_LEDGER_ACCESS */
136
137 /* Determine whether a ledger entry exists */
138 static inline bool
is_entry_valid(ledger_t l,int entry)139 is_entry_valid(ledger_t l, int entry)
140 {
141 uint32_t size, offset, end_offset;
142 size = ENTRY_ID_SIZE(entry);
143 offset = ENTRY_ID_OFFSET(entry);
144 if (l == NULL) {
145 return false;
146 }
147 if (os_mul_overflow(offset, sizeof(struct ledger_entry_small), &offset) || offset >= l->l_size) {
148 INVALID_LEDGER_ACCESS(l, entry);
149 return false;
150 }
151 if (os_add_overflow(size, offset, &end_offset) || end_offset > l->l_size) {
152 INVALID_LEDGER_ACCESS(l, entry);
153 return false;
154 }
155 return true;
156 }
157
158 static inline bool
is_entry_active(ledger_t l,int entry)159 is_entry_active(ledger_t l, int entry)
160 {
161 uint32_t flags = *get_entry_flags(l, entry);
162 if ((flags & LF_ENTRY_ACTIVE) != LF_ENTRY_ACTIVE) {
163 return false;
164 }
165
166 return true;
167 }
168
169 static inline bool
is_entry_valid_and_active(ledger_t l,int entry)170 is_entry_valid_and_active(ledger_t l, int entry)
171 {
172 return is_entry_valid(l, entry) && is_entry_active(l, entry);
173 }
174
175 #define ASSERT(a) assert(a)
176
177 #ifdef LEDGER_DEBUG
178 int ledger_debug = 0;
179
180 #define lprintf(a) if (ledger_debug) { \
181 printf("%lld ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \
182 printf a ; \
183 }
184 #else
185 #define lprintf(a)
186 #endif
187
188 struct ledger_callback {
189 ledger_callback_t lc_func;
190 const void *lc_param0;
191 const void *lc_param1;
192 };
193
194 struct entry_template {
195 char et_key[LEDGER_NAME_MAX];
196 char et_group[LEDGER_NAME_MAX];
197 char et_units[LEDGER_NAME_MAX];
198 uint32_t et_flags;
199 uint16_t et_size;
200 uint16_t et_offset;
201 struct ledger_callback *et_callback;
202 };
203
204 LCK_GRP_DECLARE(ledger_lck_grp, "ledger");
205 os_refgrp_decl(static, ledger_refgrp, "ledger", NULL);
206
207 /*
208 * Modifying the reference count, table size, table contents, lt_next_offset, or lt_entries_lut,
209 * requires holding the lt_lock. Modfying the table address requires both
210 * lt_lock and setting the inuse bit. This means that the lt_entries field can
211 * be safely dereferenced if you hold either the lock or the inuse bit. The
212 * inuse bit exists solely to allow us to swap in a new, larger entries
213 * table without requiring a full lock to be acquired on each lookup.
214 * Accordingly, the inuse bit should never be held for longer than it takes
215 * to extract a value from the table - i.e., 2 or 3 memory references.
216 */
217 struct ledger_template {
218 const char *lt_name;
219 int lt_refs;
220 volatile uint32_t lt_inuse;
221 lck_mtx_t lt_lock;
222 zone_t lt_zone;
223 bool lt_initialized;
224 uint16_t lt_next_offset;
225 uint16_t lt_cnt;
226 uint16_t lt_table_size;
227 struct entry_template *lt_entries;
228 /* Lookup table to go from entry_offset to index in the lt_entries table. */
229 uint16_t *lt_entries_lut;
230 #if ATOMIC_COUNTER_USE_PERCPU
231 /* Number of counters in this template */
232 uint16_t lt_counters;
233 /* Offset of the first counter entry, used to free the counters */
234 uint16_t lt_counter_offset;
235 zone_t lt_counter_zone;
236 char lt_counter_zone_name[32];
237 #endif
238 };
239
240 static inline uint16_t
ledger_template_entries_lut_size(uint16_t lt_table_size)241 ledger_template_entries_lut_size(uint16_t lt_table_size)
242 {
243 /*
244 * The lookup table needs to be big enough to store lt_table_size entries of the largest
245 * entry size (struct ledger_entry) given a stride of the smallest entry size (struct ledger_entry_small)
246 */
247 if (os_mul_overflow(lt_table_size, (sizeof(struct ledger_entry) / sizeof(struct ledger_entry_small)), <_table_size)) {
248 /*
249 * This means MAX_LEDGER_ENTRIES is misconfigured or
250 * someone has accidently passed in an lt_table_size that is > MAX_LEDGER_ENTRIES
251 */
252 panic("Attempt to create a lookup table for a ledger template with too many entries. lt_table_size=%u, MAX_LEDGER_ENTRIES=%lu\n", lt_table_size, MAX_LEDGER_ENTRIES);
253 }
254 return lt_table_size;
255 }
256
257 #define template_lock(template) lck_mtx_lock(&(template)->lt_lock)
258 #define template_unlock(template) lck_mtx_unlock(&(template)->lt_lock)
259
260 #define TEMPLATE_INUSE(s, t) { \
261 s = splsched(); \
262 while (OSCompareAndSwap(0, 1, &((t)->lt_inuse))) \
263 ; \
264 }
265
266 #define TEMPLATE_IDLE(s, t) { \
267 (t)->lt_inuse = 0; \
268 splx(s); \
269 }
270
271 static int ledger_cnt = 0;
272 /* ledger ast helper functions */
273 static uint32_t ledger_check_needblock(ledger_t l, uint64_t now);
274 static kern_return_t ledger_perform_blocking(ledger_t l);
275 static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
276 static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
277
278 static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
279 int entry);
280 #if DEBUG || DEVELOPMENT
281 static inline bool ledger_is_diag_threshold_enabled_internal(struct ledger_entry *le);
282 #endif
283 #if 0
284 static void
285 debug_callback(const void *p0, __unused const void *p1)
286 {
287 printf("ledger: resource exhausted [%s] for task %p\n",
288 (const char *)p0, p1);
289 }
290 #endif
291
292 /************************************/
293
294 static uint64_t
abstime_to_nsecs(uint64_t abstime)295 abstime_to_nsecs(uint64_t abstime)
296 {
297 uint64_t nsecs;
298
299 absolutetime_to_nanoseconds(abstime, &nsecs);
300 return nsecs;
301 }
302
303 static uint64_t
nsecs_to_abstime(uint64_t nsecs)304 nsecs_to_abstime(uint64_t nsecs)
305 {
306 uint64_t abstime;
307
308 nanoseconds_to_absolutetime(nsecs, &abstime);
309 return abstime;
310 }
311
312 static const uint16_t *
ledger_entry_to_template_idx(ledger_template_t template,int index)313 ledger_entry_to_template_idx(ledger_template_t template, int index)
314 {
315 uint16_t offset = ENTRY_ID_OFFSET(index);
316 if (offset / sizeof(struct ledger_entry_small) >= template->lt_cnt) {
317 return NULL;
318 }
319
320 return &template->lt_entries_lut[offset];
321 }
322
323 /*
324 * Convert the id to a ledger entry.
325 * It's the callers responsibility to ensure the id is valid and a full size
326 * ledger entry.
327 */
328 static struct ledger_entry *
ledger_entry_identifier_to_entry(ledger_t ledger,int id)329 ledger_entry_identifier_to_entry(ledger_t ledger, int id)
330 {
331 assert(is_entry_valid(ledger, id));
332 assert(ENTRY_ID_SIZE(id) == sizeof(struct ledger_entry));
333 return (struct ledger_entry *) &ledger->l_entries[ENTRY_ID_OFFSET(id)];
334 }
335
336
337 ledger_template_t
ledger_template_create(const char * name)338 ledger_template_create(const char *name)
339 {
340 ledger_template_t template;
341
342 template = kalloc_type(struct ledger_template, Z_WAITOK | Z_ZERO | Z_NOFAIL);
343 template->lt_name = name;
344 template->lt_refs = 1;
345 template->lt_table_size = 1;
346 lck_mtx_init(&template->lt_lock, &ledger_lck_grp, LCK_ATTR_NULL);
347
348 template->lt_entries = kalloc_type(struct entry_template,
349 template->lt_table_size, Z_WAITOK | Z_ZERO);
350 if (template->lt_entries == NULL) {
351 kfree_type(struct ledger_template, template);
352 return NULL;
353 }
354 template->lt_entries_lut = kalloc_type(uint16_t, ledger_template_entries_lut_size(template->lt_table_size),
355 Z_WAITOK | Z_ZERO);
356 if (template->lt_entries_lut == NULL) {
357 kfree_type(struct entry_template, template->lt_entries);
358 kfree_type(struct ledger_template, template);
359 template = NULL;
360 }
361
362 return template;
363 }
364
365 static void
ledger_template_create_counter_zone(ledger_template_t template)366 ledger_template_create_counter_zone(ledger_template_t template)
367 {
368 #if ATOMIC_COUNTER_USE_PERCPU
369 if (template->lt_counters) {
370 snprintf(
371 template->lt_counter_zone_name,
372 sizeof(template->lt_counter_zone_name),
373 "%s.c",
374 template->lt_name);
375
376 template->lt_counter_zone = zone_create(
377 template->lt_counter_zone_name,
378 sizeof(uint64_t) * template->lt_counters,
379 ZC_PERCPU | ZC_ALIGNMENT_REQUIRED | ZC_KASAN_NOREDZONE | ZC_DESTRUCTIBLE);
380 }
381 #else /* ATOMIC_COUNTER_USE_PERCPU */
382 (void) template;
383 #endif /* !ATOMIC_COUNTER_USE_PERCPU */
384 }
385
386 ledger_template_t
ledger_template_copy(ledger_template_t template,const char * name)387 ledger_template_copy(ledger_template_t template, const char *name)
388 {
389 struct entry_template * new_entries = NULL;
390 uint16_t *new_entries_lut = NULL;
391 size_t new_entries_lut_size = 0;
392 ledger_template_t new_template = ledger_template_create(name);
393
394 if (new_template == NULL) {
395 return new_template;
396 }
397
398 template_lock(template);
399 assert(template->lt_initialized);
400
401 new_entries = kalloc_type(struct entry_template, template->lt_table_size,
402 Z_WAITOK | Z_ZERO);
403
404 if (new_entries == NULL) {
405 /* Tear down the new template; we've failed. :( */
406 ledger_template_dereference(new_template);
407 new_template = NULL;
408 goto out;
409 }
410 new_entries_lut_size = ledger_template_entries_lut_size(template->lt_table_size);
411
412 new_entries_lut = kalloc_type(uint16_t, new_entries_lut_size,
413 Z_WAITOK | Z_ZERO);
414 if (new_entries_lut == NULL) {
415 /* Tear down the new template; we've failed. :( */
416 ledger_template_dereference(new_template);
417 new_template = NULL;
418 goto out;
419 }
420
421 /* Copy the template entries. */
422 bcopy(template->lt_entries, new_entries, sizeof(struct entry_template) * template->lt_table_size);
423 kfree_type(struct entry_template, new_template->lt_table_size, new_template->lt_entries);
424 /* Copy the look up table. */
425 bcopy(template->lt_entries_lut, new_entries_lut, sizeof(uint16_t) * new_entries_lut_size);
426 kfree_type(uint16_t, ledger_template_entries_lut_size(new_template->lt_table_size), new_template->lt_entries_lut);
427
428 new_template->lt_entries = new_entries;
429 new_template->lt_table_size = template->lt_table_size;
430 new_template->lt_cnt = template->lt_cnt;
431 new_template->lt_next_offset = template->lt_next_offset;
432 new_template->lt_entries_lut = new_entries_lut;
433 #if ATOMIC_COUNTER_USE_PERCPU
434 new_template->lt_counters = template->lt_counters;
435 new_template->lt_counter_offset = template->lt_counter_offset;
436 ledger_template_create_counter_zone(new_template);
437 #endif
438
439 out:
440 template_unlock(template);
441
442 return new_template;
443 }
444
445 void
ledger_template_dereference(ledger_template_t template)446 ledger_template_dereference(ledger_template_t template)
447 {
448 template_lock(template);
449 template->lt_refs--;
450 template_unlock(template);
451
452 if (template->lt_refs == 0) {
453 kfree_type(struct entry_template, template->lt_table_size, template->lt_entries);
454 kfree_type(uint16_t, ledger_template_entries_lut_size(template->lt_table_size), template->lt_entries_lut);
455 lck_mtx_destroy(&template->lt_lock, &ledger_lck_grp);
456 if (template->lt_zone) {
457 zdestroy(template->lt_zone);
458 }
459 #if ATOMIC_COUNTER_USE_PERCPU
460 if (template->lt_counter_zone) {
461 zdestroy(template->lt_counter_zone);
462 }
463 #endif
464 kfree_type(struct ledger_template, template);
465 }
466 }
467
468 static inline int
ledger_entry_id(uint16_t size,uint16_t offset)469 ledger_entry_id(uint16_t size, uint16_t offset)
470 {
471 int idx = offset;
472 idx |= (size << ENTRY_ID_SIZE_SHIFT);
473 assert(idx >= 0);
474 return idx;
475 }
476
477 static inline int
ledger_entry_id_from_template_entry(const struct entry_template * et)478 ledger_entry_id_from_template_entry(const struct entry_template *et)
479 {
480 return ledger_entry_id(et->et_size, et->et_offset);
481 }
482
483 int
ledger_entry_add_with_flags(ledger_template_t template,const char * key,const char * group,const char * units,uint64_t flags)484 ledger_entry_add_with_flags(ledger_template_t template, const char *key,
485 const char *group, const char *units, uint64_t flags)
486 {
487 uint16_t template_idx;
488 struct entry_template *et;
489 uint16_t size = 0, next_offset = 0, entry_idx = 0;
490
491 if ((key == NULL) || (strlen(key) >= LEDGER_NAME_MAX) || (template->lt_zone != NULL)) {
492 return -1;
493 }
494
495 template_lock(template);
496
497 /* Make sure we have space for this entry */
498 if (template->lt_cnt == MAX_LEDGER_ENTRIES) {
499 template_unlock(template);
500 return -1;
501 }
502
503 /* If the table is full, attempt to double its size */
504 if (template->lt_cnt == template->lt_table_size) {
505 struct entry_template *new_entries, *old_entries;
506 uint16_t *new_entries_lut = NULL, *old_entries_lut = NULL;
507 uint16_t old_cnt, new_cnt;
508 spl_t s;
509
510 old_cnt = template->lt_table_size;
511 /* double old_sz allocation, but check for overflow */
512 if (os_mul_overflow(old_cnt, 2, &new_cnt)) {
513 template_unlock(template);
514 return -1;
515 }
516
517 if (new_cnt > MAX_LEDGER_ENTRIES) {
518 template_unlock(template);
519 panic("Attempt to create a ledger template with more than MAX_LEDGER_ENTRIES. MAX_LEDGER_ENTRIES=%lu, old_cnt=%u, new_cnt=%u\n", MAX_LEDGER_ENTRIES, old_cnt, new_cnt);
520 }
521
522 new_entries = kalloc_type(struct entry_template, new_cnt,
523 Z_WAITOK | Z_ZERO);
524 if (new_entries == NULL) {
525 template_unlock(template);
526 return -1;
527 }
528 new_entries_lut = kalloc_type(uint16_t, ledger_template_entries_lut_size(new_cnt),
529 Z_WAITOK | Z_ZERO);
530 if (new_entries_lut == NULL) {
531 template_unlock(template);
532 kfree_type(struct entry_template, new_cnt, new_entries);
533 return -1;
534 }
535
536 memcpy(new_entries, template->lt_entries,
537 old_cnt * sizeof(struct entry_template));
538 template->lt_table_size = new_cnt;
539
540 memcpy(new_entries_lut, template->lt_entries_lut,
541 ledger_template_entries_lut_size(old_cnt) * sizeof(uint16_t));
542
543 old_entries = template->lt_entries;
544 old_entries_lut = template->lt_entries_lut;
545
546 TEMPLATE_INUSE(s, template);
547 template->lt_entries = new_entries;
548 template->lt_entries_lut = new_entries_lut;
549 TEMPLATE_IDLE(s, template);
550
551 kfree_type(struct entry_template, old_cnt, old_entries);
552 kfree_type(uint16_t, ledger_template_entries_lut_size(old_cnt), old_entries_lut);
553 }
554
555 et = &template->lt_entries[template->lt_cnt];
556 strlcpy(et->et_key, key, LEDGER_NAME_MAX);
557 strlcpy(et->et_group, group, LEDGER_NAME_MAX);
558 strlcpy(et->et_units, units, LEDGER_NAME_MAX);
559 et->et_flags = LF_ENTRY_ACTIVE;
560 /*
561 * Currently we have three types of ledger entries:
562 * - full-fledged ledger entries
563 * - smaller CREDIT_ONLY entries
564 * - smaller counter entries
565 */
566 if ((flags & LEDGER_ENTRY_USE_COUNTER) != 0) {
567 /* We cannot use any other flags with scalable counter. */
568 assert((flags & (~LEDGER_ENTRY_USE_COUNTER)) == 0);
569 size = sizeof(struct ledger_entry_counter);
570 et->et_flags |= LF_IS_COUNTER;
571 #if ATOMIC_COUNTER_USE_PERCPU
572 if (template->lt_counters == 0) {
573 template->lt_counter_offset = (template->lt_next_offset / sizeof(struct ledger_entry_small));
574 }
575 template->lt_counters++;
576 #endif
577 } else if ((flags & ~(LEDGER_ENTRY_SMALL_FLAGS)) == 0) {
578 size = sizeof(struct ledger_entry_small);
579 et->et_flags |= LF_TRACK_CREDIT_ONLY;
580 } else {
581 size = sizeof(struct ledger_entry);
582 }
583 et->et_size = size;
584 et->et_offset = (template->lt_next_offset / sizeof(struct ledger_entry_small));
585 et->et_callback = NULL;
586
587 template_idx = template->lt_cnt++;
588 next_offset = template->lt_next_offset;
589 entry_idx = next_offset / sizeof(struct ledger_entry_small);
590 template->lt_next_offset += size;
591 assert(template->lt_next_offset > next_offset);
592 template->lt_entries_lut[entry_idx] = template_idx;
593 template_unlock(template);
594
595 return ledger_entry_id(size, entry_idx);
596 }
597
598 /*
599 * Add a new entry to the list of entries in a ledger template. There is
600 * currently no mechanism to remove an entry. Implementing such a mechanism
601 * would require us to maintain per-entry reference counts, which we would
602 * prefer to avoid if possible.
603 */
604 int
ledger_entry_add(ledger_template_t template,const char * key,const char * group,const char * units)605 ledger_entry_add(ledger_template_t template, const char *key,
606 const char *group, const char *units)
607 {
608 /*
609 * When using the legacy interface we have to be pessimistic
610 * and allocate memory for all of the features.
611 */
612 return ledger_entry_add_with_flags(template, key, group, units,
613 LEDGER_ENTRY_ALLOW_CALLBACK | LEDGER_ENTRY_ALLOW_MAXIMUM |
614 LEDGER_ENTRY_ALLOW_DEBIT | LEDGER_ENTRY_ALLOW_LIMIT |
615 LEDGER_ENTRY_ALLOW_ACTION | LEDGER_ENTRY_ALLOW_INACTIVE);
616 }
617
618
619 kern_return_t
ledger_entry_setactive(ledger_t ledger,int entry)620 ledger_entry_setactive(ledger_t ledger, int entry)
621 {
622 volatile uint32_t *flags = NULL;
623
624 if (!is_entry_valid(ledger, entry)) {
625 return KERN_INVALID_ARGUMENT;
626 }
627
628 flags = get_entry_flags(ledger, entry);
629
630 if ((*flags & LF_ENTRY_ACTIVE) == 0) {
631 flag_set(flags, LF_ENTRY_ACTIVE);
632 }
633 return KERN_SUCCESS;
634 }
635
636
637 int
ledger_key_lookup(ledger_template_t template,const char * key)638 ledger_key_lookup(ledger_template_t template, const char *key)
639 {
640 int id = -1;
641 struct entry_template *et = NULL;
642
643 template_lock(template);
644 if (template->lt_entries != NULL) {
645 for (uint16_t idx = 0; idx < template->lt_cnt; idx++) {
646 et = &template->lt_entries[idx];
647 if (strcmp(key, et->et_key) == 0) {
648 id = ledger_entry_id(et->et_size, et->et_offset);
649 break;
650 }
651 }
652 }
653
654 template_unlock(template);
655
656 return id;
657 }
658
659 /*
660 * Complete the initialization of ledger template
661 * by initializing ledger zone. After initializing
662 * the ledger zone, adding an entry in the ledger
663 * template will fail.
664 */
665 void
ledger_template_complete(ledger_template_t template)666 ledger_template_complete(ledger_template_t template)
667 {
668 size_t ledger_size;
669 ledger_size = sizeof(struct ledger) + template->lt_next_offset;
670 assert(ledger_size > sizeof(struct ledger));
671 template->lt_zone = zone_create(template->lt_name, ledger_size,
672 ZC_DESTRUCTIBLE);
673 ledger_template_create_counter_zone(template);
674 template->lt_initialized = true;
675 }
676
677 /*
678 * Like ledger_template_complete, except we'll ask
679 * the pmap layer to manage allocations for us.
680 * Meant for ledgers that should be owned by the
681 * pmap layer.
682 */
683 void
ledger_template_complete_secure_alloc(ledger_template_t template)684 ledger_template_complete_secure_alloc(ledger_template_t template)
685 {
686 size_t ledger_size;
687 ledger_size = sizeof(struct ledger) + template->lt_next_offset;
688
689 /**
690 * Ensure that the amount of space being allocated by the PPL for each
691 * ledger is large enough.
692 */
693 pmap_ledger_verify_size(ledger_size);
694 ledger_template_create_counter_zone(template);
695 template->lt_initialized = true;
696 }
697
698 /*
699 * Create a new ledger based on the specified template. As part of the
700 * ledger creation we need to allocate space for a table of ledger entries.
701 * The size of the table is based on the size of the template at the time
702 * the ledger is created. If additional entries are added to the template
703 * after the ledger is created, they will not be tracked in this ledger.
704 */
705 ledger_t
ledger_instantiate(ledger_template_t template,int entry_type)706 ledger_instantiate(ledger_template_t template, int entry_type)
707 {
708 ledger_t ledger;
709 uint16_t entries_size;
710 uint16_t num_entries;
711 uint16_t i;
712 #if ATOMIC_COUNTER_USE_PERCPU
713 int counters_inited = 0;
714 counter_t counters;
715 #endif
716
717 template_lock(template);
718 template->lt_refs++;
719 entries_size = template->lt_next_offset;
720 num_entries = template->lt_cnt;
721 template_unlock(template);
722
723 if (template->lt_zone) {
724 ledger = (ledger_t)zalloc(template->lt_zone);
725 } else {
726 /**
727 * If the template doesn't contain a zone to allocate ledger objects
728 * from, then assume that these ledger objects should be allocated by
729 * the pmap. This is done on PPL-enabled systems to give the PPL a
730 * method of validating ledger objects when updating them from within
731 * the PPL.
732 */
733 ledger = pmap_ledger_alloc();
734 }
735
736 if (ledger == NULL) {
737 ledger_template_dereference(template);
738 return LEDGER_NULL;
739 }
740
741 #if ATOMIC_COUNTER_USE_PERCPU
742 if (template->lt_counter_zone) {
743 counters = zalloc_percpu(template->lt_counter_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL);
744 }
745 #endif
746
747 ledger->l_template = template;
748 ledger->l_id = ledger_cnt++;
749 os_ref_init(&ledger->l_refs, &ledger_refgrp);
750 assert(entries_size > 0);
751 ledger->l_size = (uint16_t) entries_size;
752
753 template_lock(template);
754 assert(ledger->l_size <= template->lt_next_offset);
755 for (i = 0; i < num_entries; i++) {
756 uint16_t size, offset;
757 struct entry_template *et = &template->lt_entries[i];
758 size = et->et_size;
759 offset = et->et_offset;
760 assert(offset < ledger->l_size);
761
762 struct ledger_entry_small *les = &ledger->l_entries[offset];
763 if (size == sizeof(struct ledger_entry)) {
764 struct ledger_entry *le = (struct ledger_entry *) les;
765
766 le->le_flags = et->et_flags;
767 /* make entry inactive by removing active bit */
768 if (entry_type == LEDGER_CREATE_INACTIVE_ENTRIES) {
769 flag_clear(&le->le_flags, LF_ENTRY_ACTIVE);
770 }
771 /*
772 * If template has a callback, this entry is opted-in,
773 * by default.
774 */
775 if (et->et_callback != NULL) {
776 flag_set(&le->le_flags, LEDGER_ACTION_CALLBACK);
777 }
778 le->le_credit = 0;
779 le->le_debit = 0;
780 le->le_limit = LEDGER_LIMIT_INFINITY;
781 le->le_warn_percent = LEDGER_PERCENT_NONE;
782 le->le_diag_threshold_scaled = LEDGER_DIAG_MEM_THRESHOLD_INFINITY;
783 le->_le.le_refill.le_refill_period = 0;
784 le->_le.le_refill.le_last_refill = 0;
785 } else if (et->et_flags & LF_IS_COUNTER) {
786 struct ledger_entry_counter *lec = (struct ledger_entry_counter *) les;
787 lec->lec_flags = et->et_flags;
788 #if ATOMIC_COUNTER_USE_PERCPU
789 assert(template->lt_counter_zone != NULL);
790 assert(counters_inited < template->lt_counters);
791 lec->lec_counter = &counters[counters_inited];
792 counters_inited++;
793 #else /* ATOMIC_COUNTER_USE_PERCPU */
794 /*
795 * When we're using regular (non-percpu) atomic counters,
796 * this is just a wide store.
797 */
798 counter_alloc(&lec->lec_counter);
799 #endif /* !ATOMIC_COUNTER_USE_PERCPU */
800 } else {
801 les->les_flags = et->et_flags;
802 les->les_credit = 0;
803 }
804 }
805 template_unlock(template);
806
807 return ledger;
808 }
809
810 static uint32_t
flag_set(volatile uint32_t * flags,uint32_t bit)811 flag_set(volatile uint32_t *flags, uint32_t bit)
812 {
813 return OSBitOrAtomic(bit, flags);
814 }
815
816 static uint32_t
flag_clear(volatile uint32_t * flags,uint32_t bit)817 flag_clear(volatile uint32_t *flags, uint32_t bit)
818 {
819 return OSBitAndAtomic(~bit, flags);
820 }
821
822 /*
823 * Take a reference on a ledger
824 */
825 void
ledger_reference(ledger_t ledger)826 ledger_reference(ledger_t ledger)
827 {
828 if (!LEDGER_VALID(ledger)) {
829 return;
830 }
831
832 os_ref_retain(&ledger->l_refs);
833 }
834
835 #if ATOMIC_COUNTER_USE_PERCPU
836 static void
ledger_free_counters(ledger_t ledger)837 ledger_free_counters(ledger_t ledger)
838 {
839 struct ledger_entry_counter *lec;
840 ledger_template_t template = ledger->l_template;
841
842 if (!template->lt_counter_zone) {
843 /* Nothing to do */
844 assert(!template->lt_counters);
845 return;
846 }
847
848 /* We hold the index of the first counter entry which has the pointer to the allocation */
849 lec = (struct ledger_entry_counter *) &ledger->l_entries[template->lt_counter_offset];
850 assert(lec->lec_flags & LF_IS_COUNTER);
851 zfree_percpu(template->lt_counter_zone, lec->lec_counter);
852 }
853 #endif /* ATOMIC_COUNTER_USE_PERCPU */
854
855 /*
856 * Remove a reference on a ledger. If this is the last reference,
857 * deallocate the unused ledger.
858 */
859 void
ledger_dereference(ledger_t ledger)860 ledger_dereference(ledger_t ledger)
861 {
862 if (!LEDGER_VALID(ledger)) {
863 return;
864 }
865
866 if (os_ref_release(&ledger->l_refs) == 0) {
867 ledger_template_t template = ledger->l_template;
868
869 #if ATOMIC_COUNTER_USE_PERCPU
870 ledger_free_counters(ledger);
871 #endif /* ATOMIC_COUNTER_USE_PERCPU */
872
873 if (template->lt_zone) {
874 zfree(template->lt_zone, ledger);
875 } else {
876 /**
877 * If the template doesn't contain a zone to allocate ledger objects
878 * from, then assume that these ledger objects were allocated by the
879 * pmap. This is done on PPL-enabled systems to give the PPL a
880 * method of validating ledger objects when updating them from
881 * within the PPL.
882 */
883 pmap_ledger_free(ledger);
884 }
885 ledger_template_dereference(template);
886 }
887 }
888
889 /*
890 * Determine whether an entry has exceeded its warning level.
891 */
892 static inline bool
warn_level_exceeded(struct ledger_entry * le)893 warn_level_exceeded(struct ledger_entry *le)
894 {
895 ledger_amount_t balance;
896
897 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
898 assert(le->le_debit == 0);
899 } else {
900 assert((le->le_credit >= 0) && (le->le_debit >= 0));
901 }
902
903 /*
904 * XXX - Currently, we only support warnings for ledgers which
905 * use positive limits.
906 */
907 balance = le->le_credit - le->le_debit;
908 if (le->le_warn_percent != LEDGER_PERCENT_NONE &&
909 ((balance > (le->le_limit * le->le_warn_percent) >> 16))) {
910 return true;
911 }
912 return false;
913 }
914 #if DEBUG || DEVELOPMENT
915
916 /*
917 * Determine whether an entry has exceeded its diag mem threshold level.
918 */
919 static inline bool
diag_mem_threshold_exceeded(struct ledger_entry * le)920 diag_mem_threshold_exceeded(struct ledger_entry *le)
921 {
922 ledger_amount_t balance;
923 ledger_amount_t diag_mem_threshold;
924
925 if ((le->le_diag_threshold_scaled != LEDGER_DIAG_MEM_THRESHOLD_INFINITY) && (ledger_is_diag_threshold_enabled_internal(le) == true)) {
926 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
927 assert(le->le_debit == 0);
928 } else {
929 assert((le->le_credit >= 0) && (le->le_debit >= 0));
930 }
931
932 diag_mem_threshold = LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(le->le_diag_threshold_scaled);
933 balance = le->le_credit - le->le_debit;
934 if ((diag_mem_threshold <= 0) && (balance < diag_mem_threshold)) {
935 return 1;
936 }
937 if ((diag_mem_threshold > 0) && (balance > diag_mem_threshold)) {
938 return 1;
939 }
940 }
941 return 0;
942 }
943 #endif
944 /*
945 * Determine whether an entry has exceeded its limit.
946 */
947 static inline bool
limit_exceeded(struct ledger_entry * le)948 limit_exceeded(struct ledger_entry *le)
949 {
950 ledger_amount_t balance;
951
952 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
953 assert(le->le_debit == 0);
954 } else {
955 assert((le->le_credit >= 0) && (le->le_debit >= 0));
956 }
957
958 balance = le->le_credit - le->le_debit;
959 if ((le->le_limit <= 0) && (balance < le->le_limit)) {
960 return true;
961 }
962
963 if ((le->le_limit > 0) && (balance > le->le_limit)) {
964 return true;
965 }
966 return false;
967 }
968
969 static inline struct ledger_callback *
entry_get_callback(ledger_t ledger,int entry)970 entry_get_callback(ledger_t ledger, int entry)
971 {
972 struct ledger_callback *callback = NULL;
973 spl_t s;
974 const uint16_t *ledger_template_idx_p = NULL;
975
976 TEMPLATE_INUSE(s, ledger->l_template);
977 ledger_template_idx_p = ledger_entry_to_template_idx(ledger->l_template, entry);
978 if (ledger_template_idx_p != NULL) {
979 callback = ledger->l_template->lt_entries[*ledger_template_idx_p].et_callback;
980 }
981 TEMPLATE_IDLE(s, ledger->l_template);
982
983 return callback;
984 }
985
986 /*
987 * If the ledger value is positive, wake up anybody waiting on it.
988 */
989 static inline void
ledger_limit_entry_wakeup(struct ledger_entry * le)990 ledger_limit_entry_wakeup(struct ledger_entry *le)
991 {
992 if (!limit_exceeded(le)) {
993 while (le->le_flags & LF_WAKE_NEEDED) {
994 flag_clear(&le->le_flags, LF_WAKE_NEEDED);
995 thread_wakeup((event_t)le);
996 }
997 }
998 }
999
1000 /*
1001 * Refill the coffers.
1002 */
1003 static void
ledger_refill(uint64_t now,ledger_t ledger,int entry)1004 ledger_refill(uint64_t now, ledger_t ledger, int entry)
1005 {
1006 uint64_t elapsed, period, periods;
1007 struct ledger_entry *le;
1008 ledger_amount_t balance, due;
1009
1010 if (!is_entry_valid(ledger, entry)) {
1011 return;
1012 }
1013
1014 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1015 /* Small & counter entries can't do refills */
1016 return;
1017 }
1018
1019 le = ledger_entry_identifier_to_entry(ledger, entry);
1020
1021 assert(le->le_limit != LEDGER_LIMIT_INFINITY);
1022
1023 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
1024 assert(le->le_debit == 0);
1025 return;
1026 }
1027
1028 /*
1029 * If another thread is handling the refill already, we're not
1030 * needed.
1031 */
1032 if (flag_set(&le->le_flags, LF_REFILL_INPROGRESS) & LF_REFILL_INPROGRESS) {
1033 return;
1034 }
1035
1036 /*
1037 * If the timestamp we're about to use to refill is older than the
1038 * last refill, then someone else has already refilled this ledger
1039 * and there's nothing for us to do here.
1040 */
1041 if (now <= le->_le.le_refill.le_last_refill) {
1042 flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
1043 return;
1044 }
1045
1046 /*
1047 * See how many refill periods have passed since we last
1048 * did a refill.
1049 */
1050 period = le->_le.le_refill.le_refill_period;
1051 elapsed = now - le->_le.le_refill.le_last_refill;
1052 if ((period == 0) || (elapsed < period)) {
1053 flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
1054 return;
1055 }
1056
1057 /*
1058 * Optimize for the most common case of only one or two
1059 * periods elapsing.
1060 */
1061 periods = 0;
1062 while ((periods < 2) && (elapsed > 0)) {
1063 periods++;
1064 elapsed -= period;
1065 }
1066
1067 /*
1068 * OK, it's been a long time. Do a divide to figure out
1069 * how long.
1070 */
1071 if (elapsed > 0) {
1072 periods = (now - le->_le.le_refill.le_last_refill) / period;
1073 }
1074
1075 balance = le->le_credit - le->le_debit;
1076 due = periods * le->le_limit;
1077
1078 if (balance - due < 0) {
1079 due = balance;
1080 }
1081
1082 if (due < 0 && (le->le_flags & LF_PANIC_ON_NEGATIVE)) {
1083 assertf(due >= 0, "now=%llu, ledger=%p, entry=%d, balance=%lld, due=%lld", now, ledger, entry, balance, due);
1084 } else {
1085 OSAddAtomic64(due, &le->le_debit);
1086 assert(le->le_debit >= 0);
1087 }
1088 /*
1089 * If we've completely refilled the pool, set the refill time to now.
1090 * Otherwise set it to the time at which it last should have been
1091 * fully refilled.
1092 */
1093 if (balance == due) {
1094 le->_le.le_refill.le_last_refill = now;
1095 } else {
1096 le->_le.le_refill.le_last_refill += (le->_le.le_refill.le_refill_period * periods);
1097 }
1098
1099 flag_clear(&le->le_flags, LF_REFILL_INPROGRESS);
1100
1101 lprintf(("Refill %lld %lld->%lld\n", periods, balance, balance - due));
1102 if (!limit_exceeded(le)) {
1103 flag_clear(&le->le_flags, LF_CALLED_BACK);
1104 ledger_limit_entry_wakeup(le);
1105 }
1106 }
1107
1108 void
ledger_entry_check_new_balance(thread_t thread,ledger_t ledger,int entry)1109 ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
1110 int entry)
1111 {
1112 uint16_t size, offset;
1113 struct ledger_entry *le = NULL;
1114 struct ledger_entry_small *les = NULL;
1115 if (!is_entry_valid(ledger, entry)) {
1116 return;
1117 }
1118 size = ENTRY_ID_SIZE(entry);
1119 offset = ENTRY_ID_OFFSET(entry);
1120 les = &ledger->l_entries[offset];
1121 if (size == sizeof(struct ledger_entry_small)) {
1122 if (les->les_flags & LF_IS_COUNTER) {
1123 return; /* Nothing to do with a counter */
1124 }
1125 if ((les->les_flags & LF_PANIC_ON_NEGATIVE) && les->les_credit < 0) {
1126 panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:0 balance:%lld",
1127 ledger, entry, les,
1128 les->les_credit,
1129 les->les_credit);
1130 }
1131 } else if (size == sizeof(struct ledger_entry)) {
1132 le = (struct ledger_entry *)les;
1133 if (le->le_flags & LF_TRACKING_MAX) {
1134 ledger_amount_t balance = le->le_credit - le->le_debit;
1135
1136 if (balance > le->_le._le_max.le_lifetime_max) {
1137 le->_le._le_max.le_lifetime_max = balance;
1138 }
1139
1140 #if CONFIG_LEDGER_INTERVAL_MAX
1141 if (balance > le->_le._le_max.le_interval_max) {
1142 le->_le._le_max.le_interval_max = balance;
1143 }
1144 #endif /* LEDGER_CONFIG_INTERVAL_MAX */
1145 }
1146
1147 /* Check to see whether we're due a refill */
1148 if (le->le_flags & LF_REFILL_SCHEDULED) {
1149 assert(!(le->le_flags & LF_TRACKING_MAX));
1150
1151 uint64_t now = mach_absolute_time();
1152 if ((now - le->_le.le_refill.le_last_refill) > le->_le.le_refill.le_refill_period) {
1153 ledger_refill(now, ledger, entry);
1154 }
1155 }
1156
1157 if (limit_exceeded(le)) {
1158 /*
1159 * We've exceeded the limit for this entry. There
1160 * are several possible ways to handle it. We can block,
1161 * we can execute a callback, or we can ignore it. In
1162 * either of the first two cases, we want to set the AST
1163 * flag so we can take the appropriate action just before
1164 * leaving the kernel. The one caveat is that if we have
1165 * already called the callback, we don't want to do it
1166 * again until it gets rearmed.
1167 */
1168 if ((le->le_flags & LEDGER_ACTION_BLOCK) ||
1169 (!(le->le_flags & LF_CALLED_BACK) &&
1170 entry_get_callback(ledger, entry))) {
1171 act_set_astledger_async(thread);
1172 }
1173 } else {
1174 flag_clear(&le->le_flags, LF_CALLED_BACK);
1175
1176 /*
1177 * The balance on the account is below the limit.
1178 *
1179 * If there are any threads blocked on this entry, now would
1180 * be a good time to wake them up.
1181 */
1182 if (le->le_flags & LF_WAKE_NEEDED) {
1183 ledger_limit_entry_wakeup(le);
1184 }
1185
1186 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
1187 if (warn_level_exceeded(le)) {
1188 /*
1189 * This ledger's balance is above the warning level.
1190 */
1191 if ((le->le_flags & LF_WARNED) == 0) {
1192 /*
1193 * If we are above the warning level and
1194 * have not yet invoked the callback,
1195 * set the AST so it can be done before returning
1196 * to userland.
1197 */
1198 act_set_astledger_async(thread);
1199 }
1200 } else {
1201 /*
1202 * This ledger's balance is below the warning level.
1203 */
1204 if (le->le_flags & LF_WARNED) {
1205 /*
1206 * If we are below the warning level and
1207 * the LF_WARNED flag is still set, we need
1208 * to invoke the callback to let the client
1209 * know the ledger balance is now back below
1210 * the warning level.
1211 */
1212 act_set_astledger_async(thread);
1213 }
1214 }
1215 }
1216 }
1217 #if DEBUG || DEVELOPMENT
1218 if (diag_mem_threshold_exceeded(le)) {
1219 /*
1220 * Even if the limit is below the threshold, we may be interested
1221 * in diagnostics limits. Lets process them if the ast is not
1222 * invoked
1223 */
1224 if ((le->le_flags & LF_DIAG_WARNED) == 0) {
1225 act_set_astledger_async(thread);
1226 }
1227 }
1228 #endif
1229 if ((le->le_flags & LF_PANIC_ON_NEGATIVE) &&
1230 (le->le_credit < le->le_debit)) {
1231 panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:%lld balance:%lld",
1232 ledger, entry, le,
1233 le->le_credit,
1234 le->le_debit,
1235 le->le_credit - le->le_debit);
1236 }
1237 } else {
1238 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, size);
1239 }
1240 }
1241
1242 void
ledger_check_new_balance(thread_t thread,ledger_t ledger,int entry)1243 ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry)
1244 {
1245 ledger_entry_check_new_balance(thread, ledger, entry);
1246 }
1247
1248 /*
1249 * Add value to an entry in a ledger for a specific thread.
1250 */
1251 kern_return_t
ledger_credit_thread(thread_t thread,ledger_t ledger,int entry,ledger_amount_t amount)1252 ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
1253 {
1254 ledger_amount_t old, new;
1255 struct ledger_entry *le;
1256 uint16_t entry_size = ENTRY_ID_SIZE(entry);
1257
1258 if (!is_entry_valid_and_active(ledger, entry) || (amount < 0)) {
1259 return KERN_INVALID_VALUE;
1260 }
1261
1262 if (amount == 0) {
1263 return KERN_SUCCESS;
1264 }
1265
1266 if (entry_size == sizeof(struct ledger_entry_small)) {
1267 struct ledger_entry_small *les = &ledger->l_entries[ENTRY_ID_OFFSET(entry)];
1268 if (les->les_flags & LF_IS_COUNTER) {
1269 struct ledger_entry_counter *lec = (struct ledger_entry_counter *) les;
1270 counter_add(&lec->lec_counter, amount);
1271 return KERN_SUCCESS;
1272 } else {
1273 old = OSAddAtomic64(amount, &les->les_credit);
1274 new = old + amount;
1275 }
1276 } else if (entry_size == sizeof(struct ledger_entry)) {
1277 le = ledger_entry_identifier_to_entry(ledger, entry);
1278
1279 old = OSAddAtomic64(amount, &le->le_credit);
1280 new = old + amount;
1281 } else {
1282 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
1283 }
1284
1285 lprintf(("%p Credit %lld->%lld\n", thread, old, new));
1286 if (thread) {
1287 ledger_entry_check_new_balance(thread, ledger, entry);
1288 }
1289
1290 return KERN_SUCCESS;
1291 }
1292
1293 /*
1294 * Add value to an entry in a ledger.
1295 */
1296 kern_return_t
ledger_credit(ledger_t ledger,int entry,ledger_amount_t amount)1297 ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
1298 {
1299 return ledger_credit_thread(current_thread(), ledger, entry, amount);
1300 }
1301
1302 /*
1303 * Add value to an entry in a ledger; do not check balance after update.
1304 */
1305 kern_return_t
ledger_credit_nocheck(ledger_t ledger,int entry,ledger_amount_t amount)1306 ledger_credit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
1307 {
1308 return ledger_credit_thread(NULL, ledger, entry, amount);
1309 }
1310
1311 /* Add all of one ledger's values into another.
1312 * They must have been created from the same template.
1313 * This is not done atomically. Another thread (if not otherwise synchronized)
1314 * may see bogus values when comparing one entry to another.
1315 * As each entry's credit & debit are modified one at a time, the warning/limit
1316 * may spuriously trip, or spuriously fail to trip, or another thread (if not
1317 * otherwise synchronized) may see a bogus balance.
1318 */
1319 kern_return_t
ledger_rollup(ledger_t to_ledger,ledger_t from_ledger)1320 ledger_rollup(ledger_t to_ledger, ledger_t from_ledger)
1321 {
1322 int id;
1323 ledger_template_t template = NULL;
1324 struct entry_template *et = NULL;
1325
1326 assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt);
1327 template = from_ledger->l_template;
1328 assert(template->lt_initialized);
1329
1330 for (uint16_t i = 0; i < template->lt_cnt; i++) {
1331 et = &template->lt_entries[i];
1332 uint16_t size = et->et_size;
1333 id = ledger_entry_id(size, et->et_offset);
1334 ledger_rollup_entry(to_ledger, from_ledger, id);
1335 }
1336
1337 return KERN_SUCCESS;
1338 }
1339
1340 /* Add one ledger entry value to another.
1341 * They must have been created from the same template.
1342 * Since the credit and debit values are added one
1343 * at a time, other thread might read the a bogus value.
1344 */
1345 kern_return_t
ledger_rollup_entry(ledger_t to_ledger,ledger_t from_ledger,int entry)1346 ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry)
1347 {
1348 struct ledger_entry_small *from_les, *to_les;
1349 uint16_t entry_size, entry_offset;
1350 entry_size = ENTRY_ID_SIZE(entry);
1351 entry_offset = ENTRY_ID_OFFSET(entry);
1352
1353 assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt);
1354 if (is_entry_valid(from_ledger, entry) && is_entry_valid(to_ledger, entry)) {
1355 from_les = &from_ledger->l_entries[entry_offset];
1356 to_les = &to_ledger->l_entries[entry_offset];
1357 if (entry_size == sizeof(struct ledger_entry)) {
1358 struct ledger_entry *from = (struct ledger_entry *)from_les;
1359 struct ledger_entry *to = (struct ledger_entry *)to_les;
1360 OSAddAtomic64(from->le_credit, &to->le_credit);
1361 OSAddAtomic64(from->le_debit, &to->le_debit);
1362 } else if (entry_size == sizeof(struct ledger_entry_small)) {
1363 if (from_les->les_flags & LF_IS_COUNTER) {
1364 struct ledger_entry_counter *from_lec = (struct ledger_entry_counter *) from_les;
1365 struct ledger_entry_counter *to_lec = (struct ledger_entry_counter *) to_les;
1366 uint64_t from_val = counter_load(&from_lec->lec_counter);
1367 counter_add(&to_lec->lec_counter, from_val);
1368 } else {
1369 OSAddAtomic64(from_les->les_credit, &to_les->les_credit);
1370 }
1371 } else {
1372 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", from_ledger, entry, entry_size);
1373 }
1374 }
1375
1376 return KERN_SUCCESS;
1377 }
1378
1379 /*
1380 * Zero the balance of a ledger by adding to its credit or debit, whichever is smaller.
1381 * Note that some clients of ledgers (notably, task wakeup statistics) require that
1382 * le_credit only ever increase as a function of ledger_credit().
1383 */
1384 kern_return_t
ledger_zero_balance(ledger_t ledger,int entry)1385 ledger_zero_balance(ledger_t ledger, int entry)
1386 {
1387 struct ledger_entry *le;
1388 struct ledger_entry_small *les;
1389 ledger_amount_t debit, credit;
1390 uint16_t entry_size, entry_offset;
1391 entry_size = ENTRY_ID_SIZE(entry);
1392 entry_offset = ENTRY_ID_OFFSET(entry);
1393
1394 if (!is_entry_valid_and_active(ledger, entry)) {
1395 return KERN_INVALID_VALUE;
1396 }
1397
1398 les = &ledger->l_entries[entry_offset];
1399 if (entry_size == sizeof(struct ledger_entry_small)) {
1400 if (les->les_flags & LF_IS_COUNTER) {
1401 return KERN_INVALID_ARGUMENT;
1402 }
1403 while (true) {
1404 credit = les->les_credit;
1405 if (OSCompareAndSwap64(credit, 0, &les->les_credit)) {
1406 break;
1407 }
1408 }
1409 } else if (entry_size == sizeof(struct ledger_entry)) {
1410 le = (struct ledger_entry *)les;
1411 top:
1412 debit = le->le_debit;
1413 credit = le->le_credit;
1414
1415 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
1416 assert(le->le_debit == 0);
1417 if (!OSCompareAndSwap64(credit, 0, &le->le_credit)) {
1418 goto top;
1419 }
1420 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, 0));
1421 } else if (credit > debit) {
1422 if (!OSCompareAndSwap64(debit, credit, &le->le_debit)) {
1423 goto top;
1424 }
1425 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));
1426 } else if (credit < debit) {
1427 if (!OSCompareAndSwap64(credit, debit, &le->le_credit)) {
1428 goto top;
1429 }
1430 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, le->le_debit));
1431 }
1432 } else {
1433 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
1434 }
1435
1436 return KERN_SUCCESS;
1437 }
1438
1439 kern_return_t
ledger_get_limit(ledger_t ledger,int entry,ledger_amount_t * limit)1440 ledger_get_limit(ledger_t ledger, int entry, ledger_amount_t *limit)
1441 {
1442 struct ledger_entry *le;
1443
1444 if (!is_entry_valid_and_active(ledger, entry)) {
1445 return KERN_INVALID_VALUE;
1446 }
1447
1448 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1449 /* Small entries can't have limits */
1450 *limit = LEDGER_LIMIT_INFINITY;
1451 } else {
1452 le = ledger_entry_identifier_to_entry(ledger, entry);
1453 *limit = le->le_limit;
1454 }
1455
1456 lprintf(("ledger_get_limit: %lld\n", *limit));
1457
1458 return KERN_SUCCESS;
1459 }
1460
1461 /*
1462 * Adjust the limit of a limited resource. This does not affect the
1463 * current balance, so the change doesn't affect the thread until the
1464 * next refill.
1465 *
1466 * warn_level: If non-zero, causes the callback to be invoked when
1467 * the balance exceeds this level. Specified as a percentage [of the limit].
1468 */
1469 kern_return_t
ledger_set_limit(ledger_t ledger,int entry,ledger_amount_t limit,uint8_t warn_level_percentage)1470 ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit,
1471 uint8_t warn_level_percentage)
1472 {
1473 struct ledger_entry *le;
1474
1475 if (!is_entry_valid_and_active(ledger, entry)) {
1476 return KERN_INVALID_VALUE;
1477 }
1478
1479 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1480 /* Small entries can't have limits */
1481 return KERN_INVALID_ARGUMENT;
1482 }
1483
1484 lprintf(("ledger_set_limit: %lld\n", limit));
1485 le = ledger_entry_identifier_to_entry(ledger, entry);
1486
1487 if (limit == LEDGER_LIMIT_INFINITY) {
1488 /*
1489 * Caller wishes to disable the limit. This will implicitly
1490 * disable automatic refill, as refills implicitly depend
1491 * on the limit.
1492 */
1493 ledger_disable_refill(ledger, entry);
1494 }
1495
1496 le->le_limit = limit;
1497 if (le->le_flags & LF_REFILL_SCHEDULED) {
1498 assert(!(le->le_flags & LF_TRACKING_MAX));
1499 le->_le.le_refill.le_last_refill = 0;
1500 }
1501 flag_clear(&le->le_flags, LF_CALLED_BACK);
1502 flag_clear(&le->le_flags, LF_WARNED);
1503 ledger_limit_entry_wakeup(le);
1504
1505 if (warn_level_percentage != 0) {
1506 assert(warn_level_percentage <= 100);
1507 assert(limit > 0); /* no negative limit support for warnings */
1508 assert(limit != LEDGER_LIMIT_INFINITY); /* warn % without limit makes no sense */
1509 le->le_warn_percent = warn_level_percentage * (1u << 16) / 100;
1510 } else {
1511 le->le_warn_percent = LEDGER_PERCENT_NONE;
1512 }
1513
1514 return KERN_SUCCESS;
1515 }
1516
1517 #if CONFIG_LEDGER_INTERVAL_MAX
1518 kern_return_t
ledger_get_interval_max(ledger_t ledger,int entry,ledger_amount_t * max_interval_balance,int reset)1519 ledger_get_interval_max(ledger_t ledger, int entry,
1520 ledger_amount_t *max_interval_balance, int reset)
1521 {
1522 kern_return_t kr = KERN_SUCCESS;
1523 struct ledger_entry *le;
1524
1525 if (!is_entry_valid_and_active(ledger, entry)) {
1526 return KERN_INVALID_VALUE;
1527 }
1528
1529 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1530 /* Small entries can't track max */
1531 return KERN_INVALID_ARGUMENT;
1532 }
1533
1534 le = ledger_entry_identifier_to_entry(ledger, entry);
1535
1536 if (!(le->le_flags & LF_TRACKING_MAX)) {
1537 return KERN_INVALID_VALUE;
1538 }
1539
1540 *max_interval_balance = le->_le._le_max.le_interval_max;
1541 lprintf(("ledger_get_interval_max: %lld%s\n", *max_interval_balance,
1542 (reset) ? " --> 0" : ""));
1543
1544 if (reset) {
1545 kr = ledger_get_balance(ledger, entry, &le->_le._le_max.le_interval_max);
1546 }
1547
1548 return kr;
1549 }
1550 #endif /* CONFIG_LEDGER_INTERVAL_MAX */
1551
1552 kern_return_t
ledger_get_lifetime_max(ledger_t ledger,int entry,ledger_amount_t * max_lifetime_balance)1553 ledger_get_lifetime_max(ledger_t ledger, int entry,
1554 ledger_amount_t *max_lifetime_balance)
1555 {
1556 struct ledger_entry *le;
1557
1558 if (!is_entry_valid_and_active(ledger, entry)) {
1559 return KERN_INVALID_VALUE;
1560 }
1561
1562 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1563 /* Small entries can't track max */
1564 return KERN_INVALID_ARGUMENT;
1565 }
1566
1567 le = ledger_entry_identifier_to_entry(ledger, entry);
1568
1569 if (!(le->le_flags & LF_TRACKING_MAX)) {
1570 return KERN_INVALID_VALUE;
1571 }
1572
1573 *max_lifetime_balance = le->_le._le_max.le_lifetime_max;
1574 lprintf(("ledger_get_lifetime_max: %lld\n", *max_lifetime_balance));
1575
1576 return KERN_SUCCESS;
1577 }
1578
1579 /*
1580 * Enable tracking of periodic maximums for this ledger entry.
1581 */
1582 kern_return_t
ledger_track_maximum(ledger_template_t template,int entry,__unused int period_in_secs)1583 ledger_track_maximum(ledger_template_t template, int entry,
1584 __unused int period_in_secs)
1585 {
1586 uint16_t idx;
1587 const uint16_t *idx_p;
1588 struct entry_template *et = NULL;
1589 kern_return_t kr = KERN_INVALID_VALUE;
1590
1591 template_lock(template);
1592
1593 idx_p = ledger_entry_to_template_idx(template, entry);
1594 if (idx_p == NULL) {
1595 kr = KERN_INVALID_VALUE;
1596 goto out;
1597 }
1598 idx = *idx_p;
1599 if (idx >= template->lt_cnt) {
1600 kr = KERN_INVALID_VALUE;
1601 goto out;
1602 }
1603 et = &template->lt_entries[idx];
1604 /* Ensure the caller asked for enough space up front */
1605 if (et->et_size != sizeof(struct ledger_entry)) {
1606 kr = KERN_INVALID_VALUE;
1607 goto out;
1608 }
1609
1610 /* Refill is incompatible with max tracking. */
1611 if (et->et_flags & LF_REFILL_SCHEDULED) {
1612 kr = KERN_INVALID_VALUE;
1613 goto out;
1614 }
1615
1616 et->et_flags |= LF_TRACKING_MAX;
1617 kr = KERN_SUCCESS;
1618 out:
1619 template_unlock(template);
1620
1621 return kr;
1622 }
1623
1624 kern_return_t
ledger_panic_on_negative(ledger_template_t template,int entry)1625 ledger_panic_on_negative(ledger_template_t template, int entry)
1626 {
1627 const uint16_t *idx_p;
1628 uint16_t idx;
1629 template_lock(template);
1630
1631 idx_p = ledger_entry_to_template_idx(template, entry);
1632 if (idx_p == NULL) {
1633 template_unlock(template);
1634 return KERN_INVALID_VALUE;
1635 }
1636 idx = *idx_p;
1637 if (idx >= template->lt_cnt) {
1638 template_unlock(template);
1639 return KERN_INVALID_VALUE;
1640 }
1641
1642 if (template->lt_entries[idx].et_flags & LF_IS_COUNTER) {
1643 return KERN_INVALID_ARGUMENT;
1644 }
1645
1646 template->lt_entries[idx].et_flags |= LF_PANIC_ON_NEGATIVE;
1647
1648 template_unlock(template);
1649
1650 return KERN_SUCCESS;
1651 }
1652
1653 kern_return_t
ledger_track_credit_only(ledger_template_t template,int entry)1654 ledger_track_credit_only(ledger_template_t template, int entry)
1655 {
1656 const uint16_t *idx_p;
1657 uint16_t idx;
1658 struct entry_template *et = NULL;
1659 kern_return_t kr = KERN_INVALID_VALUE;
1660 template_lock(template);
1661
1662 idx_p = ledger_entry_to_template_idx(template, entry);
1663 if (idx_p == NULL) {
1664 kr = KERN_INVALID_VALUE;
1665 goto out;
1666 }
1667 idx = *idx_p;
1668 if (idx >= template->lt_cnt) {
1669 kr = KERN_INVALID_VALUE;
1670 goto out;
1671 }
1672 et = &template->lt_entries[idx];
1673 /* Ensure the caller asked for enough space up front */
1674 if (et->et_size != sizeof(struct ledger_entry)) {
1675 kr = KERN_INVALID_VALUE;
1676 goto out;
1677 }
1678
1679 et->et_flags |= LF_TRACK_CREDIT_ONLY;
1680 kr = KERN_SUCCESS;
1681
1682 out:
1683 template_unlock(template);
1684
1685 return kr;
1686 }
1687
1688 /*
1689 * Add a callback to be executed when the resource goes into deficit.
1690 */
1691 kern_return_t
ledger_set_callback(ledger_template_t template,int entry,ledger_callback_t func,const void * param0,const void * param1)1692 ledger_set_callback(ledger_template_t template, int entry,
1693 ledger_callback_t func, const void *param0, const void *param1)
1694 {
1695 struct entry_template *et;
1696 struct ledger_callback *old_cb, *new_cb;
1697 const uint16_t *idx_p;
1698 uint16_t idx;
1699
1700 idx_p = ledger_entry_to_template_idx(template, entry);
1701 if (idx_p == NULL) {
1702 return KERN_INVALID_VALUE;
1703 }
1704 idx = *idx_p;
1705
1706 if (idx >= template->lt_cnt) {
1707 return KERN_INVALID_VALUE;
1708 }
1709
1710 if (func) {
1711 new_cb = kalloc_type(struct ledger_callback, Z_WAITOK);
1712 new_cb->lc_func = func;
1713 new_cb->lc_param0 = param0;
1714 new_cb->lc_param1 = param1;
1715 } else {
1716 new_cb = NULL;
1717 }
1718
1719 template_lock(template);
1720 et = &template->lt_entries[idx];
1721 /* Ensure the caller asked for enough space up front */
1722 if (et->et_size != sizeof(struct ledger_entry)) {
1723 kfree_type(struct ledger_callback, new_cb);
1724 template_unlock(template);
1725 return KERN_INVALID_VALUE;
1726 }
1727 old_cb = et->et_callback;
1728 et->et_callback = new_cb;
1729 template_unlock(template);
1730 if (old_cb) {
1731 kfree_type(struct ledger_callback, old_cb);
1732 }
1733
1734 return KERN_SUCCESS;
1735 }
1736
1737 /*
1738 * Disable callback notification for a specific ledger entry.
1739 *
1740 * Otherwise, if using a ledger template which specified a
1741 * callback function (ledger_set_callback()), it will be invoked when
1742 * the resource goes into deficit.
1743 */
1744 kern_return_t
ledger_disable_callback(ledger_t ledger,int entry)1745 ledger_disable_callback(ledger_t ledger, int entry)
1746 {
1747 struct ledger_entry *le = NULL;
1748
1749 if (!is_entry_valid_and_active(ledger, entry)) {
1750 return KERN_INVALID_VALUE;
1751 }
1752
1753 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1754 /* Small entries can't have callbacks */
1755 return KERN_INVALID_ARGUMENT;
1756 }
1757
1758 le = ledger_entry_identifier_to_entry(ledger, entry);
1759
1760 /*
1761 * le_warn_percent is used to indicate *if* this ledger has a warning configured,
1762 * in addition to what that warning level is set to.
1763 * This means a side-effect of ledger_disable_callback() is that the
1764 * warning level is forgotten.
1765 */
1766 le->le_warn_percent = LEDGER_PERCENT_NONE;
1767 flag_clear(&le->le_flags, LEDGER_ACTION_CALLBACK);
1768 return KERN_SUCCESS;
1769 }
1770
1771 /*
1772 * Enable callback notification for a specific ledger entry.
1773 *
1774 * This is only needed if ledger_disable_callback() has previously
1775 * been invoked against an entry; there must already be a callback
1776 * configured.
1777 */
1778 kern_return_t
ledger_enable_callback(ledger_t ledger,int entry)1779 ledger_enable_callback(ledger_t ledger, int entry)
1780 {
1781 struct ledger_entry *le = NULL;
1782
1783 if (!is_entry_valid_and_active(ledger, entry)) {
1784 return KERN_INVALID_VALUE;
1785 }
1786
1787 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1788 /* Small entries can't have callbacks */
1789 return KERN_INVALID_ARGUMENT;
1790 }
1791
1792 le = ledger_entry_identifier_to_entry(ledger, entry);
1793
1794 assert(entry_get_callback(ledger, entry) != NULL);
1795
1796 flag_set(&le->le_flags, LEDGER_ACTION_CALLBACK);
1797 return KERN_SUCCESS;
1798 }
1799
1800 /*
1801 * Query the automatic refill period for this ledger entry.
1802 *
1803 * A period of 0 means this entry has none configured.
1804 */
1805 kern_return_t
ledger_get_period(ledger_t ledger,int entry,uint64_t * period)1806 ledger_get_period(ledger_t ledger, int entry, uint64_t *period)
1807 {
1808 struct ledger_entry *le;
1809
1810 if (!is_entry_valid_and_active(ledger, entry)) {
1811 return KERN_INVALID_VALUE;
1812 }
1813
1814 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1815 /* Small entries can't do refills */
1816 return KERN_INVALID_ARGUMENT;
1817 }
1818
1819 le = ledger_entry_identifier_to_entry(ledger, entry);
1820
1821 *period = abstime_to_nsecs(le->_le.le_refill.le_refill_period);
1822 lprintf(("ledger_get_period: %llx\n", *period));
1823 return KERN_SUCCESS;
1824 }
1825
1826 /*
1827 * Adjust the automatic refill period.
1828 */
1829 kern_return_t
ledger_set_period(ledger_t ledger,int entry,uint64_t period)1830 ledger_set_period(ledger_t ledger, int entry, uint64_t period)
1831 {
1832 struct ledger_entry *le = NULL;
1833
1834 if (!is_entry_valid_and_active(ledger, entry)) {
1835 return KERN_INVALID_VALUE;
1836 }
1837
1838 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1839 /* Small entries can't do refills */
1840 return KERN_INVALID_ARGUMENT;
1841 }
1842
1843 lprintf(("ledger_set_period: %llx\n", period));
1844
1845 le = ledger_entry_identifier_to_entry(ledger, entry);
1846
1847 /*
1848 * A refill period refills the ledger in multiples of the limit,
1849 * so if you haven't set one yet, you need a lesson on ledgers.
1850 */
1851 assert(le->le_limit != LEDGER_LIMIT_INFINITY);
1852
1853 if (le->le_flags & LF_TRACKING_MAX) {
1854 /*
1855 * Refill is incompatible with rolling max tracking.
1856 */
1857 return KERN_INVALID_VALUE;
1858 }
1859
1860 le->_le.le_refill.le_refill_period = nsecs_to_abstime(period);
1861
1862 /*
1863 * Set the 'starting time' for the next refill to now. Since
1864 * we're resetting the balance to zero here, we consider this
1865 * moment the starting time for accumulating a balance that
1866 * counts towards the limit.
1867 */
1868 le->_le.le_refill.le_last_refill = mach_absolute_time();
1869 ledger_zero_balance(ledger, entry);
1870
1871 flag_set(&le->le_flags, LF_REFILL_SCHEDULED);
1872
1873 return KERN_SUCCESS;
1874 }
1875
1876 /*
1877 * Disable automatic refill.
1878 */
1879 kern_return_t
ledger_disable_refill(ledger_t ledger,int entry)1880 ledger_disable_refill(ledger_t ledger, int entry)
1881 {
1882 struct ledger_entry *le = NULL;
1883
1884 if (!is_entry_valid_and_active(ledger, entry)) {
1885 return KERN_INVALID_VALUE;
1886 }
1887
1888 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1889 /* Small entries can't do refills */
1890 return KERN_INVALID_ARGUMENT;
1891 }
1892
1893 le = ledger_entry_identifier_to_entry(ledger, entry);
1894
1895 flag_clear(&le->le_flags, LF_REFILL_SCHEDULED);
1896
1897 return KERN_SUCCESS;
1898 }
1899
1900 kern_return_t
ledger_get_actions(ledger_t ledger,int entry,int * actions)1901 ledger_get_actions(ledger_t ledger, int entry, int *actions)
1902 {
1903 struct ledger_entry *le = NULL;
1904 *actions = 0;
1905
1906 if (!is_entry_valid_and_active(ledger, entry)) {
1907 return KERN_INVALID_VALUE;
1908 }
1909
1910 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1911 /* Small entries can't have actions */
1912 return KERN_INVALID_ARGUMENT;
1913 }
1914
1915 le = ledger_entry_identifier_to_entry(ledger, entry);
1916
1917 *actions = le->le_flags & LEDGER_ACTION_MASK;
1918 lprintf(("ledger_get_actions: %#x\n", *actions));
1919 return KERN_SUCCESS;
1920 }
1921
1922 kern_return_t
ledger_set_action(ledger_t ledger,int entry,int action)1923 ledger_set_action(ledger_t ledger, int entry, int action)
1924 {
1925 lprintf(("ledger_set_action: %#x\n", action));
1926 struct ledger_entry *le = NULL;
1927
1928 if (!is_entry_valid_and_active(ledger, entry)) {
1929 return KERN_INVALID_VALUE;
1930 }
1931
1932 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1933 /* Small entries can't have actions */
1934 return KERN_INVALID_ARGUMENT;
1935 }
1936
1937 le = ledger_entry_identifier_to_entry(ledger, entry);
1938
1939 flag_set(&le->le_flags, action);
1940 return KERN_SUCCESS;
1941 }
1942
1943 kern_return_t
ledger_debit_thread(thread_t thread,ledger_t ledger,int entry,ledger_amount_t amount)1944 ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
1945 {
1946 struct ledger_entry *le;
1947 ledger_amount_t old, new;
1948 uint16_t entry_size = ENTRY_ID_SIZE(entry);
1949
1950 if (!is_entry_valid_and_active(ledger, entry) || (amount < 0)) {
1951 return KERN_INVALID_ARGUMENT;
1952 }
1953
1954 if (amount == 0) {
1955 return KERN_SUCCESS;
1956 }
1957
1958 if (entry_size == sizeof(struct ledger_entry_small)) {
1959 struct ledger_entry_small *les = &ledger->l_entries[ENTRY_ID_OFFSET(entry)];
1960 if (les->les_flags & LF_IS_COUNTER) {
1961 struct ledger_entry_counter *lec = (struct ledger_entry_counter *) les;
1962 counter_add(&lec->lec_counter, -amount);
1963 return KERN_SUCCESS;
1964 } else {
1965 old = OSAddAtomic64(-amount, &les->les_credit);
1966 new = old - amount;
1967 }
1968 } else if (entry_size == sizeof(struct ledger_entry)) {
1969 le = ledger_entry_identifier_to_entry(ledger, entry);
1970
1971 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
1972 assert(le->le_debit == 0);
1973 old = OSAddAtomic64(-amount, &le->le_credit);
1974 new = old - amount;
1975 } else {
1976 old = OSAddAtomic64(amount, &le->le_debit);
1977 new = old + amount;
1978 }
1979 } else {
1980 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
1981 }
1982 lprintf(("%p Debit %lld->%lld\n", thread, old, new));
1983
1984 if (thread) {
1985 ledger_entry_check_new_balance(thread, ledger, entry);
1986 }
1987
1988 return KERN_SUCCESS;
1989 }
1990
1991 kern_return_t
ledger_debit(ledger_t ledger,int entry,ledger_amount_t amount)1992 ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
1993 {
1994 return ledger_debit_thread(current_thread(), ledger, entry, amount);
1995 }
1996
1997 kern_return_t
ledger_debit_nocheck(ledger_t ledger,int entry,ledger_amount_t amount)1998 ledger_debit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
1999 {
2000 return ledger_debit_thread(NULL, ledger, entry, amount);
2001 }
2002
2003 void
ledger_ast(thread_t thread)2004 ledger_ast(thread_t thread)
2005 {
2006 struct ledger *l = thread->t_ledger;
2007 struct ledger *thl;
2008 struct ledger *coalition_ledger;
2009 uint32_t block;
2010 uint64_t now;
2011 uint8_t task_flags;
2012 uint8_t task_percentage;
2013 uint64_t task_interval;
2014
2015 kern_return_t ret;
2016 task_t task = get_threadtask(thread);
2017
2018 lprintf(("Ledger AST for %p\n", thread));
2019
2020 ASSERT(task != NULL);
2021 ASSERT(thread == current_thread());
2022
2023 #if CONFIG_SCHED_RT_ALLOW
2024 /*
2025 * The RT policy may have forced a CPU limit on the thread. Check if
2026 * that's the case and apply the limit as requested.
2027 */
2028 spl_t s = splsched();
2029 thread_lock(thread);
2030
2031 int req_action = thread->t_ledger_req_action;
2032 uint8_t req_percentage = thread->t_ledger_req_percentage;
2033 uint64_t req_interval_ns = thread->t_ledger_req_interval_ms * NSEC_PER_MSEC;
2034
2035 thread->t_ledger_req_action = 0;
2036
2037 thread_unlock(thread);
2038 splx(s);
2039
2040 if (req_action != 0) {
2041 assert(req_action == THREAD_CPULIMIT_DISABLE ||
2042 req_action == THREAD_CPULIMIT_BLOCK);
2043
2044 if (req_action == THREAD_CPULIMIT_DISABLE &&
2045 (thread->options & TH_OPT_FORCED_LEDGER) != 0) {
2046 thread->options &= ~TH_OPT_FORCED_LEDGER;
2047 ret = thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, 0, 0);
2048 assert3u(ret, ==, KERN_SUCCESS);
2049 }
2050
2051 if (req_action == THREAD_CPULIMIT_BLOCK) {
2052 thread->options &= ~TH_OPT_FORCED_LEDGER;
2053 ret = thread_set_cpulimit(THREAD_CPULIMIT_BLOCK,
2054 req_percentage, req_interval_ns);
2055 assert3u(ret, ==, KERN_SUCCESS);
2056 thread->options |= TH_OPT_FORCED_LEDGER;
2057 }
2058 }
2059 #endif /* CONFIG_SCHED_RT_ALLOW */
2060
2061 top:
2062 /*
2063 * Take a self-consistent snapshot of the CPU usage monitor parameters. The task
2064 * can change them at any point (with the task locked).
2065 */
2066 task_lock(task);
2067 task_flags = task->rusage_cpu_flags;
2068 task_percentage = task->rusage_cpu_perthr_percentage;
2069 task_interval = task->rusage_cpu_perthr_interval;
2070 task_unlock(task);
2071
2072 /*
2073 * Make sure this thread is up to date with regards to any task-wide per-thread
2074 * CPU limit, but only if it doesn't have a thread-private blocking CPU limit.
2075 */
2076 if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) != 0) &&
2077 ((thread->options & TH_OPT_PRVT_CPULIMIT) == 0)) {
2078 uint8_t percentage;
2079 uint64_t interval;
2080 int action;
2081
2082 thread_get_cpulimit(&action, &percentage, &interval);
2083
2084 /*
2085 * If the thread's CPU limits no longer match the task's, or the
2086 * task has a limit but the thread doesn't, update the limit.
2087 */
2088 if (((thread->options & TH_OPT_PROC_CPULIMIT) == 0) ||
2089 (interval != task_interval) || (percentage != task_percentage)) {
2090 thread_set_cpulimit(THREAD_CPULIMIT_EXCEPTION, task_percentage, task_interval);
2091 assert((thread->options & TH_OPT_PROC_CPULIMIT) != 0);
2092 }
2093 } else if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) &&
2094 (thread->options & TH_OPT_PROC_CPULIMIT)) {
2095 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
2096
2097 /*
2098 * Task no longer has a per-thread CPU limit; remove this thread's
2099 * corresponding CPU limit.
2100 */
2101 thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, 0, 0);
2102 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
2103 }
2104
2105 /*
2106 * If the task or thread is being terminated, let's just get on with it
2107 */
2108 if ((l == NULL) || !task->active || task->halting || !thread->active) {
2109 return;
2110 }
2111
2112 /*
2113 * Examine all entries in deficit to see which might be eligble for
2114 * an automatic refill, which require callbacks to be issued, and
2115 * which require blocking.
2116 */
2117 block = 0;
2118 now = mach_absolute_time();
2119
2120 /*
2121 * Note that thread->t_threadledger may have been changed by the
2122 * thread_set_cpulimit() call above - so don't examine it until afterwards.
2123 */
2124 thl = thread->t_threadledger;
2125 if (LEDGER_VALID(thl)) {
2126 block |= ledger_check_needblock(thl, now);
2127 }
2128 block |= ledger_check_needblock(l, now);
2129
2130 coalition_ledger = coalition_ledger_get_from_task(task);
2131 if (LEDGER_VALID(coalition_ledger)) {
2132 block |= ledger_check_needblock(coalition_ledger, now);
2133 }
2134 ledger_dereference(coalition_ledger);
2135 /*
2136 * If we are supposed to block on the availability of one or more
2137 * resources, find the first entry in deficit for which we should wait.
2138 * Schedule a refill if necessary and then sleep until the resource
2139 * becomes available.
2140 */
2141 if (block) {
2142 if (LEDGER_VALID(thl)) {
2143 ret = ledger_perform_blocking(thl);
2144 if (ret != KERN_SUCCESS) {
2145 goto top;
2146 }
2147 }
2148 ret = ledger_perform_blocking(l);
2149 if (ret != KERN_SUCCESS) {
2150 goto top;
2151 }
2152 } /* block */
2153 }
2154
2155 static uint32_t
ledger_check_needblock(ledger_t l,uint64_t now)2156 ledger_check_needblock(ledger_t l, uint64_t now)
2157 {
2158 int i;
2159 uint32_t flags, block = 0;
2160 struct ledger_entry *le;
2161 struct ledger_callback *lc;
2162 struct entry_template *et = NULL;
2163 ledger_template_t template = NULL;
2164
2165 template = l->l_template;
2166 assert(template != NULL);
2167 assert(template->lt_initialized);
2168 /*
2169 * The template has been initialized so the entries table can't change.
2170 * Thus we don't need to acquire the template lock or the inuse bit.
2171 */
2172
2173
2174 for (i = 0; i < template->lt_cnt; i++) {
2175 spl_t s;
2176 et = &template->lt_entries[i];
2177 if (et->et_size == sizeof(struct ledger_entry_small)) {
2178 /* Small entries don't track limits or have callbacks */
2179 continue;
2180 }
2181 assert(et->et_size == sizeof(struct ledger_entry));
2182 le = (struct ledger_entry *) &l->l_entries[et->et_offset];
2183
2184 TEMPLATE_INUSE(s, template);
2185 lc = template->lt_entries[i].et_callback;
2186 TEMPLATE_IDLE(s, template);
2187
2188 if (limit_exceeded(le) == FALSE) {
2189 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
2190 /*
2191 * If needed, invoke the callback as a warning.
2192 * This needs to happen both when the balance rises above
2193 * the warning level, and also when it dips back below it.
2194 */
2195 assert(lc != NULL);
2196 /*
2197 * See comments for matching logic in ledger_check_new_balance().
2198 */
2199 if (warn_level_exceeded(le)) {
2200 flags = flag_set(&le->le_flags, LF_WARNED);
2201 if ((flags & LF_WARNED) == 0) {
2202 lc->lc_func(LEDGER_WARNING_ROSE_ABOVE, lc->lc_param0, lc->lc_param1);
2203 }
2204 } else {
2205 flags = flag_clear(&le->le_flags, LF_WARNED);
2206 if (flags & LF_WARNED) {
2207 lc->lc_func(LEDGER_WARNING_DIPPED_BELOW, lc->lc_param0, lc->lc_param1);
2208 }
2209 }
2210 }
2211 #if DEBUG || DEVELOPMENT
2212 if (diag_mem_threshold_exceeded(le)) {
2213 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
2214 assert(lc != NULL);
2215 flags = flag_set(&le->le_flags, LF_DIAG_WARNED);
2216 if ((flags & LF_DIAG_WARNED) == 0) {
2217 lc->lc_func(LEDGER_WARNING_DIAG_MEM_THRESHOLD, lc->lc_param0, lc->lc_param1);
2218 }
2219 }
2220 }
2221 #endif
2222 continue;
2223 }
2224 #if DEBUG || DEVELOPMENT
2225 if (diag_mem_threshold_exceeded(le)) {
2226 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
2227 assert(lc != NULL);
2228 flags = flag_set(&le->le_flags, LF_DIAG_WARNED);
2229 if ((flags & LF_DIAG_WARNED) == 0) {
2230 lc->lc_func(LEDGER_WARNING_DIAG_MEM_THRESHOLD, lc->lc_param0, lc->lc_param1);
2231 }
2232 }
2233 }
2234 #endif
2235
2236 /* We're over the limit, so refill if we are eligible and past due. */
2237 if (le->le_flags & LF_REFILL_SCHEDULED) {
2238 assert(!(le->le_flags & LF_TRACKING_MAX));
2239
2240 if ((le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period) <= now) {
2241 ledger_refill(now, l, i);
2242 if (limit_exceeded(le) == FALSE) {
2243 continue;
2244 }
2245 }
2246 }
2247
2248 if (le->le_flags & LEDGER_ACTION_BLOCK) {
2249 block = 1;
2250 }
2251 if ((le->le_flags & LEDGER_ACTION_CALLBACK) == 0) {
2252 continue;
2253 }
2254
2255 /*
2256 * If the LEDGER_ACTION_CALLBACK flag is on, we expect there to
2257 * be a registered callback.
2258 */
2259 assert(lc != NULL);
2260 flags = flag_set(&le->le_flags, LF_CALLED_BACK);
2261 /* Callback has already been called */
2262 if (flags & LF_CALLED_BACK) {
2263 continue;
2264 }
2265 lc->lc_func(FALSE, lc->lc_param0, lc->lc_param1);
2266 }
2267 return block;
2268 }
2269
2270
2271 /* return KERN_SUCCESS to continue, KERN_FAILURE to restart */
2272 static kern_return_t
ledger_perform_blocking(ledger_t l)2273 ledger_perform_blocking(ledger_t l)
2274 {
2275 int i;
2276 kern_return_t ret;
2277 struct ledger_entry *le;
2278 ledger_template_t template = NULL;
2279 struct entry_template *et = NULL;
2280
2281 template = l->l_template;
2282 assert(template->lt_initialized);
2283
2284 for (i = 0; i < template->lt_cnt; i++) {
2285 et = &template->lt_entries[i];
2286 if (et->et_size != sizeof(struct ledger_entry)) {
2287 /* Small entries do not block for anything. */
2288 continue;
2289 }
2290 le = (struct ledger_entry *) &l->l_entries[et->et_offset];
2291 if ((!limit_exceeded(le)) ||
2292 ((le->le_flags & LEDGER_ACTION_BLOCK) == 0)) {
2293 continue;
2294 }
2295
2296 assert(!(le->le_flags & LF_TRACKING_MAX));
2297
2298 /* Prepare to sleep until the resource is refilled */
2299 ret = assert_wait_deadline(le, THREAD_INTERRUPTIBLE,
2300 le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period);
2301 if (ret != THREAD_WAITING) {
2302 return KERN_SUCCESS;
2303 }
2304
2305 /* Mark that somebody is waiting on this entry */
2306 flag_set(&le->le_flags, LF_WAKE_NEEDED);
2307
2308 ret = thread_block_reason(THREAD_CONTINUE_NULL, NULL,
2309 AST_LEDGER);
2310 if (ret != THREAD_AWAKENED) {
2311 return KERN_SUCCESS;
2312 }
2313
2314 /*
2315 * The world may have changed while we were asleep.
2316 * Some other resource we need may have gone into
2317 * deficit. Or maybe we're supposed to die now.
2318 * Go back to the top and reevaluate.
2319 */
2320 return KERN_FAILURE;
2321 }
2322 return KERN_SUCCESS;
2323 }
2324
2325
2326 kern_return_t
ledger_get_entries(ledger_t ledger,int entry,ledger_amount_t * credit,ledger_amount_t * debit)2327 ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit,
2328 ledger_amount_t *debit)
2329 {
2330 struct ledger_entry *le = NULL;
2331 struct ledger_entry_small *les = NULL;
2332 uint16_t entry_size, entry_offset;
2333
2334 if (!is_entry_valid_and_active(ledger, entry)) {
2335 return KERN_INVALID_ARGUMENT;
2336 }
2337
2338 entry_size = ENTRY_ID_SIZE(entry);
2339 entry_offset = ENTRY_ID_OFFSET(entry);
2340 les = &ledger->l_entries[entry_offset];
2341 if (entry_size == sizeof(struct ledger_entry)) {
2342 le = (struct ledger_entry *)les;
2343 *credit = le->le_credit;
2344 *debit = le->le_debit;
2345 } else if (entry_size == sizeof(struct ledger_entry_small)) {
2346 if (les->les_flags & LF_IS_COUNTER) {
2347 struct ledger_entry_counter *lec = (struct ledger_entry_counter *) les;
2348 *credit = counter_load(&lec->lec_counter);
2349 } else {
2350 *credit = les->les_credit;
2351 }
2352 *debit = 0;
2353 } else {
2354 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
2355 }
2356
2357 return KERN_SUCCESS;
2358 }
2359
2360 kern_return_t
ledger_reset_callback_state(ledger_t ledger,int entry)2361 ledger_reset_callback_state(ledger_t ledger, int entry)
2362 {
2363 struct ledger_entry *le = NULL;
2364
2365 if (!is_entry_valid_and_active(ledger, entry)) {
2366 return KERN_INVALID_ARGUMENT;
2367 }
2368
2369 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2370 /* small entries can't have callbacks */
2371 return KERN_INVALID_ARGUMENT;
2372 }
2373
2374 le = ledger_entry_identifier_to_entry(ledger, entry);
2375
2376 flag_clear(&le->le_flags, LF_CALLED_BACK);
2377
2378 return KERN_SUCCESS;
2379 }
2380
2381 kern_return_t
ledger_disable_panic_on_negative(ledger_t ledger,int entry)2382 ledger_disable_panic_on_negative(ledger_t ledger, int entry)
2383 {
2384 volatile uint32_t *flags;
2385
2386 if (!is_entry_valid_and_active(ledger, entry)) {
2387 return KERN_INVALID_ARGUMENT;
2388 }
2389 flags = get_entry_flags(ledger, entry);
2390
2391 flag_clear(flags, LF_PANIC_ON_NEGATIVE);
2392
2393 return KERN_SUCCESS;
2394 }
2395
2396 kern_return_t
ledger_get_panic_on_negative(ledger_t ledger,int entry,int * panic_on_negative)2397 ledger_get_panic_on_negative(ledger_t ledger, int entry, int *panic_on_negative)
2398 {
2399 volatile uint32_t flags;
2400
2401 if (!is_entry_valid_and_active(ledger, entry)) {
2402 return KERN_INVALID_ARGUMENT;
2403 }
2404 flags = *get_entry_flags(ledger, entry);
2405
2406 if (flags & LF_PANIC_ON_NEGATIVE) {
2407 *panic_on_negative = TRUE;
2408 } else {
2409 *panic_on_negative = FALSE;
2410 }
2411
2412 return KERN_SUCCESS;
2413 }
2414
2415 kern_return_t
ledger_get_balance(ledger_t ledger,int entry,ledger_amount_t * balance)2416 ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
2417 {
2418 kern_return_t kr;
2419 ledger_amount_t credit, debit;
2420
2421 kr = ledger_get_entries(ledger, entry, &credit, &debit);
2422 if (kr != KERN_SUCCESS) {
2423 return kr;
2424 }
2425 *balance = credit - debit;
2426
2427 return KERN_SUCCESS;
2428 }
2429
2430 int
ledger_template_info(void ** buf,int * len)2431 ledger_template_info(void **buf, int *len)
2432 {
2433 struct ledger_template_info *lti;
2434 struct entry_template *et;
2435 ledger_template_t template;
2436 int i;
2437 ledger_t l;
2438
2439 /*
2440 * Since all tasks share a ledger template, we'll just use the
2441 * caller's as the source.
2442 */
2443 l = current_task()->ledger;
2444 if ((*len < 0) || (l == NULL)) {
2445 return EINVAL;
2446 }
2447 template = l->l_template;
2448 assert(template);
2449 assert(template->lt_initialized);
2450
2451 if (*len > template->lt_cnt) {
2452 *len = template->lt_cnt;
2453 }
2454 lti = kalloc_data((*len) * sizeof(struct ledger_template_info),
2455 Z_WAITOK);
2456 if (lti == NULL) {
2457 return ENOMEM;
2458 }
2459 *buf = lti;
2460
2461 template_lock(template);
2462 et = template->lt_entries;
2463
2464 for (i = 0; i < *len; i++) {
2465 memset(lti, 0, sizeof(*lti));
2466 strlcpy(lti->lti_name, et->et_key, LEDGER_NAME_MAX);
2467 strlcpy(lti->lti_group, et->et_group, LEDGER_NAME_MAX);
2468 strlcpy(lti->lti_units, et->et_units, LEDGER_NAME_MAX);
2469 et++;
2470 lti++;
2471 }
2472 template_unlock(template);
2473
2474 return 0;
2475 }
2476
2477 static kern_return_t
_ledger_fill_entry_info(ledger_t ledger,int entry,struct ledger_entry_info * lei,uint64_t now)2478 _ledger_fill_entry_info(ledger_t ledger,
2479 int entry,
2480 struct ledger_entry_info *lei,
2481 uint64_t now)
2482 {
2483 assert(ledger != NULL);
2484 assert(lei != NULL);
2485 if (!is_entry_valid(ledger, entry)) {
2486 return KERN_INVALID_ARGUMENT;
2487 }
2488 uint16_t entry_size, entry_offset;
2489 struct ledger_entry_small *les = NULL;
2490 struct ledger_entry *le = NULL;
2491 entry_size = ENTRY_ID_SIZE(entry);
2492 entry_offset = ENTRY_ID_OFFSET(entry);
2493
2494 les = &ledger->l_entries[entry_offset];
2495 memset(lei, 0, sizeof(*lei));
2496 if (entry_size == sizeof(struct ledger_entry_small)) {
2497 if (les->les_flags & LF_IS_COUNTER) {
2498 struct ledger_entry_counter *lec = (struct ledger_entry_counter *) les;
2499 lei->lei_credit = counter_load(&lec->lec_counter);
2500 } else {
2501 lei->lei_credit = les->les_credit;
2502 }
2503 lei->lei_limit = LEDGER_LIMIT_INFINITY;
2504 lei->lei_debit = 0;
2505 lei->lei_refill_period = 0;
2506 lei->lei_last_refill = abstime_to_nsecs(now);
2507 } else if (entry_size == sizeof(struct ledger_entry)) {
2508 le = (struct ledger_entry *) les;
2509 lei->lei_limit = le->le_limit;
2510 lei->lei_credit = le->le_credit;
2511 lei->lei_debit = le->le_debit;
2512 lei->lei_refill_period = (le->le_flags & LF_REFILL_SCHEDULED) ?
2513 abstime_to_nsecs(le->_le.le_refill.le_refill_period) : 0;
2514 lei->lei_last_refill = abstime_to_nsecs(now - le->_le.le_refill.le_last_refill);
2515 } else {
2516 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
2517 }
2518
2519 lei->lei_balance = lei->lei_credit - lei->lei_debit;
2520
2521 return KERN_SUCCESS;
2522 }
2523
2524 static kern_return_t
ledger_fill_entry_info(ledger_t ledger,int entry,void * lei_generic,uint64_t now,bool v2)2525 ledger_fill_entry_info(ledger_t ledger,
2526 int entry,
2527 void *lei_generic,
2528 uint64_t now,
2529 bool v2)
2530 {
2531 ledger_amount_t max;
2532 kern_return_t kr;
2533 struct ledger_entry_info *lei = (struct ledger_entry_info *)lei_generic;
2534 struct ledger_entry_info_v2 *lei_v2 = (struct ledger_entry_info_v2 *)lei_generic;
2535
2536 kr = _ledger_fill_entry_info(ledger, entry, lei, now);
2537 if (kr != KERN_SUCCESS) {
2538 return kr;
2539 }
2540
2541 if (v2) {
2542 lei_v2->lei_lifetime_max = -1;
2543 if (KERN_SUCCESS == ledger_get_lifetime_max(ledger, entry, &max)) {
2544 lei_v2->lei_lifetime_max = max;
2545 }
2546 }
2547
2548 return KERN_SUCCESS;
2549 }
2550
2551
2552 int
ledger_get_task_entry_info_multiple(task_t task,void ** buf,int * len,bool v2)2553 ledger_get_task_entry_info_multiple(task_t task, void **buf, int *len, bool v2)
2554 {
2555 void *lei_buf = NULL, *lei_curr = NULL;
2556 uint64_t now = mach_absolute_time();
2557 vm_size_t buf_size = 0, entry_size = 0;
2558 int i;
2559 ledger_t l;
2560 ledger_template_t template;
2561 struct entry_template *et = NULL;
2562
2563 if ((*len < 0) || ((l = task->ledger) == NULL)) {
2564 return EINVAL;
2565 }
2566 template = l->l_template;
2567 assert(template && template->lt_initialized);
2568
2569 if (*len > template->lt_cnt) {
2570 *len = template->lt_cnt;
2571 }
2572 entry_size = (v2) ? sizeof(struct ledger_entry_info_v2) : sizeof(struct ledger_entry_info);
2573 buf_size = (*len) * entry_size;
2574 lei_buf = kalloc_data(buf_size, Z_WAITOK);
2575 if (lei_buf == NULL) {
2576 return ENOMEM;
2577 }
2578 lei_curr = lei_buf;
2579
2580 for (i = 0; i < *len; i++) {
2581 et = &template->lt_entries[i];
2582 int index = ledger_entry_id_from_template_entry(et);
2583 if (ledger_fill_entry_info(l, index, lei_curr, now, v2) != KERN_SUCCESS) {
2584 kfree_data(lei_buf, buf_size);
2585 lei_buf = NULL;
2586 return EINVAL;
2587 }
2588 lei_curr = (void *)((mach_vm_address_t)lei_curr + entry_size);
2589 }
2590
2591 *buf = lei_buf;
2592 return 0;
2593 }
2594
2595 void
ledger_get_entry_info(ledger_t ledger,int entry,struct ledger_entry_info * lei)2596 ledger_get_entry_info(ledger_t ledger,
2597 int entry,
2598 struct ledger_entry_info *lei)
2599 {
2600 uint64_t now = mach_absolute_time();
2601
2602 assert(ledger != NULL);
2603 assert(lei != NULL);
2604
2605 _ledger_fill_entry_info(ledger, entry, lei, now);
2606 }
2607
2608 int
ledger_info(task_t task,struct ledger_info * info)2609 ledger_info(task_t task, struct ledger_info *info)
2610 {
2611 ledger_t l;
2612
2613 if ((l = task->ledger) == NULL) {
2614 return ENOENT;
2615 }
2616
2617 memset(info, 0, sizeof(*info));
2618
2619 strlcpy(info->li_name, l->l_template->lt_name, LEDGER_NAME_MAX);
2620 info->li_id = l->l_id;
2621 info->li_entries = l->l_template->lt_cnt;
2622 return 0;
2623 }
2624
2625 /*
2626 * Returns the amount that would be required to hit the limit.
2627 * Must be a valid, active, full-sized ledger.
2628 */
2629 ledger_amount_t
ledger_get_remaining(ledger_t ledger,int entry)2630 ledger_get_remaining(ledger_t ledger, int entry)
2631 {
2632 const struct ledger_entry *le =
2633 ledger_entry_identifier_to_entry(ledger, entry);
2634 const ledger_amount_t limit = le->le_limit;
2635 const ledger_amount_t balance = le->le_credit - le->le_debit;
2636
2637 /* +1 here as the limit isn't hit until the limit is exceeded. */
2638 return limit > balance ? limit - balance + 1 : 0;
2639 }
2640
2641 /*
2642 * Balances the ledger by modifying the debit only and sets the last refill time
2643 * to `now`.
2644 * WARNING: It is up to the caller to enforce consistency.
2645 * Must be a valid, active, full-sized ledger.
2646 */
2647 void
ledger_restart(ledger_t ledger,int entry,uint64_t now)2648 ledger_restart(ledger_t ledger, int entry, uint64_t now)
2649 {
2650 struct ledger_entry *le = ledger_entry_identifier_to_entry(ledger, entry);
2651
2652 le->le_debit = le->le_credit;
2653 le->_le.le_refill.le_last_refill = now;
2654 }
2655
2656 /*
2657 * Returns the amount of time that would have to pass to expire the current
2658 * interval.
2659 * Must be a valid, active, full-sized ledger.
2660 */
2661 uint64_t
ledger_get_interval_remaining(ledger_t ledger,int entry,uint64_t now)2662 ledger_get_interval_remaining(ledger_t ledger, int entry, uint64_t now)
2663 {
2664 const struct ledger_entry *le =
2665 ledger_entry_identifier_to_entry(ledger, entry);
2666
2667 if ((now - le->_le.le_refill.le_last_refill) >
2668 le->_le.le_refill.le_refill_period) {
2669 return 0;
2670 } else {
2671 return le->_le.le_refill.le_refill_period -
2672 (now - le->_le.le_refill.le_last_refill) + 1;
2673 }
2674 }
2675
2676 #ifdef LEDGER_DEBUG
2677 int
ledger_limit(task_t task,struct ledger_limit_args * args)2678 ledger_limit(task_t task, struct ledger_limit_args *args)
2679 {
2680 ledger_t l;
2681 int64_t limit;
2682 int idx;
2683
2684 if ((l = task->ledger) == NULL) {
2685 return EINVAL;
2686 }
2687
2688 idx = ledger_key_lookup(l->l_template, args->lla_name);
2689 if (idx < 0) {
2690 return EINVAL;
2691 }
2692 if (ENTRY_ID_SIZE(idx) == sizeof(ledger_entry_small)) {
2693 /* Small entries can't have limits */
2694 return EINVAL;
2695 }
2696
2697 /*
2698 * XXX - this doesn't really seem like the right place to have
2699 * a context-sensitive conversion of userspace units into kernel
2700 * units. For now I'll handwave and say that the ledger() system
2701 * call isn't meant for civilians to use - they should be using
2702 * the process policy interfaces.
2703 */
2704 if (idx == task_ledgers.cpu_time) {
2705 int64_t nsecs;
2706
2707 if (args->lla_refill_period) {
2708 /*
2709 * If a refill is scheduled, then the limit is
2710 * specified as a percentage of one CPU. The
2711 * syscall specifies the refill period in terms of
2712 * milliseconds, so we need to convert to nsecs.
2713 */
2714 args->lla_refill_period *= 1000000;
2715 nsecs = args->lla_limit *
2716 (args->lla_refill_period / 100);
2717 lprintf(("CPU limited to %lld nsecs per second\n",
2718 nsecs));
2719 } else {
2720 /*
2721 * If no refill is scheduled, then this is a
2722 * fixed amount of CPU time (in nsecs) that can
2723 * be consumed.
2724 */
2725 nsecs = args->lla_limit;
2726 lprintf(("CPU limited to %lld nsecs\n", nsecs));
2727 }
2728 limit = nsecs_to_abstime(nsecs);
2729 } else {
2730 limit = args->lla_limit;
2731 lprintf(("%s limited to %lld\n", args->lla_name, limit));
2732 }
2733
2734 if (args->lla_refill_period > 0) {
2735 ledger_set_period(l, idx, args->lla_refill_period);
2736 }
2737
2738 ledger_set_limit(l, idx, limit);
2739
2740 flag_set(ledger_entry_identifier_to_entry(l, idx)->le_flags, LEDGER_ACTION_BLOCK);
2741 return 0;
2742 }
2743 #endif
2744
2745 /*
2746 * Adjust the diag mem threshold limit of a resource. The diag mem threshold limit only
2747 * works prescaled by 20 bits (mb)
2748 */
2749 #if DEBUG || DEVELOPMENT
2750 kern_return_t
ledger_set_diag_mem_threshold(ledger_t ledger,int entry,ledger_amount_t limit)2751 ledger_set_diag_mem_threshold(ledger_t ledger, int entry, ledger_amount_t limit)
2752 {
2753 struct ledger_entry *le;
2754
2755 if (!is_entry_valid_and_active(ledger, entry)) {
2756 return KERN_INVALID_VALUE;
2757 }
2758
2759 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2760 /* Small entries can't have limits */
2761 return KERN_INVALID_ARGUMENT;
2762 }
2763
2764 lprintf(("ledger_set_diag mem threshold_limit: %lld\n", limit));
2765 le = ledger_entry_identifier_to_entry(ledger, entry);
2766 le->le_diag_threshold_scaled = (int16_t)LEDGER_DIAG_MEM_AMOUNT_TO_THRESHOLD(limit);
2767 lprintf(("ledger_set_diag mem threshold_limit new : %lld\n", limit));
2768 flag_clear(&le->le_flags, LF_DIAG_WARNED);
2769
2770 return KERN_SUCCESS;
2771 }
2772
2773 kern_return_t
ledger_get_diag_mem_threshold(ledger_t ledger,int entry,ledger_amount_t * limit)2774 ledger_get_diag_mem_threshold(ledger_t ledger, int entry, ledger_amount_t *limit)
2775 {
2776 struct ledger_entry *le;
2777
2778 if (!is_entry_valid_and_active(ledger, entry)) {
2779 return KERN_INVALID_VALUE;
2780 }
2781
2782 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2783 /* Small entries can't have limits */
2784 *limit = LEDGER_LIMIT_INFINITY;
2785 } else {
2786 le = ledger_entry_identifier_to_entry(ledger, entry);
2787 if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2788 *limit = LEDGER_LIMIT_INFINITY;
2789 } else {
2790 *limit = LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(le->le_diag_threshold_scaled);
2791 }
2792 }
2793
2794 lprintf(("ledger_get_diag mem threshold_limit: %lld\n", *limit));
2795
2796 return KERN_SUCCESS;
2797 }
2798
2799 static inline void
ledger_set_diag_mem_threshold_flag_disabled_internal(struct ledger_entry * le,bool value)2800 ledger_set_diag_mem_threshold_flag_disabled_internal(struct ledger_entry *le, bool value)
2801 {
2802 if (value == true) {
2803 flag_set(&le->le_flags, LF_DIAG_DISABLED);
2804 } else {
2805 flag_clear(&le->le_flags, LF_DIAG_DISABLED);
2806 }
2807 }
2808
2809 static inline bool
ledger_is_diag_threshold_enabled_internal(struct ledger_entry * le)2810 ledger_is_diag_threshold_enabled_internal( struct ledger_entry *le)
2811 {
2812 return ((le->le_flags & LF_DIAG_DISABLED) == 0)? true : false;
2813 }
2814
2815 /**
2816 * Disable the diagnostics threshold due to overlap with footprint limit
2817 */
2818 kern_return_t
ledger_set_diag_mem_threshold_disabled(ledger_t ledger,int entry)2819 ledger_set_diag_mem_threshold_disabled(ledger_t ledger, int entry)
2820 {
2821 struct ledger_entry *le;
2822
2823 if (!is_entry_valid_and_active(ledger, entry)) {
2824 return KERN_INVALID_VALUE;
2825 }
2826
2827 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2828 /* Small entries can't have limits */
2829 return KERN_INVALID_ARGUMENT;
2830 }
2831
2832 lprintf(("ledger_set_diag_mem_threshold_disabled"));
2833 le = ledger_entry_identifier_to_entry(ledger, entry);
2834 if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2835 lprintf(("ledger_set_diag_mem_threshold_disabled, cannot disable a ledger entry that have no value, returning error"));
2836 return KERN_INVALID_ARGUMENT;
2837 }
2838 ledger_set_diag_mem_threshold_flag_disabled_internal(le, true);
2839 return KERN_SUCCESS;
2840 }
2841 /**
2842 * Enable the diagnostics threshold for a specific entry
2843 */
2844 kern_return_t
ledger_set_diag_mem_threshold_enabled(ledger_t ledger,int entry)2845 ledger_set_diag_mem_threshold_enabled(ledger_t ledger, int entry)
2846 {
2847 struct ledger_entry *le;
2848
2849 if (!is_entry_valid_and_active(ledger, entry)) {
2850 return KERN_INVALID_VALUE;
2851 }
2852
2853 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2854 /* Small entries can't have limits */
2855 return KERN_INVALID_ARGUMENT;
2856 }
2857
2858 lprintf(("ledger_set_diag_mem_threshold_enabled"));
2859 le = ledger_entry_identifier_to_entry(ledger, entry);
2860 /*
2861 * if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2862 * lprintf(("ledger_set_diag_mem_threshold_enabled, cannot disable a ledger entry that have no value, returning error"));
2863 * return KERN_INVALID_ARGUMENT;
2864 * }
2865 */
2866 ledger_set_diag_mem_threshold_flag_disabled_internal(le, false);
2867
2868 return KERN_SUCCESS;
2869 }
2870 /**
2871 * Obtain the diagnostics threshold enabled flag. If the diagnostics threshold is enabled, returns true
2872 * else returns false.
2873 */
2874 kern_return_t
ledger_is_diag_threshold_enabled(ledger_t ledger,int entry,bool * status)2875 ledger_is_diag_threshold_enabled(ledger_t ledger, int entry, bool *status)
2876 {
2877 struct ledger_entry *le;
2878
2879 if (!is_entry_valid_and_active(ledger, entry)) {
2880 return KERN_INVALID_VALUE;
2881 }
2882
2883 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2884 /* Small entries can't have limits */
2885 return KERN_INVALID_ARGUMENT;
2886 }
2887
2888 lprintf(("ledger_is_diag_threshold_enabled"));
2889 le = ledger_entry_identifier_to_entry(ledger, entry);
2890 /*
2891 * if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2892 * lprintf(("ledger_is_diag_threshold_enabled, get enabled flag for a ledger entry that have no value, returning error"));
2893 * return KERN_INVALID_ARGUMENT;
2894 * }
2895 */
2896 *status = ledger_is_diag_threshold_enabled_internal(le);
2897 return KERN_SUCCESS;
2898 }
2899 #endif // DEBUG || DEVELOPMENT
2900