1 /*
2 * Copyright (c) 2020-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 * log_queue_failed_intr);
21 *
22 * @APPLE_LICENSE_HEADER_END@
23 */
24
25 #include <kern/assert.h>
26 #include <kern/counter.h>
27 #include <kern/cpu_data.h>
28 #include <kern/percpu.h>
29 #include <kern/kalloc.h>
30 #include <kern/thread_call.h>
31 #include <libkern/libkern.h>
32 #include <sys/queue.h>
33 #include <vm/vm_kern.h>
34
35 #include "log_queue.h"
36 #include "log_mem.h"
37
38 #define LQ_DEFAULT_SZ_ORDER 15 // 32K per slot
39 #define LQ_DEFAULT_FREE_AFTER_CNT 15000 // Deallocate log queue after N logs
40 #define LQ_MAX_SZ_ORDER 20 // 1MB per CPU should really be enough and a hard cap
41 #define LQ_MIN_LOG_SZ_ORDER 5
42 #define LQ_MAX_LOG_SZ_ORDER 11
43 #define LQ_BATCH_SIZE 24
44 #define LQ_MAX_LM_SLOTS 9
45 #define LQ_LOW_MEM_SCALE 3
46 #define LQ_MIN_ALLOCATED_LM_SLOTS 2
47 #define LQ_METADATA_START_SLOT 0
48 #define LQ_OTHER_START_SLOT (LQ_MIN_ALLOCATED_LM_SLOTS - 1)
49
50 #define LQ_MEM_ENABLE(q, i) ((q)->lq_mem_set |= (1 << (i)))
51 #define LQ_MEM_ENABLED(q, i) ((q)->lq_mem_set & (1 << (i)))
52 #define LQ_MEM_DISABLE(q, i) ((q)->lq_mem_set &= ~(1 << (i)))
53
54 OS_ENUM(log_queue_entry_state, uint8_t,
55 LOG_QUEUE_ENTRY_STATE_INVALID = 0,
56 LOG_QUEUE_ENTRY_STATE_STORED,
57 LOG_QUEUE_ENTRY_STATE_DISPATCHED,
58 LOG_QUEUE_ENTRY_STATE_SENT,
59 LOG_QUEUE_ENTRY_STATE_FAILED
60 );
61
62 OS_ENUM(lq_mem_state, uint8_t,
63 LQ_MEM_STATE_READY = 0,
64 LQ_MEM_STATE_ALLOCATING,
65 LQ_MEM_STATE_RELEASING
66 );
67
68 OS_ENUM(lq_req_state, uint8_t,
69 LQ_REQ_STATE_INVALID = 0,
70 LQ_REQ_STATE_ALLOCATING,
71 LQ_REQ_STATE_RELEASING,
72 LQ_REQ_STATE_READY
73 );
74
75 typedef struct log_queue_entry {
76 STAILQ_ENTRY(log_queue_entry) lqe_link;
77 uint16_t lqe_size;
78 uint16_t lqe_lm_id;
79 _Atomic log_queue_entry_state_t lqe_state;
80 log_payload_s lqe_payload;
81 } log_queue_entry_s, *log_queue_entry_t;
82
83 typedef STAILQ_HEAD(, log_queue_entry) log_queue_list_s, *log_queue_list_t;
84
85 typedef struct {
86 log_queue_list_s lq_log_list;
87 log_queue_list_s lq_dispatch_list;
88 logmem_t lq_mem[LQ_MAX_LM_SLOTS];
89 size_t lq_mem_set;
90 size_t lq_mem_size;
91 size_t lq_mem_size_order;
92 lq_mem_state_t lq_mem_state;
93 thread_call_t lq_mem_handler;
94 size_t lq_cnt_mem_active;
95 size_t lq_cnt_mem_avail;
96 size_t lq_cnt_mem_meta_avail;
97 _Atomic lq_req_state_t lq_req_state;
98 void *lq_req_mem;
99 uint32_t lq_ready : 1;
100 uint32_t lq_suspend : 1;
101 } log_queue_s, *log_queue_t;
102
103 extern bool os_log_disabled(void);
104
105 /*
106 * Log Queue
107 *
108 * Log queues are allocated and set up per cpu. When a firehose memory is full
109 * logs are stored in a log queue and sent into the firehose once it has a free
110 * space again. Each log queue (memory) can grow and shrink based on demand by
111 * adding/removing additional memory to/from its memory slots. There are
112 * LQ_MAX_LM_SLOTS memory slots available for every log queue to use. Memory
113 * slots are released when not needed, with one slot always allocated per queue
114 * as a minimum.
115 *
116 * Boot args:
117 *
118 * lq_size_order: Per slot memory size defined as a power of 2 exponent
119 * (i.e. 2^lq_bootarg_size_order). Zero disables queues.
120 *
121 * lq_nslots: Number of allocated slots to boot with per each log queue.
122 * Once initial log traffic decreases, log queues release
123 * slots as needed.
124 *
125 * If extensive number of logs is expected, setting aforementioned boot-args as
126 * needed allows to capture the vast majority of logs and avoid drops.
127 */
128 TUNABLE(size_t, lq_bootarg_size_order, "lq_size_order", LQ_DEFAULT_SZ_ORDER);
129 TUNABLE(size_t, lq_bootarg_nslots, "lq_nslots", LQ_MAX_LM_SLOTS);
130
131 SCALABLE_COUNTER_DEFINE(log_queue_cnt_received);
132 SCALABLE_COUNTER_DEFINE(log_queue_cnt_rejected_fh);
133 SCALABLE_COUNTER_DEFINE(log_queue_cnt_queued);
134 SCALABLE_COUNTER_DEFINE(log_queue_cnt_sent);
135 SCALABLE_COUNTER_DEFINE(log_queue_cnt_dropped_nomem);
136 SCALABLE_COUNTER_DEFINE(log_queue_cnt_dropped_off);
137 SCALABLE_COUNTER_DEFINE(log_queue_cnt_mem_allocated);
138 SCALABLE_COUNTER_DEFINE(log_queue_cnt_mem_released);
139 SCALABLE_COUNTER_DEFINE(log_queue_cnt_mem_failed);
140
141 static log_queue_s PERCPU_DATA(oslog_queue);
142 static size_t lq_low_mem_limit;
143
144 static void *
log_queue_buffer_alloc(size_t amount)145 log_queue_buffer_alloc(size_t amount)
146 {
147 return kalloc_data_tag(amount, Z_WAITOK_ZERO, VM_KERN_MEMORY_LOG);
148 }
149
150 static void
log_queue_buffer_free(void * addr,size_t amount)151 log_queue_buffer_free(void *addr, size_t amount)
152 {
153 kfree_data(addr, amount);
154 }
155
156 static void
log_queue_increment_mem_avail(const log_queue_t lq,size_t idx,size_t size)157 log_queue_increment_mem_avail(const log_queue_t lq, size_t idx, size_t size)
158 {
159 lq->lq_cnt_mem_avail += size;
160 if (idx < LQ_OTHER_START_SLOT) {
161 lq->lq_cnt_mem_meta_avail += size;
162 }
163 }
164
165 static void
log_queue_decrement_mem_avail(const log_queue_t lq,size_t idx,size_t size)166 log_queue_decrement_mem_avail(const log_queue_t lq, size_t idx, size_t size)
167 {
168 lq->lq_cnt_mem_avail -= size;
169 if (idx < LQ_OTHER_START_SLOT) {
170 lq->lq_cnt_mem_meta_avail -= size;
171 }
172 }
173
174 #define log_queue_entry_size(p) (sizeof(log_queue_entry_s) + (p)->lp_data_size)
175
176 #define publish(a, v) os_atomic_store((a), (v), release)
177 #define read_dependency(v) os_atomic_load((v), dependency)
178 #define read_dependent(v, t) os_atomic_load_with_dependency_on((v), (uintptr_t)(t))
179 #define read_dependent_w(v, t) ({ \
180 __auto_type _v = os_atomic_inject_dependency((v), (uintptr_t)(t)); \
181 os_atomic_load_wide(_v, dependency); \
182 })
183
184 static log_queue_entry_state_t
log_queue_entry_state(const log_queue_entry_t lqe)185 log_queue_entry_state(const log_queue_entry_t lqe)
186 {
187 log_queue_entry_state_t state = read_dependency(&lqe->lqe_state);
188 assert(state != LOG_QUEUE_ENTRY_STATE_INVALID);
189 return state;
190 }
191
192 static log_queue_entry_t
log_queue_entry_alloc(log_queue_t lq,size_t lqe_size,firehose_stream_t stream_type)193 log_queue_entry_alloc(log_queue_t lq, size_t lqe_size, firehose_stream_t stream_type)
194 {
195 // some slots are exclusively reserved for metadata stream
196 short start = LQ_METADATA_START_SLOT;
197 if (stream_type != firehose_stream_metadata) {
198 start = LQ_OTHER_START_SLOT;
199 }
200
201 for (short i = start; i < LQ_MAX_LM_SLOTS; i++) {
202 if (!LQ_MEM_ENABLED(lq, i)) {
203 continue;
204 }
205 log_queue_entry_t lqe = logmem_alloc(&lq->lq_mem[i], &lqe_size);
206 if (lqe) {
207 assert(lqe_size <= lq->lq_cnt_mem_avail);
208 assert(lqe_size <= UINT16_MAX);
209 log_queue_decrement_mem_avail(lq, i, lqe_size);
210 lqe->lqe_size = (uint16_t)lqe_size;
211 lqe->lqe_lm_id = i;
212 return lqe;
213 }
214 }
215
216 return NULL;
217 }
218
219 static void
log_queue_entry_free(log_queue_t lq,log_queue_entry_t lqe)220 log_queue_entry_free(log_queue_t lq, log_queue_entry_t lqe)
221 {
222 const size_t lqe_size = lqe->lqe_size;
223 const uint16_t lqe_lm_id = lqe->lqe_lm_id;
224
225 bzero(lqe, lqe_size);
226 logmem_free(&lq->lq_mem[lqe_lm_id], lqe, lqe_size);
227 log_queue_increment_mem_avail(lq, lqe_lm_id, lqe_size);
228 }
229
230 static bool
log_queue_add_entry(log_queue_t lq,log_payload_t lp,const uint8_t * lp_data)231 log_queue_add_entry(log_queue_t lq, log_payload_t lp, const uint8_t *lp_data)
232 {
233 log_queue_entry_t lqe = log_queue_entry_alloc(lq, log_queue_entry_size(lp), lp->lp_stream);
234 if (!lqe) {
235 counter_inc_preemption_disabled(&log_queue_cnt_dropped_nomem);
236 return false;
237 }
238 assert(lqe->lqe_size >= lp->lp_data_size);
239
240 lqe->lqe_payload = *lp;
241 (void) memcpy((uint8_t *)lqe + sizeof(*lqe), lp_data, lqe->lqe_payload.lp_data_size);
242 STAILQ_INSERT_TAIL(&lq->lq_log_list, lqe, lqe_link);
243 publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_STORED);
244
245 counter_inc_preemption_disabled(&log_queue_cnt_queued);
246
247 return true;
248 }
249
250 /*
251 * Remove successfully sent logs from a dispatch list and free them.
252 */
253 static size_t
dispatch_list_cleanup(log_queue_t lq)254 dispatch_list_cleanup(log_queue_t lq)
255 {
256 log_queue_entry_t lqe, lqe_tmp;
257 size_t freed = 0;
258
259 STAILQ_FOREACH_SAFE(lqe, &lq->lq_dispatch_list, lqe_link, lqe_tmp) {
260 log_queue_entry_state_t lqe_state = log_queue_entry_state(lqe);
261 assert(lqe_state != LOG_QUEUE_ENTRY_STATE_STORED);
262
263 if (lqe_state == LOG_QUEUE_ENTRY_STATE_SENT) {
264 STAILQ_REMOVE(&lq->lq_dispatch_list, lqe, log_queue_entry, lqe_link);
265 publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_INVALID);
266 log_queue_entry_free(lq, lqe);
267 counter_dec_preemption_disabled(&log_queue_cnt_queued);
268 freed++;
269 }
270 }
271
272 return freed;
273 }
274
275 /*
276 * Walk and collect logs stored in the log queue suitable for dispatching.
277 * First, collect previously failed logs, then (if still enough space) grab new
278 * logs.
279 */
280 static size_t
log_dispatch_prepare(log_queue_t lq,size_t requested,log_queue_entry_t * buf)281 log_dispatch_prepare(log_queue_t lq, size_t requested, log_queue_entry_t *buf)
282 {
283 log_queue_entry_t lqe, lqe_tmp;
284 size_t collected = 0;
285
286 STAILQ_FOREACH(lqe, &lq->lq_dispatch_list, lqe_link) {
287 log_queue_entry_state_t lqe_state = log_queue_entry_state(lqe);
288 assert(lqe_state != LOG_QUEUE_ENTRY_STATE_STORED);
289
290 if (lqe_state == LOG_QUEUE_ENTRY_STATE_FAILED) {
291 publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_DISPATCHED);
292 buf[collected++] = lqe;
293 }
294
295 if (collected == requested) {
296 return collected;
297 }
298 }
299 assert(collected < requested);
300
301 STAILQ_FOREACH_SAFE(lqe, &lq->lq_log_list, lqe_link, lqe_tmp) {
302 assert(log_queue_entry_state(lqe) == LOG_QUEUE_ENTRY_STATE_STORED);
303
304 STAILQ_REMOVE(&lq->lq_log_list, lqe, log_queue_entry, lqe_link);
305 STAILQ_INSERT_TAIL(&lq->lq_dispatch_list, lqe, lqe_link);
306 publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_DISPATCHED);
307
308 buf[collected++] = lqe;
309 if (collected == requested) {
310 break;
311 }
312 }
313
314 return collected;
315 }
316
317 /*
318 * Send dispatched logs to the firehose. Skip streaming when replaying.
319 * Streaming does not process timestamps and would therefore show logs out of
320 * order.
321 */
322 static void
log_queue_dispatch_logs(size_t logs_count,log_queue_entry_t * logs)323 log_queue_dispatch_logs(size_t logs_count, log_queue_entry_t *logs)
324 {
325 for (size_t i = 0; i < logs_count; i++) {
326 const log_queue_entry_t lqe = logs[i];
327 log_queue_entry_state_t lqe_state = log_queue_entry_state(lqe);
328
329 if (lqe_state == LOG_QUEUE_ENTRY_STATE_DISPATCHED) {
330 const log_payload_t lqe_lp = &lqe->lqe_payload;
331
332 log_payload_s lp = {
333 .lp_ftid = read_dependent_w(&lqe_lp->lp_ftid, lqe_state),
334 .lp_timestamp = read_dependent_w(&lqe_lp->lp_timestamp, lqe_state),
335 .lp_stream = read_dependent(&lqe_lp->lp_stream, lqe_state),
336 .lp_pub_data_size = read_dependent(&lqe_lp->lp_pub_data_size, lqe_state),
337 .lp_data_size = read_dependent(&lqe_lp->lp_data_size, lqe_state)
338 };
339 const void *lp_data = (uint8_t *)lqe + sizeof(*lqe);
340
341 /*
342 * The log queue mechanism expects only the state to be
343 * modified here since we are likely running on a
344 * different cpu. Queue cleanup will be done safely
345 * later in dispatch_list_cleanup().
346 */
347 if (log_payload_send(&lp, lp_data, false)) {
348 publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_SENT);
349 counter_inc(&log_queue_cnt_sent);
350 } else {
351 publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_FAILED);
352 }
353 }
354 }
355 }
356
357 static bool
log_queue_empty(const log_queue_t lq)358 log_queue_empty(const log_queue_t lq)
359 {
360 return STAILQ_EMPTY(&lq->lq_log_list) && STAILQ_EMPTY(&lq->lq_dispatch_list);
361 }
362
363 static boolean_t
log_queue_low_mem(const log_queue_t lq)364 log_queue_low_mem(const log_queue_t lq)
365 {
366 size_t mem_avail = lq->lq_cnt_mem_avail - lq->lq_cnt_mem_meta_avail;
367 size_t low_mem_threshold = (lq->lq_cnt_mem_active - LQ_OTHER_START_SLOT) * lq_low_mem_limit;
368 return mem_avail < low_mem_threshold;
369 }
370
371 static lq_req_state_t
log_queue_request_state(log_queue_t lq)372 log_queue_request_state(log_queue_t lq)
373 {
374 lq_req_state_t req_state = read_dependency(&lq->lq_req_state);
375 return req_state;
376 }
377
378 static void
log_queue_mem_init(log_queue_t lq,size_t idx,void * buf,size_t buflen)379 log_queue_mem_init(log_queue_t lq, size_t idx, void *buf, size_t buflen)
380 {
381 assert(buf);
382 assert(buflen > 0);
383 assert(idx < LQ_MAX_LM_SLOTS);
384 assert(!LQ_MEM_ENABLED(lq, idx));
385
386 logmem_init(&lq->lq_mem[idx], buf, buflen, lq->lq_mem_size_order,
387 LQ_MIN_LOG_SZ_ORDER, LQ_MAX_LOG_SZ_ORDER);
388 }
389
390 static int
log_queue_mem_free_slot(log_queue_t lq)391 log_queue_mem_free_slot(log_queue_t lq)
392 {
393 assert(LQ_MEM_ENABLED(lq, 0));
394
395 for (int i = LQ_MIN_ALLOCATED_LM_SLOTS; i < LQ_MAX_LM_SLOTS; i++) {
396 if (!LQ_MEM_ENABLED(lq, i)) {
397 return i;
398 }
399 }
400 return -1;
401 }
402
403 static void
log_queue_memory_handler(thread_call_param_t a0,__unused thread_call_param_t a1)404 log_queue_memory_handler(thread_call_param_t a0, __unused thread_call_param_t a1)
405 {
406 log_queue_t lq = (log_queue_t)a0;
407 lq_req_state_t req_state = log_queue_request_state(lq);
408
409 assert(req_state != LQ_REQ_STATE_INVALID);
410
411 if (req_state == LQ_REQ_STATE_ALLOCATING) {
412 lq->lq_req_mem = log_queue_buffer_alloc(lq->lq_mem_size);
413 publish(&lq->lq_req_state, LQ_REQ_STATE_READY);
414
415 if (lq->lq_req_mem) {
416 counter_inc(&log_queue_cnt_mem_allocated);
417 } else {
418 counter_inc(&log_queue_cnt_mem_failed);
419 }
420 } else if (req_state == LQ_REQ_STATE_RELEASING) {
421 void *buf = read_dependent(&lq->lq_req_mem, req_state);
422
423 log_queue_buffer_free(buf, lq->lq_mem_size);
424 lq->lq_req_mem = NULL;
425 publish(&lq->lq_req_state, LQ_REQ_STATE_READY);
426
427 counter_inc(&log_queue_cnt_mem_released);
428 }
429 }
430
431 static void
log_queue_order_memory(log_queue_t lq)432 log_queue_order_memory(log_queue_t lq)
433 {
434 boolean_t __assert_only running;
435
436 lq->lq_req_mem = NULL;
437 publish(&lq->lq_req_state, LQ_REQ_STATE_ALLOCATING);
438
439 running = thread_call_enter(lq->lq_mem_handler);
440 assert(!running);
441 }
442
443 static void
log_queue_release_memory(log_queue_t lq,void * buf)444 log_queue_release_memory(log_queue_t lq, void *buf)
445 {
446 boolean_t __assert_only running;
447
448 assert(buf);
449 lq->lq_req_mem = buf;
450 publish(&lq->lq_req_state, LQ_REQ_STATE_RELEASING);
451
452 running = thread_call_enter(lq->lq_mem_handler);
453 assert(!running);
454 }
455
456 static void
log_queue_mem_enable(log_queue_t lq,size_t i)457 log_queue_mem_enable(log_queue_t lq, size_t i)
458 {
459 logmem_t *lm = &lq->lq_mem[i];
460 assert(!LQ_MEM_ENABLED(lq, i));
461
462 LQ_MEM_ENABLE(lq, i);
463 lq->lq_cnt_mem_active++;
464 log_queue_increment_mem_avail(lq, i, lm->lm_cnt_free);
465 }
466
467 static void
log_queue_mem_disable(log_queue_t lq,size_t i)468 log_queue_mem_disable(log_queue_t lq, size_t i)
469 {
470 logmem_t *lm = &lq->lq_mem[i];
471 assert(LQ_MEM_ENABLED(lq, i));
472
473 LQ_MEM_DISABLE(lq, i);
474 lq->lq_cnt_mem_active--;
475 log_queue_decrement_mem_avail(lq, i, lm->lm_cnt_free);
476 }
477
478 static void *
log_queue_mem_reclaim(log_queue_t lq)479 log_queue_mem_reclaim(log_queue_t lq)
480 {
481 for (int i = LQ_MIN_ALLOCATED_LM_SLOTS; i < LQ_MAX_LM_SLOTS; i++) {
482 logmem_t *lm = &lq->lq_mem[i];
483 if (LQ_MEM_ENABLED(lq, i) && logmem_empty(lm)) {
484 assert(lm->lm_mem_size == lq->lq_mem_size);
485 void *reclaimed = lm->lm_mem;
486 log_queue_mem_disable(lq, i);
487 /* Do not use bzero here, see rdar://116922009 */
488 *lm = (logmem_t){ };
489 return reclaimed;
490 }
491 }
492 return NULL;
493 }
494
495 static void
log_queue_mem_reconfigure(log_queue_t lq)496 log_queue_mem_reconfigure(log_queue_t lq)
497 {
498 assert(lq->lq_mem_state == LQ_MEM_STATE_ALLOCATING ||
499 lq->lq_mem_state == LQ_MEM_STATE_RELEASING);
500
501 lq_req_state_t req_state = log_queue_request_state(lq);
502
503 if (req_state == LQ_REQ_STATE_READY) {
504 if (lq->lq_mem_state == LQ_MEM_STATE_ALLOCATING) {
505 void *buf = read_dependent(&lq->lq_req_mem, req_state);
506 if (buf) {
507 const int i = log_queue_mem_free_slot(lq);
508 assert(i > 0);
509 log_queue_mem_init(lq, i, buf, lq->lq_mem_size);
510 log_queue_mem_enable(lq, i);
511 }
512 }
513 lq->lq_mem_state = LQ_MEM_STATE_READY;
514 publish(&lq->lq_req_state, LQ_REQ_STATE_INVALID);
515 }
516 }
517
518 static boolean_t
log_queue_needs_memory(log_queue_t lq,boolean_t new_suspend)519 log_queue_needs_memory(log_queue_t lq, boolean_t new_suspend)
520 {
521 if (new_suspend || log_queue_low_mem(lq)) {
522 return lq->lq_cnt_mem_active < LQ_MAX_LM_SLOTS;
523 }
524 return false;
525 }
526
527 static boolean_t
log_queue_can_release_memory(log_queue_t lq)528 log_queue_can_release_memory(log_queue_t lq)
529 {
530 assert(lq->lq_mem_state == LQ_MEM_STATE_READY);
531
532 if (lq->lq_cnt_mem_active > LQ_MIN_ALLOCATED_LM_SLOTS
533 && log_queue_empty(lq) && !lq->lq_suspend) {
534 const uint64_t total_log_cnt = counter_load(&log_queue_cnt_received);
535 return total_log_cnt > LQ_DEFAULT_FREE_AFTER_CNT;
536 }
537 return false;
538 }
539
540 extern boolean_t tasks_suspend_state;
541
542 static boolean_t
detect_new_suspend(log_queue_t lq)543 detect_new_suspend(log_queue_t lq)
544 {
545 if (!tasks_suspend_state) {
546 lq->lq_suspend = false;
547 return false;
548 }
549
550 if (!lq->lq_suspend) {
551 lq->lq_suspend = true;
552 return true;
553 }
554
555 return false;
556 }
557
558 static void
log_queue_dispatch(void)559 log_queue_dispatch(void)
560 {
561 lq_mem_state_t new_mem_state = LQ_MEM_STATE_READY;
562 void *reclaimed_memory = NULL;
563
564 disable_preemption();
565
566 log_queue_t lq = PERCPU_GET(oslog_queue);
567 if (__improbable(!lq->lq_ready)) {
568 enable_preemption();
569 return;
570 }
571
572 dispatch_list_cleanup(lq);
573
574 log_queue_entry_t logs[LQ_BATCH_SIZE];
575 size_t logs_count = log_dispatch_prepare(lq, LQ_BATCH_SIZE, (log_queue_entry_t *)&logs);
576
577 boolean_t new_suspend = detect_new_suspend(lq);
578
579 if (__improbable(lq->lq_mem_state != LQ_MEM_STATE_READY)) {
580 log_queue_mem_reconfigure(lq);
581 } else if (logs_count == 0 && log_queue_can_release_memory(lq)) {
582 reclaimed_memory = log_queue_mem_reclaim(lq);
583 if (reclaimed_memory) {
584 lq->lq_mem_state = LQ_MEM_STATE_RELEASING;
585 new_mem_state = lq->lq_mem_state;
586 }
587 } else if (log_queue_needs_memory(lq, new_suspend)) {
588 lq->lq_mem_state = LQ_MEM_STATE_ALLOCATING;
589 new_mem_state = lq->lq_mem_state;
590 }
591
592 enable_preemption();
593
594 switch (new_mem_state) {
595 case LQ_MEM_STATE_RELEASING:
596 assert(logs_count == 0);
597 log_queue_release_memory(lq, reclaimed_memory);
598 break;
599 case LQ_MEM_STATE_ALLOCATING:
600 log_queue_order_memory(lq);
601 /* FALLTHROUGH */
602 case LQ_MEM_STATE_READY:
603 log_queue_dispatch_logs(logs_count, logs);
604 break;
605 default:
606 panic("Invalid log memory state %u", new_mem_state);
607 break;
608 }
609 }
610
611 static bool
log_queue_add(log_payload_t lp,const uint8_t * lp_data)612 log_queue_add(log_payload_t lp, const uint8_t *lp_data)
613 {
614 boolean_t order_memory = false;
615
616 disable_preemption();
617
618 log_queue_t lq = PERCPU_GET(oslog_queue);
619 if (__improbable(!lq->lq_ready)) {
620 enable_preemption();
621 counter_inc(&log_queue_cnt_dropped_off);
622 return false;
623 }
624
625 boolean_t new_suspend = detect_new_suspend(lq);
626
627 if (__improbable(lq->lq_mem_state != LQ_MEM_STATE_READY)) {
628 log_queue_mem_reconfigure(lq);
629 } else if (log_queue_needs_memory(lq, new_suspend)) {
630 lq->lq_mem_state = LQ_MEM_STATE_ALLOCATING;
631 order_memory = true;
632 }
633
634 bool added = log_queue_add_entry(lq, lp, lp_data);
635 enable_preemption();
636
637 if (order_memory) {
638 log_queue_order_memory(lq);
639 }
640
641 return added;
642 }
643
644 __startup_func
645 static size_t
log_queue_init_memory(log_queue_t lq,size_t lm_count)646 log_queue_init_memory(log_queue_t lq, size_t lm_count)
647 {
648 assert(lm_count <= LQ_MAX_LM_SLOTS);
649
650 for (size_t i = 0; i < lm_count; i++) {
651 void *buf = log_queue_buffer_alloc(lq->lq_mem_size);
652 if (!buf) {
653 return i;
654 }
655 counter_inc(&log_queue_cnt_mem_allocated);
656 log_queue_mem_init(lq, i, buf, lq->lq_mem_size);
657 log_queue_mem_enable(lq, i);
658 }
659
660 return lm_count;
661 }
662
663 __startup_func
664 static void
oslog_init_log_queues(void)665 oslog_init_log_queues(void)
666 {
667 if (os_log_disabled()) {
668 printf("Log queues disabled: Logging disabled by ATM\n");
669 return;
670 }
671
672 if (lq_bootarg_size_order == 0) {
673 printf("Log queues disabled: Zero lq_size_order boot argument\n");
674 return;
675 }
676
677 lq_bootarg_size_order = MAX(lq_bootarg_size_order, PAGE_SHIFT);
678 lq_bootarg_size_order = MIN(lq_bootarg_size_order, LQ_MAX_SZ_ORDER);
679
680 lq_bootarg_nslots = MAX(lq_bootarg_nslots, LQ_MIN_ALLOCATED_LM_SLOTS);
681 lq_bootarg_nslots = MIN(lq_bootarg_nslots, LQ_MAX_LM_SLOTS);
682
683 lq_low_mem_limit = MAX(1 << (lq_bootarg_size_order - LQ_LOW_MEM_SCALE), 1024);
684
685 unsigned int slot_count = 0;
686
687 percpu_foreach(lq, oslog_queue) {
688 lq->lq_mem_size_order = lq_bootarg_size_order;
689 lq->lq_mem_size = round_page(logmem_required_size(lq->lq_mem_size_order, LQ_MIN_LOG_SZ_ORDER));
690 lq->lq_mem_handler = thread_call_allocate(log_queue_memory_handler, (thread_call_param_t)lq);
691 slot_count += log_queue_init_memory(lq, lq_bootarg_nslots);
692 STAILQ_INIT(&lq->lq_log_list);
693 STAILQ_INIT(&lq->lq_dispatch_list);
694 lq->lq_ready = true;
695 }
696
697 printf("Log queues configured: slot count: %u, per-slot size: %u, total size: %u\n",
698 slot_count, (1 << lq_bootarg_size_order),
699 slot_count * (1 << lq_bootarg_size_order));
700 }
701 STARTUP(OSLOG, STARTUP_RANK_SECOND, oslog_init_log_queues);
702
703 bool
log_queue_log(log_payload_t lp,const void * lp_data,bool stream)704 log_queue_log(log_payload_t lp, const void *lp_data, bool stream)
705 {
706 assert(lp);
707 assert(oslog_is_safe() || startup_phase < STARTUP_SUB_EARLY_BOOT);
708
709 counter_inc(&log_queue_cnt_received);
710
711 if (log_payload_send(lp, lp_data, stream)) {
712 counter_inc(&log_queue_cnt_sent);
713 log_queue_dispatch();
714 return true;
715 }
716 counter_inc(&log_queue_cnt_rejected_fh);
717
718 if (!log_queue_add(lp, lp_data)) {
719 return false;
720 }
721
722 return true;
723 }
724