164 log_queue_increment_mem_avail(const log_queue_t lq, size_t idx, size_t size)  in log_queue_increment_mem_avail()  argument
166 	lq->lq_cnt_mem_avail += size;  in log_queue_increment_mem_avail()
168 		lq->lq_cnt_mem_meta_avail += size;  in log_queue_increment_mem_avail()
173 log_queue_decrement_mem_avail(const log_queue_t lq, size_t idx, size_t size)  in log_queue_decrement_mem_avail()  argument
175 	lq->lq_cnt_mem_avail -= size;  in log_queue_decrement_mem_avail()
177 		lq->lq_cnt_mem_meta_avail -= size;  in log_queue_decrement_mem_avail()
200 log_queue_entry_alloc(log_queue_t lq, size_t lqe_size, firehose_stream_t stream_type)  in log_queue_entry_alloc()  argument
209 		if (!LQ_MEM_ENABLED(lq, i)) {  in log_queue_entry_alloc()
212 		log_queue_entry_t lqe = logmem_alloc(&lq->lq_mem[i], &lqe_size);  in log_queue_entry_alloc()
214 			assert(lqe_size <= lq->lq_cnt_mem_avail);  in log_queue_entry_alloc()
216 			log_queue_decrement_mem_avail(lq, i, lqe_size);  in log_queue_entry_alloc()
227 log_queue_entry_free(log_queue_t lq, log_queue_entry_t lqe)  in log_queue_entry_free()  argument
233 	logmem_free(&lq->lq_mem[lqe_lm_id], lqe, lqe_size);  in log_queue_entry_free()
234 	log_queue_increment_mem_avail(lq, lqe_lm_id, lqe_size);  in log_queue_entry_free()
238 log_queue_add_entry(log_queue_t lq, log_payload_t lp, const uint8_t *lp_data)  in log_queue_add_entry()  argument
240 	log_queue_entry_t lqe = log_queue_entry_alloc(lq, log_queue_entry_size(lp), lp->lp_stream);  in log_queue_add_entry()
249 	STAILQ_INSERT_TAIL(&lq->lq_log_list, lqe, lqe_link);  in log_queue_add_entry()
261 dispatch_list_cleanup(log_queue_t lq)  in dispatch_list_cleanup()  argument
266 	STAILQ_FOREACH_SAFE(lqe, &lq->lq_dispatch_list, lqe_link, lqe_tmp) {  in dispatch_list_cleanup()
271 			STAILQ_REMOVE(&lq->lq_dispatch_list, lqe, log_queue_entry, lqe_link);  in dispatch_list_cleanup()
273 			log_queue_entry_free(lq, lqe);  in dispatch_list_cleanup()
288 log_dispatch_prepare(log_queue_t lq, size_t requested, log_queue_entry_t *buf)  in log_dispatch_prepare()  argument
293 	STAILQ_FOREACH(lqe, &lq->lq_dispatch_list, lqe_link) {  in log_dispatch_prepare()
308 	STAILQ_FOREACH_SAFE(lqe, &lq->lq_log_list, lqe_link, lqe_tmp) {  in log_dispatch_prepare()
311 		STAILQ_REMOVE(&lq->lq_log_list, lqe, log_queue_entry, lqe_link);  in log_dispatch_prepare()
312 		STAILQ_INSERT_TAIL(&lq->lq_dispatch_list, lqe, lqe_link);  in log_dispatch_prepare()
365 log_queue_empty(const log_queue_t lq)  in log_queue_empty()  argument
367 	return STAILQ_EMPTY(&lq->lq_log_list) && STAILQ_EMPTY(&lq->lq_dispatch_list);  in log_queue_empty()
371 log_queue_low_mem(const log_queue_t lq)  in log_queue_low_mem()  argument
373 	size_t mem_avail = lq->lq_cnt_mem_avail - lq->lq_cnt_mem_meta_avail;  in log_queue_low_mem()
374 	size_t low_mem_threshold = (lq->lq_cnt_mem_active - LQ_OTHER_START_SLOT) * lq_low_mem_limit;  in log_queue_low_mem()
379 log_queue_request_state(log_queue_t lq)  in log_queue_request_state()  argument
381 	lq_req_state_t req_state = read_dependency(&lq->lq_req_state);  in log_queue_request_state()
386 log_queue_mem_init(log_queue_t lq, size_t idx, void *buf, size_t buflen)  in log_queue_mem_init()  argument
391 	assert(!LQ_MEM_ENABLED(lq, idx));  in log_queue_mem_init()
393 	logmem_init(&lq->lq_mem[idx], buf, buflen, lq->lq_mem_size_order,  in log_queue_mem_init()
417 log_queue_mem_free_slot(log_queue_t lq)  in log_queue_mem_free_slot()  argument
419 	assert(LQ_MEM_ENABLED(lq, 0));  in log_queue_mem_free_slot()
420 	assert(lq->lq_cnt_mem_max <= LQ_MAX_LM_SLOTS);  in log_queue_mem_free_slot()
422 	for (int i = LQ_MIN_ALLOCATED_LM_SLOTS; i < lq->lq_cnt_mem_max; i++) {  in log_queue_mem_free_slot()
423 		if (!LQ_MEM_ENABLED(lq, i)) {  in log_queue_mem_free_slot()
433 	log_queue_t lq = (log_queue_t)a0;  in log_queue_memory_handler()  local
434 	lq_req_state_t req_state = log_queue_request_state(lq);  in log_queue_memory_handler()
439 		lq->lq_req_mem = log_queue_buffer_alloc(lq->lq_mem_size);  in log_queue_memory_handler()
440 		publish(&lq->lq_req_state, LQ_REQ_STATE_READY);  in log_queue_memory_handler()
442 		if (lq->lq_req_mem) {  in log_queue_memory_handler()
448 		void *buf = read_dependent(&lq->lq_req_mem, req_state);  in log_queue_memory_handler()
450 		log_queue_buffer_free(buf, lq->lq_mem_size);  in log_queue_memory_handler()
451 		lq->lq_req_mem = NULL;  in log_queue_memory_handler()
452 		publish(&lq->lq_req_state, LQ_REQ_STATE_READY);  in log_queue_memory_handler()
459 log_queue_order_memory(log_queue_t lq)  in log_queue_order_memory()  argument
463 	lq->lq_req_mem = NULL;  in log_queue_order_memory()
464 	publish(&lq->lq_req_state, LQ_REQ_STATE_ALLOCATING);  in log_queue_order_memory()
466 	running = thread_call_enter(lq->lq_mem_handler);  in log_queue_order_memory()
471 log_queue_release_memory(log_queue_t lq, void *buf)  in log_queue_release_memory()  argument
476 	lq->lq_req_mem = buf;  in log_queue_release_memory()
477 	publish(&lq->lq_req_state, LQ_REQ_STATE_RELEASING);  in log_queue_release_memory()
479 	running = thread_call_enter(lq->lq_mem_handler);  in log_queue_release_memory()
484 log_queue_mem_enable(log_queue_t lq, size_t i)  in log_queue_mem_enable()  argument
486 	logmem_t *lm = &lq->lq_mem[i];  in log_queue_mem_enable()
487 	assert(!LQ_MEM_ENABLED(lq, i));  in log_queue_mem_enable()
489 	LQ_MEM_ENABLE(lq, i);  in log_queue_mem_enable()
490 	lq->lq_cnt_mem_active++;  in log_queue_mem_enable()
491 	log_queue_increment_mem_avail(lq, i, lm->lm_cnt_free);  in log_queue_mem_enable()
495 log_queue_mem_disable(log_queue_t lq, size_t i)  in log_queue_mem_disable()  argument
497 	logmem_t *lm = &lq->lq_mem[i];  in log_queue_mem_disable()
498 	assert(LQ_MEM_ENABLED(lq, i));  in log_queue_mem_disable()
500 	LQ_MEM_DISABLE(lq, i);  in log_queue_mem_disable()
501 	lq->lq_cnt_mem_active--;  in log_queue_mem_disable()
502 	log_queue_decrement_mem_avail(lq, i, lm->lm_cnt_free);  in log_queue_mem_disable()
506 log_queue_mem_reclaim(log_queue_t lq)  in log_queue_mem_reclaim()  argument
509 		logmem_t *lm = &lq->lq_mem[i];  in log_queue_mem_reclaim()
510 		if (LQ_MEM_ENABLED(lq, i) && logmem_empty(lm)) {  in log_queue_mem_reclaim()
511 			assert(lm->lm_mem_size == lq->lq_mem_size);  in log_queue_mem_reclaim()
513 			log_queue_mem_disable(lq, i);  in log_queue_mem_reclaim()
523 log_queue_mem_reconfigure(log_queue_t lq)  in log_queue_mem_reconfigure()  argument
525 	assert(lq->lq_mem_state == LQ_MEM_STATE_ALLOCATING ||  in log_queue_mem_reconfigure()
526 	    lq->lq_mem_state == LQ_MEM_STATE_RELEASING);  in log_queue_mem_reconfigure()
528 	lq_req_state_t req_state = log_queue_request_state(lq);  in log_queue_mem_reconfigure()
531 		if (lq->lq_mem_state == LQ_MEM_STATE_ALLOCATING) {  in log_queue_mem_reconfigure()
532 			void *buf = read_dependent(&lq->lq_req_mem, req_state);  in log_queue_mem_reconfigure()
534 				const int i = log_queue_mem_free_slot(lq);  in log_queue_mem_reconfigure()
536 				log_queue_mem_init(lq, i, buf, lq->lq_mem_size);  in log_queue_mem_reconfigure()
537 				log_queue_mem_enable(lq, i);  in log_queue_mem_reconfigure()
540 		lq->lq_mem_state = LQ_MEM_STATE_READY;  in log_queue_mem_reconfigure()
541 		publish(&lq->lq_req_state, LQ_REQ_STATE_INVALID);  in log_queue_mem_reconfigure()
546 log_queue_needs_memory(log_queue_t lq, boolean_t new_suspend)  in log_queue_needs_memory()  argument
549 	lq->lq_cnt_mem_max = log_queue_mem_max_slots();  in log_queue_needs_memory()
551 	if (new_suspend || log_queue_low_mem(lq)) {  in log_queue_needs_memory()
552 		return lq->lq_cnt_mem_active < lq->lq_cnt_mem_max;  in log_queue_needs_memory()
558 log_queue_can_release_memory(log_queue_t lq)  in log_queue_can_release_memory()  argument
560 	assert(lq->lq_mem_state == LQ_MEM_STATE_READY);  in log_queue_can_release_memory()
562 	if (lq->lq_cnt_mem_active > LQ_MIN_ALLOCATED_LM_SLOTS  in log_queue_can_release_memory()
563 	    && log_queue_empty(lq) && !lq->lq_suspend) {  in log_queue_can_release_memory()
573 detect_new_suspend(log_queue_t lq)  in detect_new_suspend()  argument
576 		lq->lq_suspend = false;  in detect_new_suspend()
580 	if (!lq->lq_suspend) {  in detect_new_suspend()
581 		lq->lq_suspend = true;  in detect_new_suspend()
596 	log_queue_t lq = PERCPU_GET(oslog_queue);  in log_queue_dispatch()  local
597 	if (__improbable(!lq->lq_ready)) {  in log_queue_dispatch()
602 	dispatch_list_cleanup(lq);  in log_queue_dispatch()
605 	size_t logs_count = log_dispatch_prepare(lq, LQ_BATCH_SIZE, (log_queue_entry_t *)&logs);  in log_queue_dispatch()
607 	boolean_t new_suspend = detect_new_suspend(lq);  in log_queue_dispatch()
609 	if (__improbable(lq->lq_mem_state != LQ_MEM_STATE_READY)) {  in log_queue_dispatch()
610 		log_queue_mem_reconfigure(lq);  in log_queue_dispatch()
611 	} else if (logs_count == 0 && log_queue_can_release_memory(lq)) {  in log_queue_dispatch()
612 		reclaimed_memory = log_queue_mem_reclaim(lq);  in log_queue_dispatch()
614 			lq->lq_mem_state = LQ_MEM_STATE_RELEASING;  in log_queue_dispatch()
615 			new_mem_state = lq->lq_mem_state;  in log_queue_dispatch()
617 	} else if (log_queue_needs_memory(lq, new_suspend)) {  in log_queue_dispatch()
618 		lq->lq_mem_state = LQ_MEM_STATE_ALLOCATING;  in log_queue_dispatch()
619 		new_mem_state = lq->lq_mem_state;  in log_queue_dispatch()
627 		log_queue_release_memory(lq, reclaimed_memory);  in log_queue_dispatch()
630 		log_queue_order_memory(lq);  in log_queue_dispatch()
648 	log_queue_t lq = PERCPU_GET(oslog_queue);  in log_queue_add()  local
649 	if (__improbable(!lq->lq_ready)) {  in log_queue_add()
655 	boolean_t new_suspend = detect_new_suspend(lq);  in log_queue_add()
657 	if (__improbable(lq->lq_mem_state != LQ_MEM_STATE_READY)) {  in log_queue_add()
658 		log_queue_mem_reconfigure(lq);  in log_queue_add()
659 	} else if (log_queue_needs_memory(lq, new_suspend)) {  in log_queue_add()
660 		lq->lq_mem_state = LQ_MEM_STATE_ALLOCATING;  in log_queue_add()
664 	bool added = log_queue_add_entry(lq, lp, lp_data);  in log_queue_add()
668 		log_queue_order_memory(lq);  in log_queue_add()
676 log_queue_init_memory(log_queue_t lq)  in log_queue_init_memory()  argument
678 	lq->lq_cnt_mem_max = log_queue_mem_max_slots();  in log_queue_init_memory()
679 	assert(lq->lq_cnt_mem_max <= LQ_MAX_LM_SLOTS);  in log_queue_init_memory()
681 	for (size_t i = 0; i < lq->lq_cnt_mem_max; i++) {  in log_queue_init_memory()
682 		void *buf = log_queue_buffer_alloc(lq->lq_mem_size);  in log_queue_init_memory()
687 		log_queue_mem_init(lq, i, buf, lq->lq_mem_size);  in log_queue_init_memory()
688 		log_queue_mem_enable(lq, i);  in log_queue_init_memory()
691 	return lq->lq_cnt_mem_max;  in log_queue_init_memory()
719 	percpu_foreach(lq, oslog_queue) {  in oslog_init_log_queues()
720 		lq->lq_mem_size_order = lq_bootarg_size_order;  in oslog_init_log_queues()
721 		lq->lq_mem_size = round_page(logmem_required_size(lq->lq_mem_size_order, LQ_MIN_LOG_SZ_ORDER));  in oslog_init_log_queues()
722 		lq->lq_mem_handler = thread_call_allocate(log_queue_memory_handler, (thread_call_param_t)lq);  in oslog_init_log_queues()
723 		slot_count += log_queue_init_memory(lq);  in oslog_init_log_queues()
724 		STAILQ_INIT(&lq->lq_log_list);  in oslog_init_log_queues()
725 		STAILQ_INIT(&lq->lq_dispatch_list);  in oslog_init_log_queues()
726 		lq->lq_ready = true;  in oslog_init_log_queues()