xref: /xnu-11417.140.69/osfmk/console/serial_console.c (revision 43a90889846e00bfb5cf1d255cdc0a701a1e05a4)

/*
 * Copyright (c) 2000-2020 Apple, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifdef __x86_64__
#include <i386/mp.h>
#include <i386/cpu_data.h>
#include <i386/bit_routines.h>
#include <i386/machine_routines.h>
#include <i386/misc_protos.h>
#include <i386/serial_io.h>
#endif /* __x86_64__ */

#include <machine/machine_cpu.h>
#include <libkern/OSAtomic.h>
#include <vm/vm_kern_xnu.h>
#include <vm/vm_map.h>
#include <console/video_console.h>
#include <console/serial_protos.h>
#include <kern/startup.h>
#include <kern/thread.h>
#include <kern/cpu_data.h>
#include <kern/sched_prim.h>
#include <libkern/section_keywords.h>

#if __arm64__
#include <machine/machine_routines.h>
#include <arm/cpu_data_internal.h>
#endif

#ifdef CONFIG_XNUPOST
#include <tests/xnupost.h>
kern_return_t console_serial_test(void);
kern_return_t console_serial_parallel_log_tests(void);
#endif

/* Structure representing the console ring buffer. */
static struct {
	/* The ring buffer backing store. */
	char *buffer;

	/* The total length of the ring buffer. */
	int len;

	/**
	 * The number of characters that have been written into the buffer that need
	 * to be drained.
	 */
	int used;

	/**
	 * Number of reserved regions in the buffer. These are regions that are
	 * currently being written into by various CPUs. We use this as a way of
	 * determining when it's safe to drain the buffer.
	 */
	int nreserved;

	/* The location in the buffer thats written to next. */
	char *write_ptr;

	/* The location in the buffer that will be drained next. */
	char *read_ptr;

	/* Synchronizes the flushing of the ring buffer to hardware */
	lck_mtx_t flush_lock;

	/**
	 * Synchronizes reserving space in the ring buffer and ensures that only
	 * completed writes are flushed.
	 */
	lck_ticket_t write_lock;
} console_ring;

/**
 * We don't dedicate any buffer space to specific CPUs, but this value is used
 * to scale the size of the console buffer by the number of CPUs.
 *
 * How many bytes-per-cpu to allocate in the console ring buffer. Also affects
 * the maximum number of bytes a single console thread can drain.
 */
#define CPU_CONS_BUF_SIZE 256

/* Scale the size of the console ring buffer by the number of CPUs. */
#define KERN_CONSOLE_RING_SIZE vm_map_round_page(CPU_CONS_BUF_SIZE * (MAX_CPUS + 1), PAGE_SIZE - 1)

#define MAX_FLUSH_SIZE_LOCK_HELD 16
#define MAX_TOTAL_FLUSH_SIZE (MAX(2, MAX_CPUS) * CPU_CONS_BUF_SIZE)

extern int serial_getc(void);
extern void serial_putc_options(char, bool);

#if DEBUG || DEVELOPMENT
TUNABLE(bool, allow_printf_from_interrupts_disabled_context, "nointr_consio", false);
#else
#define allow_printf_from_interrupts_disabled_context false
#endif

SECURITY_READ_ONLY_EARLY(struct console_ops) cons_ops[] = {
	{
		.putc = serial_putc_options, .getc = _serial_getc,
	},
	{
		.putc = vcputc_options, .getc = _vcgetc,
	},
};

SECURITY_READ_ONLY_EARLY(uint32_t) nconsops = (sizeof cons_ops / sizeof cons_ops[0]);

#if __x86_64__
uint32_t cons_ops_index = VC_CONS_OPS;
#else
SECURITY_READ_ONLY_LATE(uint32_t) cons_ops_index = VC_CONS_OPS;
#endif

LCK_GRP_DECLARE(console_lck_grp, "console");

/* If the NMI string is entered into the console, the system will enter the debugger. */
#define NMI_STRING_SIZE 32
char nmi_string[NMI_STRING_SIZE] = "afDIGHr84A84jh19Kphgp428DNPdnapq";
static int nmi_counter           = 0;

/**
 * This is used to prevent console output from going through the console ring
 * buffer synchronization in cases where that could cause issues (e.g., during
 * panics/stackshots and going down for sleep).
 */
static bool console_suspended = false;

/**
 * Controls console output for underlying serial or video console.
 * To be used only by core console and init accessors.
 */
int disableConsoleOutput;

/**
 * Enforce policies around when console I/O is allowed. Most importantly about
 * not performing console I/O while interrupts are disabled (which can cause
 * serious latency issues).
 *
 * @return True if console I/O should be allowed, false otherwise.
 */
static inline bool
console_io_allowed(void)
{
	if (!allow_printf_from_interrupts_disabled_context &&
	    !console_suspended &&
	    startup_phase >= STARTUP_SUB_EARLY_BOOT &&
	    !ml_get_interrupts_enabled()) {
#if defined(__arm64__) || DEBUG || DEVELOPMENT
		panic("Console I/O from interrupt-disabled context");
#else
		return false;
#endif
	}

	return true;
}

/**
 * Initialize the console ring buffer and console lock. It's still possible to
 * call console_write() before initializing the ring buffer. In that case the
 * data will get outputted directly to the underlying serial/video console
 * without synchronization.
 *
 * This function is also safe to call multiple times. Any call after the first
 * will return early without doing anything.
 */
void
console_init(void)
{
	if (console_ring.len != 0) {
		return;
	}

	kmem_alloc(kernel_map, (vm_offset_t *)&console_ring.buffer,
	    KERN_CONSOLE_RING_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT |
	    KMA_KOBJECT | KMA_PERMANENT | KMA_GUARD_FIRST | KMA_GUARD_LAST |
	    KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_OSFMK);

	console_ring.buffer   += PAGE_SIZE; /* Skip past the first guard page. */
	console_ring.len       = KERN_CONSOLE_RING_SIZE;
	console_ring.used      = 0;
	console_ring.nreserved = 0;
	console_ring.read_ptr  = console_ring.buffer;
	console_ring.write_ptr = console_ring.buffer;

	lck_mtx_init(&console_ring.flush_lock, &console_lck_grp, LCK_ATTR_NULL);
	lck_ticket_init(&console_ring.write_lock, &console_lck_grp);
}

/**
 * Returns true when the console has already been initialized.
 */
static inline bool
is_console_initialized(void)
{
	return console_ring.len == KERN_CONSOLE_RING_SIZE;
}

/**
 * Return the index to the currently selected console (serial/video). This is
 * an index into the "cons_ops[]" array of function pointer structs.
 */
static inline uint32_t
get_cons_ops_index(void)
{
	uint32_t idx = cons_ops_index;

	if (idx >= nconsops) {
		panic("Bad cons_ops_index: %d", idx);
	}

	return idx;
}

/**
 * Helper function for outputting a character to the underlying console
 * (either video or serial) with the possibility of sleeping waiting for
 * an interrupt indicating the console is ready.
 *
 * @note assumes console_ring.read lock is held if poll == false
 *
 * @param c The character to print.
 * @param poll Whether or not this call should poll instead of going to sleep
 *             waiting for an interrupt when the hardware device isn't ready
 */
static inline void
_cnputc(char c, bool poll)
{
	bool in_debugger = (kernel_debugger_entry_count > 0);
	const uint32_t idx = get_cons_ops_index();

	poll = poll || in_debugger;

	if (c == '\n') {
		_cnputc('\r', poll);
	}

	cons_ops[idx].putc(c, poll);
}

/**
 * Helper function for outputting characters directly to the underlying console
 * (either video or serial).
 *
 * @note disableConsoleOutput is to be used only by core console and init accessors
 *       such as this function. Returns early if the serial output is disabled.
 *
 * @param c The array of characters to print.
 * @param poll Whether or not this call should poll instead of going to sleep
 *             waiting for an interrupt when the hardware device isn't ready
 * @param size The number of characters to print to the console.
 */
static inline void
_cnputs(char *c, int size, bool poll)
{
	if (disableConsoleOutput) {
		return;
	}

	assert(c != NULL);

	while (size-- > 0) {
		_cnputc(*c, poll);
		c++;
	}
}

/**
 * Attempt to reserve space for a number of characters in the console ring
 * buffer. Space in the ring buffer must be reserved before new characters can
 * be entered.
 *
 * Every call to this function should be paired with a corresponding call to
 * console_ring_unreserve_space().
 *
 * @note If space is successfully reserved, this will disable preemption because
 *       otherwise, console_ring_try_empty() could take arbitrarily long.
 *
 * @param nchars The number of characters to reserve.
 *
 * @return If the wanted number of characters could not be reserved, then return
 *         NULL. Otherwise, return a pointer to the beginning of the reserved
 *         space.
 */
static inline char*
console_ring_reserve_space(int nchars)
{
	char *write_ptr = NULL;
	lck_ticket_lock(&console_ring.write_lock, &console_lck_grp);
	if ((console_ring.len - console_ring.used) >= nchars) {
		console_ring.used += nchars;
		mp_disable_preemption();
		os_atomic_inc(&console_ring.nreserved, relaxed);

		/* Return out the pointer to the beginning of the just reserved space. */
		write_ptr = console_ring.write_ptr;

		/* Move the console ring's write pointer to the beginning of the next free space. */
		const ptrdiff_t write_index = console_ring.write_ptr - console_ring.buffer;
		console_ring.write_ptr = console_ring.buffer + ((write_index + nchars) % console_ring.len);
	}
	lck_ticket_unlock(&console_ring.write_lock);
	return write_ptr;
}

/**
 * Decrement the number of reserved spaces in the console ring (now that the data
 * has been written) and re-enable preemption.
 *
 * Every call to this function should be paired with a corresponding call to
 * console_ring_reserve_space().
 */
static inline void
console_ring_unreserve_space(void)
{
	assert(console_ring.nreserved > 0);

	os_atomic_dec(&console_ring.nreserved, relaxed);
	mp_enable_preemption();
}

/**
 * Write a single character into the console ring buffer and handle moving the
 * write pointer circularly around the buffer.
 *
 * @note Space to write this character must have already been reserved using
 *       console_ring_reserve_space().
 *
 * @param write_ptr Pointer into the reserved space in the buffer to write the
 *                  character. This pointer will get moved to the next valid
 *                  location to write a character so the same pointer can be
 *                  passed into subsequent calls to write multiple characters.
 * @param ch The character to insert into the ring buffer.
 */
static inline void
console_ring_put(char **write_ptr, char ch)
{
	assert(console_ring.nreserved > 0);
	**write_ptr = ch;
	++(*write_ptr);
	if ((*write_ptr - console_ring.buffer) == console_ring.len) {
		*write_ptr = console_ring.buffer;
	}
}

/**
 * Attempt to drain the console ring buffer if no other CPUs are already doing
 * so.
 *
 * @param fail_fast If true, this function returns immediately instead of
 *                  sleeping if the thread fails to acquire the console flush
 *                  mutex.
 *
 * @note This function should not be called with preemption disabled.
 *
 * @note To prevent one CPU from holding the console lock for too long, only
 *       MAX_FLUSH_SIZE_LOCK_HELD number of characters can be drained at a time
 *       with the lock held. The lock will be dropped between each drain of size
 *       MAX_FLUSH_SIZE_LOCK_HELD to allow another CPU to grab the lock. If
 *       another CPU grabs the lock, then the original thread can stop draining
 *       and return instead of sleeping for the lock.
 *
 * @note To prevent one thread from being the drain thread for too long (presumably
 *       that thread has other things it wants to do besides draining serial), the
 *       total number of characters a single call to this function can drain is
 *       restricted to MAX_TOTAL_FLUSH_SIZE.
 */
static void
console_ring_try_empty(bool fail_fast)
{
	char flush_buf[MAX_FLUSH_SIZE_LOCK_HELD];

	int nchars_out       = 0;
	int total_chars_out  = 0;
	int size_before_wrap = 0;
	bool in_debugger = (kernel_debugger_entry_count > 0);

	if (__improbable(!console_io_allowed()) || (!in_debugger && get_preemption_level() != 0)) {
		return;
	}

	do {
		if (__probable(!in_debugger) && fail_fast && !lck_mtx_try_lock(&console_ring.flush_lock)) {
			return;
		} else if (__probable(!in_debugger) && !fail_fast) {
			lck_mtx_lock(&console_ring.flush_lock);
		}

		if (__probable(!in_debugger)) {
			lck_ticket_lock(&console_ring.write_lock, &console_lck_grp);

			/**
			 * If we've managed to grab the write lock, but there's still space
			 * reserved in the buffer, then other CPUs are actively writing into
			 * the ring, wait for them to finish.
			 */
			while (os_atomic_load(&console_ring.nreserved, relaxed) > 0) {
				cpu_pause();
			}
		}

		/* Try small chunk at a time, so we allow writes from other cpus into the buffer. */
		nchars_out = MIN(console_ring.used, (int)sizeof(flush_buf));

		/* Account for data to be read before wrap around. */
		size_before_wrap = (int)((console_ring.buffer + console_ring.len) - console_ring.read_ptr);
		if (nchars_out > size_before_wrap) {
			nchars_out = size_before_wrap;
		}

		/**
		 * Copy the characters to be drained into a separate flush buffer, and
		 * move the console read pointer to the next chunk of data that needs to
		 * be drained.
		 */
		if (nchars_out > 0) {
			memcpy(flush_buf, console_ring.read_ptr, nchars_out);
			const ptrdiff_t read_index = console_ring.read_ptr - console_ring.buffer;
			console_ring.read_ptr = console_ring.buffer + ((read_index + nchars_out) % console_ring.len);
			console_ring.used -= nchars_out;
		}

		if (__probable(!in_debugger)) {
			lck_ticket_unlock(&console_ring.write_lock);
		}

		/**
		 * Output characters to the underlying console (serial/video). We should
		 * only poll if the console is suspended.
		 */
		if (nchars_out > 0) {
			total_chars_out += nchars_out;
			_cnputs(flush_buf, nchars_out, console_suspended);
		}

		if (__probable(!in_debugger)) {
			lck_mtx_unlock(&console_ring.flush_lock);
		}

		/**
		 * Prevent this thread from sleeping on the lock again if another thread
		 * grabs it after we drop it.
		 */
		fail_fast = true;

		/*
		 * In case we end up being the console drain thread for far too long,
		 * break out. Except in panic/suspend cases where we should clear out
		 * the full buffer.
		 */
		if (!console_suspended && (total_chars_out >= MAX_TOTAL_FLUSH_SIZE)) {
			break;
		}
	} while (nchars_out > 0);
}

/**
 * Notify the console subystem that all following console writes should skip
 * synchronization and get outputted directly to the underlying console. This is
 * important for cases like panic/stackshots and going down for sleep where
 * assumptions about the state of the system could cause hangs or nested panics.
 */
void
console_suspend()
{
	console_suspended = true;
	console_ring_try_empty(false);
}

/**
 * Notify the console subsystem that it is now safe to use the console ring
 * buffer synchronization when writing console data.
 */
void
console_resume()
{
	console_suspended = false;
}

/**
 * Write a string of characters to the underlying video or serial console in a
 * synchronized manner. By synchronizing access to a global console buffer, this
 * prevents the serial output from appearing interleaved to the end user when
 * multiple CPUs are outputting to the console at the same time.
 *
 * @note It's safe to call this function even before the console buffer has been
 *       initialized. In that case, the data will be sent directly to the
 *       underlying console with no buffering. This is the same for when the
 *       console is suspended.
 *
 * @note disableConsoleOutput is to be used only by core console and init accessors
 *       such as this function. Returns early if the serial output is disabled and
 *       skips lock acquisition.
 *
 * @param str The string of characters to print.
 * @param size The number of characters in `str` to print.
 */
void
console_write(char *str, int size)
{
	if (disableConsoleOutput) {
		return;
	}

	assert(str != NULL);

	char *write_ptr = NULL;
	int chunk_size = CPU_CONS_BUF_SIZE;
	int i = 0;

	if (__improbable(console_suspended || !is_console_initialized() || pmap_in_ppl())) {
		/*
		 * Output directly to console in the following cases:
		 * 1. If this is early in boot before the console has been initialized.
		 * 2. If we're heading into suspend.
		 * 3. If we're in the kernel debugger for a panic/stackshot. If any of
		 *    the other cores happened to halt while holding any of the console
		 *    locks, attempting to use the normal path will result in sadness.
		 * 4. If we're in the PPL. As we synchronize the ring buffer with a
		 *    mutex and preemption is disabled in the PPL, any writes must go
		 *    directly to the hardware device.
		 */
		_cnputs(str, size, true);
		return;
	} else if (__improbable(!console_io_allowed())) {
		return;
	}

	while (size > 0) {
		/**
		 * Restrict the maximum number of characters that can be reserved at
		 * once. This helps prevent one CPU from reserving too much and starving
		 * out the other CPUs.
		 */
		if (size < chunk_size) {
			chunk_size = size;
		}

		/**
		 * Attempt to reserve space in the ring buffer and if that fails, then
		 * keep attempting to drain the ring buffer until there's enough space.
		 * We can't flush the serial console with preemption disabled so return
		 * early to drop the message in that case.
		 */
		while ((write_ptr = console_ring_reserve_space(chunk_size)) == NULL) {
			if (get_preemption_level() != 0) {
				return;
			}

			console_ring_try_empty(false);
		}

		for (i = 0; i < chunk_size; i++) {
			console_ring_put(&write_ptr, str[i]);
		}

		console_ring_unreserve_space();
		str = &str[i];
		size -= chunk_size;
	}

	/* Do good faith flush if preemption is not disabled */
	if (get_preemption_level() == 0) {
		console_ring_try_empty(true);
	}
}

/**
 * Output a character directly to the underlying console (either video or serial).
 * This directly bypasses the console serial buffer (as provided by console_write())
 * and all of the synchronization that provides.
 *
 * @note This function can cause serial data to get printed interleaved if being
 *       called on multiple CPUs at the same time. Only use this function if
 *       there's a specific reason why this serial data can't get synchronized
 *       through the console buffer.
 *
 * @note disableConsoleOutput is to be used only by core console and init accessors
 *       such as this function. Returns early if the serial output is disabled.
 *
 * @param c The character to print.
 */
void
console_write_unbuffered(char c)
{
	if (disableConsoleOutput) {
		return;
	}

	_cnputc(c, true);
}

/**
 * Write a single character to the selected console (video or serial).
 *
 * @param c The character to print.
 */
void
console_write_char(char c)
{
	console_write(&c, 1);
}

/**
 * Wrapper around the platform-dependent serial input method which handles
 * waiting for a new character and checking for the NMI string.
 *
 * @param wait True if this function should block until a character appears.
 *
 * @return The character if one was read, -1 otherwise.
 */
int
_serial_getc(bool wait)
{
	int c = -1;

	do {
		c = serial_getc();
	} while (wait && c < 0);

	/* Check for the NMI string. */
	if (c == nmi_string[nmi_counter]) {
		nmi_counter++;
		if (nmi_counter == NMI_STRING_SIZE) {
			/* We've got the NMI string, now do an NMI. */
			Debugger("Automatic NMI");
			nmi_counter = 0;
			return '\n';
		}
	} else if (c != -1) {
		nmi_counter = 0;
	}

	return c;
}

/**
 * Typically the video console doesn't support input, but we call into the
 * pexpert to give each platform an opportunity to provide console input through
 * alternative methods if it so desires.
 *
 * Usually a platform will either not provide any input, or will grab input from
 * the serial driver.
 *
 * @return The character if one was read, or -1 otherwise.
 */
int
_vcgetc(__unused bool wait)
{
	char c;

	if (0 == PE_stub_poll_input(0, &c)) {
		return c;
	} else {
		return -1;
	}
}

/**
 * Block until a character is available from the console and return it.
 *
 * @return The character retrieved from the console.
 */
int
console_read_char(void)
{
	const uint32_t idx = get_cons_ops_index();
	return cons_ops[idx].getc(true);
}

/**
 * Attempt to read a character from the console, and if one isn't available,
 * then return immediately.
 *
 * @return The character if one is available, -1 otherwise.
 */
int
console_try_read_char(void)
{
	const uint32_t idx = get_cons_ops_index();
	return cons_ops[idx].getc(false);
}

#ifdef CONFIG_XNUPOST
static uint32_t cons_test_ops_count = 0;

/*
 * Log to console by multiple methods - printf, unbuffered write, console_write()
 */
static void
log_to_console_func(void * arg __unused, wait_result_t wres __unused)
{
	uint64_t thread_id = current_thread()->thread_id;
	char somedata[10] = "123456789";
	for (int i = 0; i < 26; i++) {
		os_atomic_inc(&cons_test_ops_count, relaxed);
		printf(" thid: %llu printf iteration %d\n", thread_id, i);
		console_write_unbuffered((char)('A' + i));
		console_write_unbuffered('\n');
		console_write((char *)somedata, sizeof(somedata));
		delay(10);
	}
	printf("finished the log_to_console_func operations\n\n");
}

/* Test that outputting to the console can occur on multiple threads at the same time. */
kern_return_t
console_serial_parallel_log_tests(void)
{
	thread_t thread;
	kern_return_t kr;
	cons_test_ops_count = 0;

	kr = kernel_thread_start(log_to_console_func, NULL, &thread);
	T_ASSERT_EQ_INT(kr, KERN_SUCCESS, "kernel_thread_start returned successfully");

	delay(100);

	log_to_console_func(NULL, 0);

	/* wait until other thread has also finished */
	while (cons_test_ops_count < 52) {
		delay(1000);
	}

	thread_deallocate(thread);
	T_LOG("parallel_logging tests is now complete. From this point forward we expect full lines\n");
	return KERN_SUCCESS;
}

/* Basic serial test that prints serial output through various methods (printf/T_LOG). */
kern_return_t
console_serial_test(void)
{
	unsigned long i;
	char buffer[CPU_CONS_BUF_SIZE];

	T_LOG("Checking console_ring status.");
	T_ASSERT_EQ_INT(console_ring.len, KERN_CONSOLE_RING_SIZE, "Console ring size is not correct.");

	/* setup buffer to be chars */
	for (i = 0; i < CPU_CONS_BUF_SIZE; i++) {
		buffer[i] = (char)('0' + (i % 10));
	}
	buffer[CPU_CONS_BUF_SIZE - 1] = '\0';

	T_LOG("Printing %d char string to serial one char at a time.", CPU_CONS_BUF_SIZE);
	for (i = 0; i < CPU_CONS_BUF_SIZE; i++) {
		printf("%c", buffer[i]);
	}
	printf("End\n");
	T_LOG("Printing %d char string to serial as a whole", CPU_CONS_BUF_SIZE);
	printf("%s\n", buffer);

	T_LOG("Using console_write call repeatedly for 100 iterations");
	for (i = 0; i < 100; i++) {
		console_write(&buffer[0], 14);
		if ((i % 6) == 0) {
			printf("\n");
		}
	}
	printf("\n");

	T_LOG("Using T_LOG to print buffer %s", buffer);
	return KERN_SUCCESS;
}
#endif