1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * This file contains the low-level serial drivers used on ARM/ARM64 devices.
5 * The generic serial console code in osfmk/console/serial_console.c will call
6 * into this code to transmit and receive serial data.
7 *
8 * Logging can be performed on multiple serial interfaces at once through a
9 * method called serial multiplexing. This is implemented by enumerating which
10 * serial interfaces are available on boot and registering them into a linked
11 * list of interfaces pointed to by gPESF. When outputting or receiving
12 * characters, each interface is queried in turn.
13 *
14 * Please view doc/arm_serial.md for an in-depth description of these drivers.
15 */
16 #include <kern/clock.h>
17 #include <kern/debug.h>
18 #include <libkern/OSBase.h>
19 #include <libkern/section_keywords.h>
20 #include <mach/mach_time.h>
21 #include <machine/atomic.h>
22 #include <machine/machine_routines.h>
23 #include <pexpert/pexpert.h>
24 #include <pexpert/protos.h>
25 #include <pexpert/device_tree.h>
26 #include <pexpert/arm/consistent_debug.h>
27 #include <pexpert/arm64/board_config.h>
28 #include <arm64/proc_reg.h>
29 #include <pexpert/arm/protos.h>
30 #include <kern/sched_prim.h>
31 #if HIBERNATION
32 #include <machine/pal_hibernate.h>
33 #endif /* HIBERNATION */
34
35 /**
36 * Fun and helpful icon to use for announcements regarding serial deprecation,
37 * disablement. This helps catch people's eye to get them to realize the message
38 * is something other than normal serial spew. Please don't remove. :)
39 */
40 const char hexley[] =
41 " \n"
42 " %( \n"
43 " #% /((( %(& #((( \n"
44 " *(((( #(((( &(((& %((((((((((&((((((((% \n"
45 " %(. (% *(( %((((((((((%((((((((((((# \n"
46 " *(% (( #(( (((@(%((((((((%((#((((((((. \n"
47 " &(& ((# ((& (( ((((((((((@(%((((((((((% \n"
48 " &((, ((% &((# & (((((/ %(%(((((((((((((((( \n"
49 " &((((&#,%((#&%((((/ @((& &((( (&(((((((#((((((( \n"
50 " %#((((((((%# *## (((((((((@ %@@((&((((((((&((& \n"
51 " ((/ &##&######%( @&((((((((((((((% \n"
52 " ((& ###########&#((((&(((((((((& \n"
53 " #(% /##########%%%(((((((&/ \n"
54 " &(( &%///&@,##&(((((((( \n"
55 " (((((( .#&/##@((( ((((% \n"
56 " #(((%(((, ((((*.....*(((((& (((((% \n"
57 " %(&&(((((((((#.......((((((((( &#%% \n"
58 " ((# (((((((@......../(%(((((((( \n"
59 " %(( (((((@..........((((@(((((( \n"
60 " &(( ............(((((((((, \n"
61 " #((# /............(((((# \n"
62 " ((& .............(((((( \n"
63 " ((# ((...........#((((((% \n"
64 " ((( @(((((.......#(((((((( .@&&@/ \n"
65 " %((. @(((((((#.@((((((((#(@((((((((((#(@ \n"
66 " &((&@##@@((((((((&((((((((@#(((((((((&. \n"
67 " *((& ##&( /##(&#((#(& \n"
68 " %##@&### \n\n";
69
70 struct pe_serial_functions {
71 /* Initialize the underlying serial hardware. */
72 void (*init) (void);
73
74 /* Return a non-zero value if the serial interface is ready to send more data. */
75 unsigned int (*transmit_ready) (void);
76
77 /* Write a single byte of data to serial. */
78 void (*transmit_data) (uint8_t c);
79
80 /* Return a non-zero value if there's a byte of data available. */
81 unsigned int (*receive_ready) (void);
82
83 /* Read a single byte from serial. */
84 uint8_t (*receive_data) (void);
85
86 /* Enables IRQs from this device. */
87 void (*enable_irq) (void);
88
89 /* Disables IRQs from this device and reports whether IRQs were enabled. */
90 bool (*disable_irq) (void);
91
92 /* Clears this device's IRQs targeting this agent, returning true if at least one IRQ was cleared. */
93 bool (*acknowledge_irq) (void);
94
95 /**
96 * Whether this serial driver can handle irqs. This value should be set by
97 * querying the device tree to see if the serial device has interrupts
98 * associated with it.
99 *
100 * For a device to support IRQs:
101 * - enable_irq, disable_irq, and acknowledge_irq must be non-null
102 * - The AppleSerialShim kext must be able to match to the serial device
103 * in the IORegistry and call serial_enable_irq with the proper
104 * serial_device_t
105 * - The device tree entry for the serial device should have an interrupt
106 * associated with it.
107 */
108 bool has_irq;
109
110 /* enum identifying which serial device these functions belong to. */
111 serial_device_t device;
112
113 /* Pointer to the next serial interface in the linked-list. */
114 struct pe_serial_functions *next;
115 };
116
117 MARK_AS_HIBERNATE_DATA_CONST_LATE static struct pe_serial_functions* gPESF = NULL;
118
119 /**
120 * Whether uart has been initialized already. This value is kept across a
121 * sleep/wake cycle so we know we need to reinitialize when serial_init is
122 * called again after wake.
123 */
124 MARK_AS_HIBERNATE_DATA static bool uart_initted = false;
125
126 /* Whether uart should run in simple mode that works during hibernation resume. */
127 MARK_AS_HIBERNATE_DATA static bool uart_hibernation = false;
128
129 /** Set <=> transmission is authorized.
130 * Always set, unless SERIALMODE_ON_DEMAND is provided at boot,
131 * and no data has yet been received.
132 * Originaly meant to be a per-pe_serial_functions variable,
133 * but the data protection on the structs prevents it. */
134 static bool serial_do_transmit = 1;
135
136 /**
137 * Used to track if all IRQs have been initialized. Each bit of this variable
138 * represents whether or not a serial device that reports supporting IRQs has
139 * been initialized yet (1 -> not initialized, 0 -> initialized)
140 */
141 static uint32_t serial_irq_status = 0;
142
143 /**
144 * Set by the 'disable-uart-irq' boot-arg to force serial IRQs into polling mode
145 * by preventing the serial driver shim kext from registering itself with
146 * serial_enable_irq.
147 */
148 static bool disable_uart_irq = 0;
149
150 /**
151 * Indicates whether or not a given device's irqs have been set up by calling
152 * serial_enable_irq for that particular device.
153 *
154 * @param device_fns Serial functions for the device that is being checked
155 * @return Whether or not the irqs have been initialized for that device
156 */
157 static bool
irq_initialized(struct pe_serial_functions * device_fns)158 irq_initialized(struct pe_serial_functions *device_fns)
159 {
160 return (serial_irq_status & device_fns->device) == 0;
161 }
162
163 /**
164 * Indicates whether or not a given device supports irqs and if they are ready
165 * to be used.
166 *
167 * @param device_fns Serial functions for the device that is being checked
168 * @return Whether or not the device can and will send IRQs.
169 */
170 static bool
irq_available_and_ready(struct pe_serial_functions * device_fns)171 irq_available_and_ready(struct pe_serial_functions *device_fns)
172 {
173 return device_fns->has_irq && irq_initialized(device_fns);
174 }
175
176 /**
177 * Searches through the global serial functions list and returns the serial function for a particular device
178 *
179 * @param device The device identifier to search for
180 * @return Serial functions for the specified device
181 */
182 static struct pe_serial_functions *
get_serial_functions(serial_device_t device)183 get_serial_functions(serial_device_t device)
184 {
185 struct pe_serial_functions *fns = gPESF;
186 while (fns != NULL) {
187 if (fns->device == device) {
188 return fns;
189 }
190 fns = fns->next;
191 }
192 return NULL;
193 }
194
195 /**
196 * The action to take when polling and waiting for a serial device to be ready
197 * for output. On ARM64, takes a WFE because the WFE timeout will wake us up in
198 * the worst case. On ARMv7 devices, we need to hot poll.
199 */
200 static inline void
serial_poll(void)201 serial_poll(void)
202 {
203 #if __arm64__
204 if (!uart_hibernation) {
205 __builtin_arm_wfe();
206 }
207 #endif
208 }
209
210 /**
211 * This ensures that if we have a future product that supports hibernation, but
212 * doesn't support either UART serial or dock-channels, then hibernation will
213 * gracefully fall back to the serial method that is supported.
214 */
215 #if HIBERNATION || defined(APPLE_UART)
216 MARK_AS_HIBERNATE_DATA static volatile apple_uart_registers_t *apple_uart_registers = 0;
217 #endif /* HIBERNATION || defined(APPLE_UART) */
218
219 #if HIBERNATION || defined(DOCKCHANNEL_UART)
220 MARK_AS_HIBERNATE_DATA static vm_offset_t dockchannel_uart_base = 0;
221 #endif /* HIBERNATION || defined(DOCKCHANNEL_UART) */
222
223 /*****************************************************************************/
224
225 #ifdef APPLE_UART
226 static int32_t dt_sampling = -1;
227 static int32_t dt_ubrdiv = -1;
228
229 static void apple_uart_set_baud_rate(uint32_t baud_rate);
230
231 /**
232 * The Apple UART is configured to use 115200-8-N-1 communication.
233 */
234 static void
apple_uart_init(void)235 apple_uart_init(void)
236 {
237 ucon_t ucon = { .raw = 0 };
238 // Use NCLK (which is constant) instead of PCLK (which is variable).
239 ucon.clock_selection = UCON_CLOCK_SELECTION_NCLK;
240 ucon.transmit_mode = UCON_TRANSMIT_MODE_INTERRUPT_OR_POLLING;
241 ucon.receive_mode = UCON_RECEIVE_MODE_INTERRUPT_OR_POLLING;
242 apple_uart_registers->ucon = ucon;
243
244 // Configure 8-N-1 communication.
245 ulcon_t ulcon = { .raw = 0 };
246 ulcon.word_length = ULCON_WORD_LENGTH_8_BITS;
247 ulcon.parity_mode = ULCON_PARITY_MODE_NONE;
248 ulcon.number_of_stop_bits = ULCON_STOP_BITS_1;
249 apple_uart_registers->ulcon = ulcon;
250
251 apple_uart_set_baud_rate(115200);
252
253 // Enable and reset FIFOs.
254 ufcon_t ufcon = { .raw = 0 };
255 ufcon.fifo_enable = 1;
256 ufcon.tx_fifo_reset = 1;
257 ufcon.rx_fifo_reset = 1;
258 apple_uart_registers->ufcon = ufcon;
259 }
260
261 static void
apple_uart_enable_irq(void)262 apple_uart_enable_irq(void)
263 {
264 // Set the Tx FIFO interrupt trigger level to 0 bytes so interrupts occur when
265 // the Tx FIFO is completely empty; this leads to higher Tx throughput.
266 apple_uart_registers->ufcon.tx_fifo_interrupt_trigger_level_dma_watermark = UFCON_TX_FIFO_ITL_0_BYTES;
267
268 // Enable Tx interrupts.
269 apple_uart_registers->ucon.transmit_interrupt = 1;
270 }
271
272 static bool
apple_uart_disable_irq(void)273 apple_uart_disable_irq(void)
274 {
275 /* Disables Tx interrupts */
276 ucon_t ucon = apple_uart_registers->ucon;
277 const bool irqs_were_enabled = ucon.transmit_interrupt;
278
279 if (irqs_were_enabled) {
280 ucon.transmit_interrupt = 0;
281 apple_uart_registers->ucon = ucon;
282 }
283
284 return irqs_were_enabled;
285 }
286
287 static bool
apple_uart_ack_irq(void)288 apple_uart_ack_irq(void)
289 {
290 apple_uart_registers->utrstat.transmit_interrupt_status = 1;
291 return true;
292 }
293
294 static inline bool
apple_uart_fifo_is_empty(void)295 apple_uart_fifo_is_empty(void)
296 {
297 const ufstat_t ufstat = apple_uart_registers->ufstat;
298 return !(ufstat.tx_fifo_full || ufstat.tx_fifo_count);
299 }
300
301 static void
apple_uart_drain_fifo(void)302 apple_uart_drain_fifo(void)
303 {
304 while (!apple_uart_fifo_is_empty()) {
305 serial_poll();
306 }
307 }
308
309 static void
apple_uart_set_baud_rate(uint32_t baud_rate)310 apple_uart_set_baud_rate(uint32_t baud_rate)
311 {
312 uint32_t div = 0;
313 const uint32_t uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz;
314 uint32_t sample_rate = 16;
315
316 if (baud_rate < 300) {
317 baud_rate = 9600;
318 }
319
320 if (dt_sampling != -1) {
321 // Use the sampling rate specified in the Device Tree
322 sample_rate = dt_sampling & 0xf;
323 }
324
325 if (dt_ubrdiv != -1) {
326 // Use the ubrdiv specified in the Device Tree
327 div = dt_ubrdiv & 0xffff;
328 } else {
329 // Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1
330 div = uart_clock / (baud_rate * sample_rate);
331
332 uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate);
333 uint32_t baud_low = uart_clock / ((div + 1) * sample_rate);
334
335 // Adjust div to get the closest target baudrate
336 if ((baud_rate - baud_low) > (actual_baud - baud_rate)) {
337 div--;
338 }
339 }
340
341 ubrdiv_t ubrdiv = apple_uart_registers->ubrdiv;
342 ubrdiv.sample_rate = 16 - sample_rate;
343 ubrdiv.ubr_div = div;
344 apple_uart_registers->ubrdiv = ubrdiv;
345 }
346
347 MARK_AS_HIBERNATE_TEXT static unsigned int
apple_uart_transmit_ready(void)348 apple_uart_transmit_ready(void)
349 {
350 return !apple_uart_registers->ufstat.tx_fifo_full;
351 }
352
353 MARK_AS_HIBERNATE_TEXT static void
apple_uart_transmit_data(uint8_t c)354 apple_uart_transmit_data(uint8_t c)
355 {
356 apple_uart_registers->utxh.txdata = c;
357 }
358
359 static unsigned int
apple_uart_receive_ready(void)360 apple_uart_receive_ready(void)
361 {
362 const ufstat_t ufstat = apple_uart_registers->ufstat;
363 return ufstat.rx_fifo_full || ufstat.rx_fifo_count;
364 }
365
366 static uint8_t
apple_uart_receive_data(void)367 apple_uart_receive_data(void)
368 {
369 return apple_uart_registers->urxh.rxdata;
370 }
371
372 MARK_AS_HIBERNATE_DATA_CONST_LATE
373 static struct pe_serial_functions apple_serial_functions =
374 {
375 .init = apple_uart_init,
376 .transmit_ready = apple_uart_transmit_ready,
377 .transmit_data = apple_uart_transmit_data,
378 .receive_ready = apple_uart_receive_ready,
379 .receive_data = apple_uart_receive_data,
380 .enable_irq = apple_uart_enable_irq,
381 .disable_irq = apple_uart_disable_irq,
382 .acknowledge_irq = apple_uart_ack_irq,
383 .device = SERIAL_APPLE_UART
384 };
385
386 #endif /* APPLE_UART */
387
388 /*****************************************************************************/
389
390 #ifdef DOCKCHANNEL_UART
391 #define DOCKCHANNEL_WR_MAX_STALL_US (30*1000)
392
393 static vm_offset_t dock_agent_base;
394 static uint32_t max_dockchannel_drain_period;
395 static uint64_t dockchannel_drain_deadline; // Deadline for external agent to drain before a software drain occurs
396 static bool use_sw_drain;
397 static uint32_t dock_wstat_mask;
398 static uint64_t prev_dockchannel_spaces; // Previous w_stat level of the DockChannel.
399 static uint64_t dockchannel_stall_grace;
400 MARK_AS_HIBERNATE_DATA static bool use_sw_drain;
401 MARK_AS_HIBERNATE_DATA static uint32_t dock_wstat_mask;
402
403 // forward reference
404 static struct pe_serial_functions dockchannel_serial_functions;
405
406 //=======================
407 // Local funtions
408 //=======================
409
410 static int
dockchannel_drain_on_stall()411 dockchannel_drain_on_stall()
412 {
413 // Called when DockChannel runs out of spaces.
414 // Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves.
415 // Return number of bytes drained.
416
417 if (mach_absolute_time() >= dockchannel_drain_deadline) {
418 // It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO
419 // Drop a character.
420 (void)rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
421 os_atomic_inc(&prev_dockchannel_spaces, relaxed);
422 return 1;
423 }
424 return 0;
425 }
426
427 static void
dockchannel_clear_intr(void)428 dockchannel_clear_intr(void)
429 {
430 rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3);
431 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3;
432 rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3);
433 rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3;
434 }
435
436 static bool
dockchannel_disable_irq(void)437 dockchannel_disable_irq(void)
438 {
439 const uint32_t ap_intr_ctrl = rDOCKCHANNELS_AGENT_AP_INTR_CTRL;
440 const bool irqs_were_enabled = ap_intr_ctrl & 0x1;
441 if (irqs_were_enabled) {
442 rDOCKCHANNELS_AGENT_AP_INTR_CTRL = ap_intr_ctrl & ~(0x1);
443 }
444 return irqs_were_enabled;
445 }
446
447 static void
dockchannel_enable_irq(void)448 dockchannel_enable_irq(void)
449 {
450 // set interrupt to be when fifo has 255 empty
451 rDOCKCHANNELS_DEV_WR_WATERMARK(DOCKCHANNEL_UART_CHANNEL) = 0xFF;
452 rDOCKCHANNELS_AGENT_AP_INTR_CTRL |= 0x1;
453 }
454
455 static bool
dockchannel_ack_irq(void)456 dockchannel_ack_irq(void)
457 {
458 /* First check if the IRQ is for the kernel */
459 if (rDOCKCHANNELS_AGENT_AP_INTR_STATUS & 0x1) {
460 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x1;
461 return true;
462 }
463 return false;
464 }
465
466 MARK_AS_HIBERNATE_TEXT static void
dockchannel_transmit_data(uint8_t c)467 dockchannel_transmit_data(uint8_t c)
468 {
469 rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)c;
470
471 if (use_sw_drain && !uart_hibernation) {
472 os_atomic_dec(&prev_dockchannel_spaces, relaxed); // After writing a byte we have one fewer space than previously expected.
473 }
474 }
475
476 static unsigned int
dockchannel_receive_ready(void)477 dockchannel_receive_ready(void)
478 {
479 return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f;
480 }
481
482 static uint8_t
dockchannel_receive_data(void)483 dockchannel_receive_data(void)
484 {
485 return (uint8_t)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL) >> 8) & 0xff);
486 }
487
488 MARK_AS_HIBERNATE_TEXT static unsigned int
dockchannel_transmit_ready(void)489 dockchannel_transmit_ready(void)
490 {
491 uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
492
493 if (!uart_hibernation) {
494 if (use_sw_drain) {
495 if (spaces > prev_dockchannel_spaces) {
496 // More spaces showed up. That can only mean someone read the FIFO.
497 // Note that if the DockFIFO is empty we cannot tell if someone is listening,
498 // we can only give them the benefit of the doubt.
499 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
500 }
501 prev_dockchannel_spaces = spaces;
502 return spaces || dockchannel_drain_on_stall();
503 }
504 }
505
506 return spaces;
507 }
508
509 static void
dockchannel_init(void)510 dockchannel_init(void)
511 {
512 if (use_sw_drain) {
513 nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace);
514 }
515
516 // Clear all interrupt enable and status bits
517 dockchannel_clear_intr();
518
519 // Setup DRAIN timer
520 rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period;
521
522 // Drain timer doesn't get loaded with value from drain period register if fifo
523 // is already full. Drop a character from the fifo.
524 rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
525 }
526
527 MARK_AS_HIBERNATE_DATA_CONST_LATE
528 static struct pe_serial_functions dockchannel_serial_functions =
529 {
530 .init = dockchannel_init,
531 .transmit_ready = dockchannel_transmit_ready,
532 .transmit_data = dockchannel_transmit_data,
533 .receive_ready = dockchannel_receive_ready,
534 .receive_data = dockchannel_receive_data,
535 .enable_irq = dockchannel_enable_irq,
536 .disable_irq = dockchannel_disable_irq,
537 .acknowledge_irq = dockchannel_ack_irq,
538 .device = SERIAL_DOCKCHANNEL
539 };
540
541 #endif /* DOCKCHANNEL_UART */
542
543 /****************************************************************************/
544 #ifdef PI3_UART
545 vm_offset_t pi3_gpio_base_vaddr = 0;
546 vm_offset_t pi3_aux_base_vaddr = 0;
547 static unsigned int
pi3_uart_tr0(void)548 pi3_uart_tr0(void)
549 {
550 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x20;
551 }
552
553 static void
pi3_uart_td0(uint8_t c)554 pi3_uart_td0(uint8_t c)
555 {
556 BCM2837_PUT32(BCM2837_AUX_MU_IO_REG_V, (uint32_t) c);
557 }
558
559 static unsigned int
pi3_uart_rr0(void)560 pi3_uart_rr0(void)
561 {
562 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x01;
563 }
564
565 static uint8_t
pi3_uart_rd0(void)566 pi3_uart_rd0(void)
567 {
568 return (uint8_t) BCM2837_GET32(BCM2837_AUX_MU_IO_REG_V);
569 }
570
571 static void
pi3_uart_init(void)572 pi3_uart_init(void)
573 {
574 // Scratch variable
575 uint32_t i;
576
577 // Reset mini uart registers
578 BCM2837_PUT32(BCM2837_AUX_ENABLES_V, 1);
579 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 0);
580 BCM2837_PUT32(BCM2837_AUX_MU_LCR_REG_V, 3);
581 BCM2837_PUT32(BCM2837_AUX_MU_MCR_REG_V, 0);
582 BCM2837_PUT32(BCM2837_AUX_MU_IER_REG_V, 0);
583 BCM2837_PUT32(BCM2837_AUX_MU_IIR_REG_V, 0xC6);
584 BCM2837_PUT32(BCM2837_AUX_MU_BAUD_REG_V, 270);
585
586 i = (uint32_t)BCM2837_FSEL_REG(14);
587 // Configure GPIOs 14 & 15 for alternate function 5
588 i &= ~(BCM2837_FSEL_MASK(14));
589 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(14));
590 i &= ~(BCM2837_FSEL_MASK(15));
591 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(15));
592
593 BCM2837_PUT32(BCM2837_FSEL_REG(14), i);
594
595 BCM2837_PUT32(BCM2837_GPPUD_V, 0);
596
597 // Barrier before AP spinning for 150 cycles
598 __builtin_arm_isb(ISB_SY);
599
600 for (i = 0; i < 150; i++) {
601 asm volatile ("add x0, x0, xzr");
602 }
603
604 __builtin_arm_isb(ISB_SY);
605
606 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, (1 << 14) | (1 << 15));
607
608 __builtin_arm_isb(ISB_SY);
609
610 for (i = 0; i < 150; i++) {
611 asm volatile ("add x0, x0, xzr");
612 }
613
614 __builtin_arm_isb(ISB_SY);
615
616 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, 0);
617
618 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 3);
619 }
620
621 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) pi3_uart_serial_functions =
622 {
623 .init = pi3_uart_init,
624 .transmit_ready = pi3_uart_tr0,
625 .transmit_data = pi3_uart_td0,
626 .receive_ready = pi3_uart_rr0,
627 .receive_data = pi3_uart_rd0,
628 .device = SERIAL_PI3_UART
629 };
630
631 #endif /* PI3_UART */
632
633 /*****************************************************************************/
634
635 #ifdef VMAPPLE_UART
636
637 static vm_offset_t vmapple_uart0_base_vaddr = 0;
638
639 #define PL011_LCR_WORD_LENGTH_8 0x60u
640 #define PL011_LCR_FIFO_DISABLE 0x00u
641
642 #define PL011_LCR_FIFO_ENABLE 0x10u
643
644 #define PL011_LCR_ONE_STOP_BIT 0x00u
645 #define PL011_LCR_PARITY_DISABLE 0x00u
646 #define PL011_LCR_BREAK_DISABLE 0x00u
647 #define PL011_IBRD_DIV_38400 0x27u
648 #define PL011_FBRD_DIV_38400 0x09u
649 #define PL011_ICR_CLR_ALL_IRQS 0x07ffu
650 #define PL011_CR_UART_ENABLE 0x01u
651 #define PL011_CR_TX_ENABLE 0x100u
652 #define PL011_CR_RX_ENABLE 0x200u
653
654 #define VMAPPLE_UART0_DR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x00))
655 #define VMAPPLE_UART0_ECR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x04))
656 #define VMAPPLE_UART0_FR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x18))
657 #define VMAPPLE_UART0_IBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x24))
658 #define VMAPPLE_UART0_FBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x28))
659 #define VMAPPLE_UART0_LCR_H *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x2c))
660 #define VMAPPLE_UART0_CR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x30))
661 #define VMAPPLE_UART0_TIMSC *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x38))
662 #define VMAPPLE_UART0_ICR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x44))
663
664 static unsigned int
vmapple_uart_transmit_ready(void)665 vmapple_uart_transmit_ready(void)
666 {
667 return (unsigned int) !(VMAPPLE_UART0_FR & 0x20);
668 }
669
670 static void
vmapple_uart_transmit_data(uint8_t c)671 vmapple_uart_transmit_data(uint8_t c)
672 {
673 VMAPPLE_UART0_DR = (uint32_t) c;
674 }
675
676 static unsigned int
vmapple_uart_receive_ready(void)677 vmapple_uart_receive_ready(void)
678 {
679 return (unsigned int) !(VMAPPLE_UART0_FR & 0x10);
680 }
681
682 static uint8_t
vmapple_uart_receive_data(void)683 vmapple_uart_receive_data(void)
684 {
685 return (uint8_t) (VMAPPLE_UART0_DR & 0xff);
686 }
687
688 static void
vmapple_uart_init(void)689 vmapple_uart_init(void)
690 {
691 VMAPPLE_UART0_CR = 0x0;
692 VMAPPLE_UART0_ECR = 0x0;
693 VMAPPLE_UART0_LCR_H = (
694 PL011_LCR_WORD_LENGTH_8 |
695 PL011_LCR_FIFO_ENABLE |
696 PL011_LCR_ONE_STOP_BIT |
697 PL011_LCR_PARITY_DISABLE |
698 PL011_LCR_BREAK_DISABLE
699 );
700 VMAPPLE_UART0_IBRD = PL011_IBRD_DIV_38400;
701 VMAPPLE_UART0_FBRD = PL011_FBRD_DIV_38400;
702 VMAPPLE_UART0_TIMSC = 0x0;
703 VMAPPLE_UART0_ICR = PL011_ICR_CLR_ALL_IRQS;
704 VMAPPLE_UART0_CR = (
705 PL011_CR_UART_ENABLE |
706 PL011_CR_TX_ENABLE |
707 PL011_CR_RX_ENABLE
708 );
709 }
710
711 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) vmapple_uart_serial_functions =
712 {
713 .init = vmapple_uart_init,
714 .transmit_ready = vmapple_uart_transmit_ready,
715 .transmit_data = vmapple_uart_transmit_data,
716 .receive_ready = vmapple_uart_receive_ready,
717 .receive_data = vmapple_uart_receive_data,
718 .device = SERIAL_VMAPPLE_UART
719 };
720
721 #endif /* VMAPPLE_UART */
722
723 /*****************************************************************************/
724
725 /**
726 * Output @str onto every registered serial interface by polling.
727 *
728 * @param str The string to output.
729 */
730 static void uart_puts_force_poll(
731 const char *str);
732
733 /**
734 * Output @str onto a specific serial interface by polling.
735 *
736 * @param str The string to output.
737 * @param fns The functions to use to output the message.
738 */
739 static void uart_puts_force_poll_device(
740 const char *str,
741 struct pe_serial_functions *fns);
742
743 static void
register_serial_functions(struct pe_serial_functions * fns)744 register_serial_functions(struct pe_serial_functions *fns)
745 {
746 fns->next = gPESF;
747 gPESF = fns;
748 }
749
750 #if HIBERNATION
751 /**
752 * Transitions the serial driver into a mode that can be run in the hibernation
753 * resume context. In this mode, the serial driver runs at a barebones level
754 * without making sure the serial devices are properly initialized or utilizing
755 * features such as the software drain timer for dockchannels.
756 *
757 * Upon the next call to serial_init (once the hibernation image has been
758 * loaded), this mode is exited and we return to the normal operation of the
759 * driver.
760 */
761 MARK_AS_HIBERNATE_TEXT void
serial_hibernation_init(void)762 serial_hibernation_init(void)
763 {
764 uart_hibernation = true;
765 #if defined(APPLE_UART)
766 apple_uart_registers = (apple_uart_registers_t *)gHibernateGlobals.hibUartRegPhysBase;
767 #endif /* defined(APPLE_UART) */
768 #if defined(DOCKCHANNEL_UART)
769 dockchannel_uart_base = gHibernateGlobals.dockChannelRegPhysBase;
770 #endif /* defined(DOCKCHANNEL_UART) */
771 }
772
773 /**
774 * Transitions the serial driver back to non-hibernation mode so it can resume
775 * normal operations. Should only be called from serial_init on a hibernation
776 * resume.
777 */
778 MARK_AS_HIBERNATE_TEXT static void
serial_hibernation_cleanup(void)779 serial_hibernation_cleanup(void)
780 {
781 uart_hibernation = false;
782 #if defined(APPLE_UART)
783 apple_uart_registers = (apple_uart_registers_t *)gHibernateGlobals.hibUartRegVirtBase;
784 #endif /* defined(APPLE_UART) */
785 #if defined(DOCKCHANNEL_UART)
786 dockchannel_uart_base = gHibernateGlobals.dockChannelRegVirtBase;
787 #endif /* defined(DOCKCHANNEL_UART) */
788 }
789 #endif /* HIBERNATION */
790
791 int
serial_init(void)792 serial_init(void)
793 {
794 DTEntry entryP = NULL;
795 uint32_t prop_size;
796 vm_offset_t soc_base;
797 uintptr_t const *reg_prop;
798 uint32_t const *prop_value __unused = NULL;
799
800 struct pe_serial_functions *fns = gPESF;
801
802 /**
803 * Even if the serial devices have already been initialized on cold boot,
804 * when coming out of a sleep/wake, they'll need to be re-initialized. Since
805 * the uart_initted value is kept across a sleep/wake, always re-initialize
806 * to be safe.
807 */
808 if (uart_initted) {
809 #if HIBERNATION
810 if (uart_hibernation) {
811 serial_hibernation_cleanup();
812 }
813 #endif /* HIBERNATION */
814 while (fns != NULL) {
815 fns->init();
816 fns = fns->next;
817 }
818
819 return gPESF != NULL;
820 }
821
822 soc_base = pe_arm_get_soc_base_phys();
823
824 if (soc_base == 0) {
825 uart_initted = true;
826 return 0;
827 }
828
829 PE_parse_boot_argn("disable-uart-irq", &disable_uart_irq, sizeof(disable_uart_irq));
830
831 #ifdef PI3_UART
832 if (SecureDTFindEntry("name", "gpio", &entryP) == kSuccess) {
833 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
834 pi3_gpio_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
835 }
836 if (SecureDTFindEntry("name", "aux", &entryP) == kSuccess) {
837 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
838 pi3_aux_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
839 }
840 if ((pi3_gpio_base_vaddr != 0) && (pi3_aux_base_vaddr != 0)) {
841 register_serial_functions(&pi3_uart_serial_functions);
842 }
843 #endif /* PI3_UART */
844
845 #ifdef VMAPPLE_UART
846 if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
847 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
848 vmapple_uart0_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
849 }
850
851 if (vmapple_uart0_base_vaddr != 0) {
852 register_serial_functions(&vmapple_uart_serial_functions);
853 }
854 #endif /* VMAPPLE_UART */
855
856 #ifdef DOCKCHANNEL_UART
857 uint32_t no_dockchannel_uart = 0;
858 if (SecureDTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) {
859 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
860 // Should be two reg entries
861 if (prop_size / sizeof(uintptr_t) != 4) {
862 panic("Malformed dockchannel-uart property");
863 }
864 dockchannel_uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
865 dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3));
866 PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart));
867 // Keep the old name for boot-arg
868 if (no_dockchannel_uart == 0) {
869 register_serial_functions(&dockchannel_serial_functions);
870 SecureDTGetProperty(entryP, "max-aop-clk", (void const **)&prop_value, &prop_size);
871 max_dockchannel_drain_period = (uint32_t)((prop_value)? (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD);
872 prop_value = NULL;
873 SecureDTGetProperty(entryP, "enable-sw-drain", (void const **)&prop_value, &prop_size);
874 use_sw_drain = (prop_value)? *prop_value : 0;
875 prop_value = NULL;
876 SecureDTGetProperty(entryP, "dock-wstat-mask", (void const **)&prop_value, &prop_size);
877 dock_wstat_mask = (prop_value)? *prop_value : 0x1ff;
878 prop_value = NULL;
879 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
880 if (prop_value) {
881 dockchannel_serial_functions.has_irq = true;
882 }
883
884 /* Set sane defaults for dockchannel globals. */
885 prev_dockchannel_spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
886 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
887 } else {
888 dockchannel_clear_intr();
889 }
890 // If no dockchannel-uart is found in the device tree, fall back
891 // to looking for the traditional UART serial console.
892 }
893
894 #endif /* DOCKCHANNEL_UART */
895
896 #ifdef APPLE_UART
897 char const *serial_compat = 0;
898 uint32_t use_legacy_uart = 1;
899
900 /*
901 * The boot serial port should have a property named "boot-console".
902 * If we don't find it there, look for "uart0" and "uart1".
903 */
904 if (SecureDTFindEntry("boot-console", NULL, &entryP) == kSuccess) {
905 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
906 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
907 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
908 } else if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
909 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
910 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
911 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
912 } else if (SecureDTFindEntry("name", "uart1", &entryP) == kSuccess) {
913 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
914 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
915 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
916 }
917
918 if (NULL != entryP) {
919 prop_value = NULL;
920 SecureDTGetProperty(entryP, "sampling", (void const **)&prop_value, &prop_size);
921 if (prop_value) {
922 dt_sampling = *prop_value;
923 }
924
925 prop_value = NULL;
926 SecureDTGetProperty(entryP, "ubrdiv", (void const **)&prop_value, &prop_size);
927 if (prop_value) {
928 dt_ubrdiv = *prop_value;
929 }
930
931 prop_value = NULL;
932 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
933 if (prop_value) {
934 apple_serial_functions.has_irq = true;
935 }
936
937 prop_value = NULL;
938 SecureDTGetProperty(entryP, "disable-legacy-uart", (void const **)&prop_value, &prop_size);
939 if (prop_value) {
940 use_legacy_uart = 0;
941 }
942 }
943
944 /* Check if we should enable this deprecated serial device. */
945 PE_parse_boot_argn("use-legacy-uart", &use_legacy_uart, sizeof(use_legacy_uart));
946
947 if (serial_compat && !strcmp(serial_compat, "uart-1,samsung")) {
948 if (use_legacy_uart) {
949 register_serial_functions(&apple_serial_functions);
950 } else {
951 char legacy_serial_msg[] =
952 "Expecting more output? Wondering why Clippy turned into a platypus?\n\n"
953 "UART was manually disabled either via the 'use-legacy-uart=0' boot-arg or via\n"
954 "the disable-legacy-uart property in the boot-console uart device tree node.\n"
955 "Serial output over dockchannels is still enabled on devices with support.\n";
956 apple_serial_functions.init();
957 uart_puts_force_poll_device(hexley, &apple_serial_functions);
958 uart_puts_force_poll_device(legacy_serial_msg, &apple_serial_functions);
959 }
960 }
961 #endif /* APPLE_UART */
962
963 fns = gPESF;
964 while (fns != NULL) {
965 serial_do_transmit = 1;
966 fns->init();
967 if (fns->has_irq) {
968 serial_irq_status |= fns->device; // serial_device_t is one-hot
969 }
970 fns = fns->next;
971 }
972
973 #if HIBERNATION
974 /* hibernation needs to know the UART register addresses since it can't directly use this serial driver */
975 if (dockchannel_uart_base) {
976 gHibernateGlobals.dockChannelRegPhysBase = ml_vtophys(dockchannel_uart_base);
977 gHibernateGlobals.dockChannelRegVirtBase = dockchannel_uart_base;
978 gHibernateGlobals.dockChannelWstatMask = dock_wstat_mask;
979 }
980 if (apple_uart_registers) {
981 gHibernateGlobals.hibUartRegPhysBase = ml_vtophys((vm_offset_t)apple_uart_registers);
982 gHibernateGlobals.hibUartRegVirtBase = (vm_offset_t)apple_uart_registers;
983 }
984 #endif /* HIBERNATION */
985
986 /* Complete. */
987 uart_initted = true;
988 return gPESF != NULL;
989 }
990
991 /**
992 * Forbid or allow transmission over each serial until they receive data.
993 */
994 void
serial_set_on_demand(bool on_demand)995 serial_set_on_demand(bool on_demand)
996 {
997 /* Enable or disable transmission. */
998 serial_do_transmit = !on_demand;
999
1000 /* If on-demand is enabled, report it. */
1001 if (on_demand) {
1002 uart_puts_force_poll(
1003 "On-demand serial mode selected.\n"
1004 "Waiting for user input to send logs.\n"
1005 );
1006 }
1007 }
1008
1009 /**
1010 * Returns a deadline for the longest time the serial driver should wait for an
1011 * interrupt for. This serves as a timeout for the IRQ to allow for the software
1012 * drain timer that dockchannels supports.
1013 *
1014 * @param fns serial functions representing the device to find the deadline for
1015 *
1016 * @returns absolutetime deadline for this device's IRQ.
1017 */
1018 static uint64_t
serial_interrupt_deadline(__unused struct pe_serial_functions * fns)1019 serial_interrupt_deadline(__unused struct pe_serial_functions *fns)
1020 {
1021 #if defined(DOCKCHANNEL_UART)
1022 if (fns->device == SERIAL_DOCKCHANNEL && use_sw_drain) {
1023 return dockchannel_drain_deadline;
1024 }
1025 #endif
1026
1027 /**
1028 * Default to 1.5ms for all other devices. 1.5ms was chosen as the baudrate
1029 * of the AppleSerialDevice is 115200, meaning that it should only take
1030 * ~1.5ms to drain the 16 character buffer completely.
1031 */
1032 uint64_t timeout_interval;
1033 nanoseconds_to_absolutetime(1500 * NSEC_PER_USEC, &timeout_interval);
1034 return mach_absolute_time() + timeout_interval;
1035 }
1036
1037 /**
1038 * Goes to sleep waiting for an interrupt from a specificed serial device.
1039 *
1040 * @param fns serial functions representing the device to wait for
1041 */
1042 static void
serial_wait_for_interrupt(struct pe_serial_functions * fns)1043 serial_wait_for_interrupt(struct pe_serial_functions *fns)
1044 {
1045 /**
1046 * This block of code is set up to avoid a race condition in which the IRQ
1047 * is transmitted and processed by IOKit in between the time we check if the
1048 * device is ready to transmit and when we call thread_block. If the IRQ
1049 * fires in that time, thread_wakeup may have already been called in which
1050 * case we would be blocking and have nothing to wake us up.
1051 *
1052 * To avoid this issue, we first call assert_wait_deadline, which prepares
1053 * the thread to be blocked, but does not actually block the thread. After
1054 * this point, any call to thread_wakeup from IRQ handler will prevent
1055 * thread_block from actually blocking. As a performance optimization, we
1056 * then double check if the device is ready to transmit and if it is, then
1057 * we cancel the wait and just continue normally.
1058 */
1059 assert_wait_deadline(fns, THREAD_UNINT, serial_interrupt_deadline(fns));
1060 if (!fns->transmit_ready()) {
1061 fns->enable_irq();
1062 thread_block(THREAD_CONTINUE_NULL);
1063 } else {
1064 clear_wait(current_thread(), THREAD_AWAKENED);
1065 }
1066 }
1067
1068 /**
1069 * Transmit a character over the specified serial output device.
1070 *
1071 * @param c Character to send
1072 * @param poll Whether we should poll or wait for an interrupt.
1073 * @param force Whether we should force this over the device if output has not been enabled yet.
1074 * @param fns Functions for the device to output over.
1075 */
1076 static inline void
uart_putc_device(char c,bool poll,bool force,struct pe_serial_functions * fns)1077 uart_putc_device(char c, bool poll, bool force, struct pe_serial_functions *fns)
1078 {
1079 if (!(serial_do_transmit || force)) {
1080 return;
1081 }
1082
1083 while (!fns->transmit_ready()) {
1084 if (irq_available_and_ready(fns) && !poll) {
1085 serial_wait_for_interrupt(fns);
1086 } else {
1087 serial_poll();
1088 }
1089 }
1090 fns->transmit_data((uint8_t)c);
1091 }
1092
1093 /**
1094 * Output a character onto every registered serial interface whose
1095 * transmission is enabled..
1096 *
1097 * @param c The character to output.
1098 * @param poll Whether the driver should poll to send the character or if it can
1099 * wait for an interrupt
1100 */
1101 MARK_AS_HIBERNATE_TEXT void
uart_putc_options(char c,bool poll)1102 uart_putc_options(char c, bool poll)
1103 {
1104 struct pe_serial_functions *fns = gPESF;
1105
1106 while (fns != NULL) {
1107 uart_putc_device(c, poll, false, fns);
1108 fns = fns->next;
1109 }
1110 }
1111
1112 /**
1113 * Output a character onto every registered serial interface whose
1114 * transmission is enabled by polling.
1115 *
1116 * @param c The character to output.
1117 */
1118 void
uart_putc(char c)1119 uart_putc(char c)
1120 {
1121 uart_putc_options(c, true);
1122 }
1123
1124 /**
1125 * Output @str onto every registered serial interface by polling.
1126 *
1127 * @param str The string to output.
1128 */
1129 static void
uart_puts_force_poll(const char * str)1130 uart_puts_force_poll(
1131 const char *str)
1132 {
1133 struct pe_serial_functions *fns = gPESF;
1134 while (fns != NULL) {
1135 uart_puts_force_poll_device(str, fns);
1136 fns = fns->next;
1137 }
1138 }
1139
1140 /**
1141 * Output @str onto a specific serial interface by polling.
1142 *
1143 * @param str The string to output.
1144 * @param fns The functions to use to output the message.
1145 */
1146 static void
uart_puts_force_poll_device(const char * str,struct pe_serial_functions * fns)1147 uart_puts_force_poll_device(
1148 const char *str,
1149 struct pe_serial_functions *fns)
1150 {
1151 char c;
1152 while ((c = *(str++))) {
1153 uart_putc_device(c, true, true, fns);
1154 }
1155 }
1156
1157 /**
1158 * Read a character from the first registered serial interface that has data
1159 * available.
1160 *
1161 * @return The character if any interfaces have data available, otherwise -1.
1162 */
1163 int
uart_getc(void)1164 uart_getc(void)
1165 {
1166 struct pe_serial_functions *fns = gPESF;
1167 while (fns != NULL) {
1168 if (fns->receive_ready()) {
1169 serial_do_transmit = 1;
1170 return (int)fns->receive_data();
1171 }
1172 fns = fns->next;
1173 }
1174 return -1;
1175 }
1176
1177 /**
1178 * Enables IRQs for a specific serial device and returns whether or not IRQs for
1179 * that device where enabled successfully. For a serial driver to have irqs
1180 * enabled, it must have the enable_irq, disable_irq, and acknowledge_irq
1181 * functions defined and the has_irq flag set.
1182 *
1183 * @param device Serial device to enable irqs on
1184 * @note This function should only be called from the AppleSerialShim kext
1185 */
1186 kern_return_t
serial_irq_enable(serial_device_t device)1187 serial_irq_enable(serial_device_t device)
1188 {
1189 struct pe_serial_functions *fns = get_serial_functions(device);
1190
1191 if (!fns || !fns->has_irq || disable_uart_irq) {
1192 return KERN_FAILURE;
1193 }
1194
1195 serial_irq_status &= ~device;
1196
1197 return KERN_SUCCESS;
1198 }
1199
1200 /**
1201 * Performs any actions needed to handle this IRQ. Wakes up the thread waiting
1202 * on the interrupt if one exists.
1203 *
1204 * @param device Serial device that generated the IRQ.
1205 * @note Interrupts will have already been cleared and disabled by serial_irq_filter.
1206 * @note This function should only be called from the AppleSerialShim kext.
1207 */
1208 kern_return_t
serial_irq_action(serial_device_t device)1209 serial_irq_action(serial_device_t device)
1210 {
1211 struct pe_serial_functions *fns = get_serial_functions(device);
1212
1213 if (!fns || !fns->has_irq) {
1214 return KERN_FAILURE;
1215 }
1216
1217 /**
1218 * Because IRQs are enabled only when we know a thread is about to sleep, we
1219 * can call wake up and reasonably expect there to be a thread waiting.
1220 */
1221 thread_wakeup(fns);
1222
1223 return KERN_SUCCESS;
1224 }
1225
1226 /**
1227 * Returns true if the pending IRQ for device is one that can be handled by the
1228 * platform serial driver.
1229 *
1230 * @param device Serial device that generated the IRQ.
1231 * @note This function is called from a primary interrupt context and should be
1232 * kept lightweight.
1233 * @note This function should only be called from the AppleSerialShim kext
1234 */
1235 bool
serial_irq_filter(serial_device_t device)1236 serial_irq_filter(serial_device_t device)
1237 {
1238 struct pe_serial_functions *fns = get_serial_functions(device);
1239
1240 if (!fns || !fns->has_irq) {
1241 return false;
1242 }
1243
1244 /**
1245 * Disable IRQs until next time a thread waits for an interrupt to prevent an interrupt storm.
1246 */
1247 const bool had_irqs_enabled = fns->disable_irq();
1248 const bool was_our_interrupt = fns->acknowledge_irq();
1249
1250 /* Re-enable IRQs if the interrupt wasn't for us. */
1251 if (had_irqs_enabled && !was_our_interrupt) {
1252 fns->enable_irq();
1253 }
1254
1255 return was_our_interrupt;
1256 }
1257
1258 /**
1259 * Prepares all serial devices to go to sleep by draining the hardware FIFOs
1260 * and disabling interrupts.
1261 */
1262 void
serial_go_to_sleep(void)1263 serial_go_to_sleep(void)
1264 {
1265 struct pe_serial_functions *fns = gPESF;
1266 while (fns != NULL) {
1267 if (irq_available_and_ready(fns)) {
1268 fns->disable_irq();
1269 }
1270 fns = fns->next;
1271 }
1272
1273 #ifdef APPLE_UART
1274 /* APPLE_UART needs to drain FIFO before sleeping */
1275 if (get_serial_functions(SERIAL_APPLE_UART)) {
1276 apple_uart_drain_fifo();
1277 }
1278 #endif /* APPLE_UART */
1279 }
1280