1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * This file contains the low-level serial drivers used on ARM/ARM64 devices.
5 * The generic serial console code in osfmk/console/serial_console.c will call
6 * into this code to transmit and receive serial data.
7 *
8 * Logging can be performed on multiple serial interfaces at once through a
9 * method called serial multiplexing. This is implemented by enumerating which
10 * serial interfaces are available on boot and registering them into a linked
11 * list of interfaces pointed to by gPESF. When outputting or receiving
12 * characters, each interface is queried in turn.
13 *
14 * Please view doc/arm_serial.md for an in-depth description of these drivers.
15 */
16 #include <kern/clock.h>
17 #include <kern/debug.h>
18 #include <libkern/OSBase.h>
19 #include <libkern/section_keywords.h>
20 #include <mach/mach_time.h>
21 #include <machine/atomic.h>
22 #include <machine/machine_routines.h>
23 #include <pexpert/pexpert.h>
24 #include <pexpert/protos.h>
25 #include <pexpert/device_tree.h>
26 #include <pexpert/arm/consistent_debug.h>
27 #include <pexpert/arm64/board_config.h>
28 #include <arm64/proc_reg.h>
29 #include <pexpert/arm/protos.h>
30 #include <kern/sched_prim.h>
31 #if HIBERNATION
32 #include <machine/pal_hibernate.h>
33 #endif /* HIBERNATION */
34
35 struct pe_serial_functions {
36 /* Initialize the underlying serial hardware. */
37 void (*init) (void);
38
39 /* Return a non-zero value if the serial interface is ready to send more data. */
40 unsigned int (*transmit_ready) (void);
41
42 /* Write a single byte of data to serial. */
43 void (*transmit_data) (uint8_t c);
44
45 /* Return a non-zero value if there's a byte of data available. */
46 unsigned int (*receive_ready) (void);
47
48 /* Read a single byte from serial. */
49 uint8_t (*receive_data) (void);
50
51 /* Enables IRQs from this device. */
52 void (*enable_irq) (void);
53
54 /* Disables IRQs from this device and reports whether IRQs were enabled. */
55 bool (*disable_irq) (void);
56
57 /* Clears this device's IRQs targeting this agent, returning true if at least one IRQ was cleared. */
58 bool (*acknowledge_irq) (void);
59
60 /**
61 * Whether this serial driver can handle irqs. This value should be set by
62 * querying the device tree to see if the serial device has interrupts
63 * associated with it.
64 *
65 * For a device to support IRQs:
66 * - enable_irq, disable_irq, and acknowledge_irq must be non-null
67 * - The AppleSerialShim kext must be able to match to the serial device
68 * in the IORegistry and call serial_enable_irq with the proper
69 * serial_device_t
70 * - The device tree entry for the serial device should have an interrupt
71 * associated with it.
72 */
73 bool has_irq;
74
75 /* enum identifying which serial device these functions belong to. */
76 serial_device_t device;
77
78 /* Pointer to the next serial interface in the linked-list. */
79 struct pe_serial_functions *next;
80 };
81
82 MARK_AS_HIBERNATE_DATA_CONST_LATE static struct pe_serial_functions* gPESF = NULL;
83
84 /**
85 * Whether uart has been initialized already. This value is kept across a
86 * sleep/wake cycle so we know we need to reinitialize when serial_init is
87 * called again after wake.
88 */
89 MARK_AS_HIBERNATE_DATA static bool uart_initted = false;
90
91 /* Whether uart should run in simple mode that works during hibernation resume. */
92 MARK_AS_HIBERNATE_DATA static bool uart_hibernation = false;
93
94 /** Set <=> transmission is authorized.
95 * Always set, unless SERIALMODE_ON_DEMAND is provided at boot,
96 * and no data has yet been received.
97 * Originaly meant to be a per-pe_serial_functions variable,
98 * but the data protection on the structs prevents it. */
99 static bool serial_do_transmit = 1;
100
101 /**
102 * Used to track if all IRQs have been initialized. Each bit of this variable
103 * represents whether or not a serial device that reports supporting IRQs has
104 * been initialized yet (1 -> not initialized, 0 -> initialized)
105 */
106 static uint32_t serial_irq_status = 0;
107
108 /**
109 * Set by the 'disable-uart-irq' boot-arg to force serial IRQs into polling mode
110 * by preventing the serial driver shim kext from registering itself with
111 * serial_enable_irq.
112 */
113 static bool disable_uart_irq = 0;
114
115 /**
116 * Indicates whether or not a given device's irqs have been set up by calling
117 * serial_enable_irq for that particular device.
118 *
119 * @param device_fns Serial functions for the device that is being checked
120 * @return Whether or not the irqs have been initialized for that device
121 */
122 static bool
irq_initialized(struct pe_serial_functions * device_fns)123 irq_initialized(struct pe_serial_functions *device_fns)
124 {
125 return (serial_irq_status & device_fns->device) == 0;
126 }
127
128 /**
129 * Indicates whether or not a given device supports irqs and if they are ready
130 * to be used.
131 *
132 * @param device_fns Serial functions for the device that is being checked
133 * @return Whether or not the device can and will send IRQs.
134 */
135 static bool
irq_available_and_ready(struct pe_serial_functions * device_fns)136 irq_available_and_ready(struct pe_serial_functions *device_fns)
137 {
138 return device_fns->has_irq && irq_initialized(device_fns);
139 }
140
141 /**
142 * Searches through the global serial functions list and returns the serial function for a particular device
143 *
144 * @param device The device identifier to search for
145 * @return Serial functions for the specified device
146 */
147 static struct pe_serial_functions *
get_serial_functions(serial_device_t device)148 get_serial_functions(serial_device_t device)
149 {
150 struct pe_serial_functions *fns = gPESF;
151 while (fns != NULL) {
152 if (fns->device == device) {
153 return fns;
154 }
155 fns = fns->next;
156 }
157 return NULL;
158 }
159
160 /**
161 * The action to take when polling and waiting for a serial device to be ready
162 * for output. On ARM64, takes a WFE because the WFE timeout will wake us up in
163 * the worst case. On ARMv7 devices, we need to hot poll.
164 */
165 static void
serial_poll(void)166 serial_poll(void)
167 {
168 #if __arm64__
169 __builtin_arm_wfe();
170 #endif
171 }
172
173 /**
174 * This ensures that if we have a future product that supports hibernation, but
175 * doesn't support either UART serial or dock-channels, then hibernation will
176 * gracefully fall back to the serial method that is supported.
177 */
178 #if HIBERNATION || defined(APPLE_UART)
179 MARK_AS_HIBERNATE_DATA static volatile apple_uart_registers_t *apple_uart_registers = 0;
180 #endif /* HIBERNATION || defined(APPLE_UART) */
181
182 #if HIBERNATION || defined(DOCKCHANNEL_UART)
183 MARK_AS_HIBERNATE_DATA static vm_offset_t dockchannel_uart_base = 0;
184 #endif /* HIBERNATION || defined(DOCKCHANNEL_UART) */
185
186 /*****************************************************************************/
187
188 #ifdef APPLE_UART
189 static int32_t dt_sampling = -1;
190 static int32_t dt_ubrdiv = -1;
191
192 static void apple_uart_set_baud_rate(uint32_t baud_rate);
193
194 /**
195 * The Apple UART is configured to use 115200-8-N-1 communication.
196 */
197 static void
apple_uart_init(void)198 apple_uart_init(void)
199 {
200 ucon_t ucon = { .raw = 0 };
201 // Use NCLK (which is constant) instead of PCLK (which is variable).
202 ucon.clock_selection = UCON_CLOCK_SELECTION_NCLK;
203 ucon.transmit_mode = UCON_TRANSMIT_MODE_INTERRUPT_OR_POLLING;
204 ucon.receive_mode = UCON_RECEIVE_MODE_INTERRUPT_OR_POLLING;
205 apple_uart_registers->ucon = ucon;
206
207 // Configure 8-N-1 communication.
208 ulcon_t ulcon = { .raw = 0 };
209 ulcon.word_length = ULCON_WORD_LENGTH_8_BITS;
210 ulcon.parity_mode = ULCON_PARITY_MODE_NONE;
211 ulcon.number_of_stop_bits = ULCON_STOP_BITS_1;
212 apple_uart_registers->ulcon = ulcon;
213
214 apple_uart_set_baud_rate(115200);
215
216 // Enable and reset FIFOs.
217 ufcon_t ufcon = { .raw = 0 };
218 ufcon.fifo_enable = 1;
219 ufcon.tx_fifo_reset = 1;
220 ufcon.rx_fifo_reset = 1;
221 apple_uart_registers->ufcon = ufcon;
222 }
223
224 static void
apple_uart_enable_irq(void)225 apple_uart_enable_irq(void)
226 {
227 // Set the Tx FIFO interrupt trigger level to 0 bytes so interrupts occur when
228 // the Tx FIFO is completely empty; this leads to higher Tx throughput.
229 apple_uart_registers->ufcon.tx_fifo_interrupt_trigger_level_dma_watermark = UFCON_TX_FIFO_ITL_0_BYTES;
230
231 // Enable Tx interrupts.
232 apple_uart_registers->ucon.transmit_interrupt = 1;
233 }
234
235 static bool
apple_uart_disable_irq(void)236 apple_uart_disable_irq(void)
237 {
238 /* Disables Tx interrupts */
239 ucon_t ucon = apple_uart_registers->ucon;
240 const bool irqs_were_enabled = ucon.transmit_interrupt;
241
242 if (irqs_were_enabled) {
243 ucon.transmit_interrupt = 0;
244 apple_uart_registers->ucon = ucon;
245 }
246
247 return irqs_were_enabled;
248 }
249
250 static bool
apple_uart_ack_irq(void)251 apple_uart_ack_irq(void)
252 {
253 apple_uart_registers->utrstat.transmit_interrupt_status = 1;
254 return true;
255 }
256
257 static void
apple_uart_drain_fifo(void)258 apple_uart_drain_fifo(void)
259 {
260 while (({
261 ufstat_t ufstat = apple_uart_registers->ufstat;
262 ufstat.tx_fifo_full || ufstat.tx_fifo_count;
263 })) {
264 serial_poll();
265 }
266 }
267
268 static void
apple_uart_set_baud_rate(uint32_t baud_rate)269 apple_uart_set_baud_rate(uint32_t baud_rate)
270 {
271 uint32_t div = 0;
272 const uint32_t uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz;
273 uint32_t sample_rate = 16;
274
275 if (baud_rate < 300) {
276 baud_rate = 9600;
277 }
278
279 if (dt_sampling != -1) {
280 // Use the sampling rate specified in the Device Tree
281 sample_rate = dt_sampling & 0xf;
282 }
283
284 if (dt_ubrdiv != -1) {
285 // Use the ubrdiv specified in the Device Tree
286 div = dt_ubrdiv & 0xffff;
287 } else {
288 // Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1
289 div = uart_clock / (baud_rate * sample_rate);
290
291 uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate);
292 uint32_t baud_low = uart_clock / ((div + 1) * sample_rate);
293
294 // Adjust div to get the closest target baudrate
295 if ((baud_rate - baud_low) > (actual_baud - baud_rate)) {
296 div--;
297 }
298 }
299
300 ubrdiv_t ubrdiv = apple_uart_registers->ubrdiv;
301 ubrdiv.sample_rate = 16 - sample_rate;
302 ubrdiv.ubr_div = div;
303 apple_uart_registers->ubrdiv = ubrdiv;
304 }
305
306 MARK_AS_HIBERNATE_TEXT static unsigned int
apple_uart_transmit_ready(void)307 apple_uart_transmit_ready(void)
308 {
309 return !apple_uart_registers->ufstat.tx_fifo_full;
310 }
311
312 MARK_AS_HIBERNATE_TEXT static void
apple_uart_transmit_data(uint8_t c)313 apple_uart_transmit_data(uint8_t c)
314 {
315 apple_uart_registers->utxh.txdata = c;
316 }
317
318 static unsigned int
apple_uart_receive_ready(void)319 apple_uart_receive_ready(void)
320 {
321 const ufstat_t ufstat = apple_uart_registers->ufstat;
322 return ufstat.rx_fifo_full || ufstat.rx_fifo_count;
323 }
324
325 static uint8_t
apple_uart_receive_data(void)326 apple_uart_receive_data(void)
327 {
328 return apple_uart_registers->urxh.rxdata;
329 }
330
331 MARK_AS_HIBERNATE_DATA_CONST_LATE
332 static struct pe_serial_functions apple_serial_functions =
333 {
334 .init = apple_uart_init,
335 .transmit_ready = apple_uart_transmit_ready,
336 .transmit_data = apple_uart_transmit_data,
337 .receive_ready = apple_uart_receive_ready,
338 .receive_data = apple_uart_receive_data,
339 .enable_irq = apple_uart_enable_irq,
340 .disable_irq = apple_uart_disable_irq,
341 .acknowledge_irq = apple_uart_ack_irq,
342 .device = SERIAL_APPLE_UART
343 };
344
345 #endif /* APPLE_UART */
346
347 /*****************************************************************************/
348
349 #ifdef DOCKCHANNEL_UART
350 #define DOCKCHANNEL_WR_MAX_STALL_US (30*1000)
351
352 static vm_offset_t dock_agent_base;
353 static uint32_t max_dockchannel_drain_period;
354 static uint64_t dockchannel_drain_deadline; // Deadline for external agent to drain before a software drain occurs
355 static bool use_sw_drain;
356 static uint32_t dock_wstat_mask;
357 static uint64_t prev_dockchannel_spaces; // Previous w_stat level of the DockChannel.
358 static uint64_t dockchannel_stall_grace;
359 MARK_AS_HIBERNATE_DATA static bool use_sw_drain;
360 MARK_AS_HIBERNATE_DATA static uint32_t dock_wstat_mask;
361
362 // forward reference
363 static struct pe_serial_functions dockchannel_serial_functions;
364
365 //=======================
366 // Local funtions
367 //=======================
368
369 static int
dockchannel_drain_on_stall()370 dockchannel_drain_on_stall()
371 {
372 // Called when DockChannel runs out of spaces.
373 // Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves.
374 // Return number of bytes drained.
375
376 if (mach_absolute_time() >= dockchannel_drain_deadline) {
377 // It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO
378 // Drop a character.
379 (void)rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
380 os_atomic_inc(&prev_dockchannel_spaces, relaxed);
381 return 1;
382 }
383 return 0;
384 }
385
386 static void
dockchannel_clear_intr(void)387 dockchannel_clear_intr(void)
388 {
389 rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3);
390 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3;
391 rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3);
392 rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3;
393 }
394
395 static bool
dockchannel_disable_irq(void)396 dockchannel_disable_irq(void)
397 {
398 const uint32_t ap_intr_ctrl = rDOCKCHANNELS_AGENT_AP_INTR_CTRL;
399 const bool irqs_were_enabled = ap_intr_ctrl & 0x1;
400 if (irqs_were_enabled) {
401 rDOCKCHANNELS_AGENT_AP_INTR_CTRL = ap_intr_ctrl & ~(0x1);
402 }
403 return irqs_were_enabled;
404 }
405
406 static void
dockchannel_enable_irq(void)407 dockchannel_enable_irq(void)
408 {
409 // set interrupt to be when fifo has 255 empty
410 rDOCKCHANNELS_DEV_WR_WATERMARK(DOCKCHANNEL_UART_CHANNEL) = 0xFF;
411 rDOCKCHANNELS_AGENT_AP_INTR_CTRL |= 0x1;
412 }
413
414 static bool
dockchannel_ack_irq(void)415 dockchannel_ack_irq(void)
416 {
417 /* First check if the IRQ is for the kernel */
418 if (rDOCKCHANNELS_AGENT_AP_INTR_STATUS & 0x1) {
419 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x1;
420 return true;
421 }
422 return false;
423 }
424
425 MARK_AS_HIBERNATE_TEXT static void
dockchannel_transmit_data(uint8_t c)426 dockchannel_transmit_data(uint8_t c)
427 {
428 rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)c;
429
430 if (use_sw_drain && !uart_hibernation) {
431 os_atomic_dec(&prev_dockchannel_spaces, relaxed); // After writing a byte we have one fewer space than previously expected.
432 }
433 }
434
435 static unsigned int
dockchannel_receive_ready(void)436 dockchannel_receive_ready(void)
437 {
438 return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f;
439 }
440
441 static uint8_t
dockchannel_receive_data(void)442 dockchannel_receive_data(void)
443 {
444 return (uint8_t)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL) >> 8) & 0xff);
445 }
446
447 MARK_AS_HIBERNATE_TEXT static unsigned int
dockchannel_transmit_ready(void)448 dockchannel_transmit_ready(void)
449 {
450 uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
451
452 if (!uart_hibernation) {
453 if (use_sw_drain) {
454 if (spaces > prev_dockchannel_spaces) {
455 // More spaces showed up. That can only mean someone read the FIFO.
456 // Note that if the DockFIFO is empty we cannot tell if someone is listening,
457 // we can only give them the benefit of the doubt.
458 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
459 }
460 prev_dockchannel_spaces = spaces;
461 return spaces || dockchannel_drain_on_stall();
462 }
463 }
464
465 return spaces;
466 }
467
468 static void
dockchannel_init(void)469 dockchannel_init(void)
470 {
471 if (use_sw_drain) {
472 nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace);
473 }
474
475 // Clear all interrupt enable and status bits
476 dockchannel_clear_intr();
477
478 // Setup DRAIN timer
479 rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period;
480
481 // Drain timer doesn't get loaded with value from drain period register if fifo
482 // is already full. Drop a character from the fifo.
483 rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
484 }
485
486 MARK_AS_HIBERNATE_DATA_CONST_LATE
487 static struct pe_serial_functions dockchannel_serial_functions =
488 {
489 .init = dockchannel_init,
490 .transmit_ready = dockchannel_transmit_ready,
491 .transmit_data = dockchannel_transmit_data,
492 .receive_ready = dockchannel_receive_ready,
493 .receive_data = dockchannel_receive_data,
494 .enable_irq = dockchannel_enable_irq,
495 .disable_irq = dockchannel_disable_irq,
496 .acknowledge_irq = dockchannel_ack_irq,
497 .device = SERIAL_DOCKCHANNEL
498 };
499
500 #endif /* DOCKCHANNEL_UART */
501
502 /****************************************************************************/
503 #ifdef PI3_UART
504 vm_offset_t pi3_gpio_base_vaddr = 0;
505 vm_offset_t pi3_aux_base_vaddr = 0;
506 static unsigned int
pi3_uart_tr0(void)507 pi3_uart_tr0(void)
508 {
509 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x20;
510 }
511
512 static void
pi3_uart_td0(uint8_t c)513 pi3_uart_td0(uint8_t c)
514 {
515 BCM2837_PUT32(BCM2837_AUX_MU_IO_REG_V, (uint32_t) c);
516 }
517
518 static unsigned int
pi3_uart_rr0(void)519 pi3_uart_rr0(void)
520 {
521 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x01;
522 }
523
524 static uint8_t
pi3_uart_rd0(void)525 pi3_uart_rd0(void)
526 {
527 return (uint8_t) BCM2837_GET32(BCM2837_AUX_MU_IO_REG_V);
528 }
529
530 static void
pi3_uart_init(void)531 pi3_uart_init(void)
532 {
533 // Scratch variable
534 uint32_t i;
535
536 // Reset mini uart registers
537 BCM2837_PUT32(BCM2837_AUX_ENABLES_V, 1);
538 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 0);
539 BCM2837_PUT32(BCM2837_AUX_MU_LCR_REG_V, 3);
540 BCM2837_PUT32(BCM2837_AUX_MU_MCR_REG_V, 0);
541 BCM2837_PUT32(BCM2837_AUX_MU_IER_REG_V, 0);
542 BCM2837_PUT32(BCM2837_AUX_MU_IIR_REG_V, 0xC6);
543 BCM2837_PUT32(BCM2837_AUX_MU_BAUD_REG_V, 270);
544
545 i = (uint32_t)BCM2837_FSEL_REG(14);
546 // Configure GPIOs 14 & 15 for alternate function 5
547 i &= ~(BCM2837_FSEL_MASK(14));
548 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(14));
549 i &= ~(BCM2837_FSEL_MASK(15));
550 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(15));
551
552 BCM2837_PUT32(BCM2837_FSEL_REG(14), i);
553
554 BCM2837_PUT32(BCM2837_GPPUD_V, 0);
555
556 // Barrier before AP spinning for 150 cycles
557 __builtin_arm_isb(ISB_SY);
558
559 for (i = 0; i < 150; i++) {
560 asm volatile ("add x0, x0, xzr");
561 }
562
563 __builtin_arm_isb(ISB_SY);
564
565 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, (1 << 14) | (1 << 15));
566
567 __builtin_arm_isb(ISB_SY);
568
569 for (i = 0; i < 150; i++) {
570 asm volatile ("add x0, x0, xzr");
571 }
572
573 __builtin_arm_isb(ISB_SY);
574
575 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, 0);
576
577 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 3);
578 }
579
580 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) pi3_uart_serial_functions =
581 {
582 .init = pi3_uart_init,
583 .transmit_ready = pi3_uart_tr0,
584 .transmit_data = pi3_uart_td0,
585 .receive_ready = pi3_uart_rr0,
586 .receive_data = pi3_uart_rd0,
587 .device = SERIAL_PI3_UART
588 };
589
590 #endif /* PI3_UART */
591
592 /*****************************************************************************/
593
594 #ifdef VMAPPLE_UART
595
596 static vm_offset_t vmapple_uart0_base_vaddr = 0;
597
598 #define PL011_LCR_WORD_LENGTH_8 0x60u
599 #define PL011_LCR_FIFO_DISABLE 0x00u
600
601 #define PL011_LCR_FIFO_ENABLE 0x10u
602
603 #define PL011_LCR_ONE_STOP_BIT 0x00u
604 #define PL011_LCR_PARITY_DISABLE 0x00u
605 #define PL011_LCR_BREAK_DISABLE 0x00u
606 #define PL011_IBRD_DIV_38400 0x27u
607 #define PL011_FBRD_DIV_38400 0x09u
608 #define PL011_ICR_CLR_ALL_IRQS 0x07ffu
609 #define PL011_CR_UART_ENABLE 0x01u
610 #define PL011_CR_TX_ENABLE 0x100u
611 #define PL011_CR_RX_ENABLE 0x200u
612
613 #define VMAPPLE_UART0_DR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x00))
614 #define VMAPPLE_UART0_ECR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x04))
615 #define VMAPPLE_UART0_FR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x18))
616 #define VMAPPLE_UART0_IBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x24))
617 #define VMAPPLE_UART0_FBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x28))
618 #define VMAPPLE_UART0_LCR_H *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x2c))
619 #define VMAPPLE_UART0_CR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x30))
620 #define VMAPPLE_UART0_TIMSC *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x38))
621 #define VMAPPLE_UART0_ICR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x44))
622
623 static unsigned int
vmapple_uart_transmit_ready(void)624 vmapple_uart_transmit_ready(void)
625 {
626 return (unsigned int) !(VMAPPLE_UART0_FR & 0x20);
627 }
628
629 static void
vmapple_uart_transmit_data(uint8_t c)630 vmapple_uart_transmit_data(uint8_t c)
631 {
632 VMAPPLE_UART0_DR = (uint32_t) c;
633 }
634
635 static unsigned int
vmapple_uart_receive_ready(void)636 vmapple_uart_receive_ready(void)
637 {
638 return (unsigned int) !(VMAPPLE_UART0_FR & 0x10);
639 }
640
641 static uint8_t
vmapple_uart_receive_data(void)642 vmapple_uart_receive_data(void)
643 {
644 return (uint8_t) (VMAPPLE_UART0_DR & 0xff);
645 }
646
647 static void
vmapple_uart_init(void)648 vmapple_uart_init(void)
649 {
650 VMAPPLE_UART0_CR = 0x0;
651 VMAPPLE_UART0_ECR = 0x0;
652 VMAPPLE_UART0_LCR_H = (
653 PL011_LCR_WORD_LENGTH_8 |
654 PL011_LCR_FIFO_ENABLE |
655 PL011_LCR_ONE_STOP_BIT |
656 PL011_LCR_PARITY_DISABLE |
657 PL011_LCR_BREAK_DISABLE
658 );
659 VMAPPLE_UART0_IBRD = PL011_IBRD_DIV_38400;
660 VMAPPLE_UART0_FBRD = PL011_FBRD_DIV_38400;
661 VMAPPLE_UART0_TIMSC = 0x0;
662 VMAPPLE_UART0_ICR = PL011_ICR_CLR_ALL_IRQS;
663 VMAPPLE_UART0_CR = (
664 PL011_CR_UART_ENABLE |
665 PL011_CR_TX_ENABLE |
666 PL011_CR_RX_ENABLE
667 );
668 }
669
670 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) vmapple_uart_serial_functions =
671 {
672 .init = vmapple_uart_init,
673 .transmit_ready = vmapple_uart_transmit_ready,
674 .transmit_data = vmapple_uart_transmit_data,
675 .receive_ready = vmapple_uart_receive_ready,
676 .receive_data = vmapple_uart_receive_data,
677 .device = SERIAL_VMAPPLE_UART
678 };
679
680 #endif /* VMAPPLE_UART */
681
682 /*****************************************************************************/
683
684 /**
685 * Output @str onto every registered serial interface by polling.
686 *
687 * @param str The stringto output.
688 */
689 static void uart_sends_force_poll(
690 const char *str
691 );
692
693 static void
register_serial_functions(struct pe_serial_functions * fns)694 register_serial_functions(struct pe_serial_functions *fns)
695 {
696 fns->next = gPESF;
697 gPESF = fns;
698 }
699
700 #if HIBERNATION
701 /**
702 * Transitions the serial driver into a mode that can be run in the hibernation
703 * resume context. In this mode, the serial driver runs at a barebones level
704 * without making sure the serial devices are properly initialized or utilizing
705 * features such as the software drain timer for dockchannels.
706 *
707 * Upon the next call to serial_init (once the hibernation image has been
708 * loaded), this mode is exited and we return to the normal operation of the
709 * driver.
710 */
711 MARK_AS_HIBERNATE_TEXT void
serial_hibernation_init(void)712 serial_hibernation_init(void)
713 {
714 uart_hibernation = true;
715 #if defined(APPLE_UART)
716 apple_uart_registers = (apple_uart_registers_t *)gHibernateGlobals.hibUartRegPhysBase;
717 #endif /* defined(APPLE_UART) */
718 #if defined(DOCKCHANNEL_UART)
719 dockchannel_uart_base = gHibernateGlobals.dockChannelRegPhysBase;
720 #endif /* defined(DOCKCHANNEL_UART) */
721 }
722
723 /**
724 * Transitions the serial driver back to non-hibernation mode so it can resume
725 * normal operations. Should only be called from serial_init on a hibernation
726 * resume.
727 */
728 MARK_AS_HIBERNATE_TEXT static void
serial_hibernation_cleanup(void)729 serial_hibernation_cleanup(void)
730 {
731 uart_hibernation = false;
732 #if defined(APPLE_UART)
733 apple_uart_registers = (apple_uart_registers_t *)gHibernateGlobals.hibUartRegVirtBase;
734 #endif /* defined(APPLE_UART) */
735 #if defined(DOCKCHANNEL_UART)
736 dockchannel_uart_base = gHibernateGlobals.dockChannelRegVirtBase;
737 #endif /* defined(DOCKCHANNEL_UART) */
738 }
739 #endif /* HIBERNATION */
740
741 int
serial_init(void)742 serial_init(void)
743 {
744 DTEntry entryP = NULL;
745 uint32_t prop_size;
746 vm_offset_t soc_base;
747 uintptr_t const *reg_prop;
748 uint32_t const *prop_value __unused = NULL;
749
750 struct pe_serial_functions *fns = gPESF;
751
752 /**
753 * Even if the serial devices have already been initialized on cold boot,
754 * when coming out of a sleep/wake, they'll need to be re-initialized. Since
755 * the uart_initted value is kept across a sleep/wake, always re-initialize
756 * to be safe.
757 */
758 if (uart_initted) {
759 #if HIBERNATION
760 if (uart_hibernation) {
761 serial_hibernation_cleanup();
762 }
763 #endif /* HIBERNATION */
764 while (fns != NULL) {
765 fns->init();
766 fns = fns->next;
767 }
768
769 return 1;
770 }
771
772 soc_base = pe_arm_get_soc_base_phys();
773
774 if (soc_base == 0) {
775 return 0;
776 }
777
778 PE_parse_boot_argn("disable-uart-irq", &disable_uart_irq, sizeof(disable_uart_irq));
779
780 #ifdef PI3_UART
781 if (SecureDTFindEntry("name", "gpio", &entryP) == kSuccess) {
782 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
783 pi3_gpio_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
784 }
785 if (SecureDTFindEntry("name", "aux", &entryP) == kSuccess) {
786 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
787 pi3_aux_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
788 }
789 if ((pi3_gpio_base_vaddr != 0) && (pi3_aux_base_vaddr != 0)) {
790 register_serial_functions(&pi3_uart_serial_functions);
791 }
792 #endif /* PI3_UART */
793
794 #ifdef VMAPPLE_UART
795 if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
796 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
797 vmapple_uart0_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
798 }
799
800 if (vmapple_uart0_base_vaddr != 0) {
801 register_serial_functions(&vmapple_uart_serial_functions);
802 }
803 #endif /* VMAPPLE_UART */
804
805 #ifdef DOCKCHANNEL_UART
806 uint32_t no_dockchannel_uart = 0;
807 if (SecureDTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) {
808 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
809 // Should be two reg entries
810 if (prop_size / sizeof(uintptr_t) != 4) {
811 panic("Malformed dockchannel-uart property");
812 }
813 dockchannel_uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
814 dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3));
815 PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart));
816 // Keep the old name for boot-arg
817 if (no_dockchannel_uart == 0) {
818 register_serial_functions(&dockchannel_serial_functions);
819 SecureDTGetProperty(entryP, "max-aop-clk", (void const **)&prop_value, &prop_size);
820 max_dockchannel_drain_period = (uint32_t)((prop_value)? (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD);
821 prop_value = NULL;
822 SecureDTGetProperty(entryP, "enable-sw-drain", (void const **)&prop_value, &prop_size);
823 use_sw_drain = (prop_value)? *prop_value : 0;
824 prop_value = NULL;
825 SecureDTGetProperty(entryP, "dock-wstat-mask", (void const **)&prop_value, &prop_size);
826 dock_wstat_mask = (prop_value)? *prop_value : 0x1ff;
827 prop_value = NULL;
828 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
829 if (prop_value) {
830 dockchannel_serial_functions.has_irq = true;
831 }
832
833 /* Set sane defaults for dockchannel globals. */
834 prev_dockchannel_spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
835 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
836 } else {
837 dockchannel_clear_intr();
838 }
839 // If no dockchannel-uart is found in the device tree, fall back
840 // to looking for the traditional UART serial console.
841 }
842
843 #endif /* DOCKCHANNEL_UART */
844
845 #ifdef APPLE_UART
846 char const *serial_compat = 0;
847 uint32_t use_legacy_uart = 0;
848
849 /* Check if we should enable this deprecated serial device. */
850 PE_parse_boot_argn("use-legacy-uart", &use_legacy_uart, sizeof(use_legacy_uart));
851
852 /*
853 * The boot serial port should have a property named "boot-console".
854 * If we don't find it there, look for "uart0" and "uart1".
855 */
856 if (SecureDTFindEntry("boot-console", NULL, &entryP) == kSuccess) {
857 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
858 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
859 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
860 } else if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
861 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
862 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
863 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
864 } else if (SecureDTFindEntry("name", "uart1", &entryP) == kSuccess) {
865 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
866 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
867 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
868 }
869
870 if (NULL != entryP) {
871 prop_value = NULL;
872 SecureDTGetProperty(entryP, "sampling", (void const **)&prop_value, &prop_size);
873 if (prop_value) {
874 dt_sampling = *prop_value;
875 }
876
877 prop_value = NULL;
878 SecureDTGetProperty(entryP, "ubrdiv", (void const **)&prop_value, &prop_size);
879 if (prop_value) {
880 dt_ubrdiv = *prop_value;
881 }
882
883 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
884 if (prop_value) {
885 apple_serial_functions.has_irq = true;
886 }
887
888 /* Workaround to enable legacy serial for fastsim targets until clients migrate to dockchannels. */
889 SecureDTGetProperty(entryP, "enable-legacy-serial", (void const **)&prop_value, &prop_size);
890 if (prop_value) {
891 use_legacy_uart = 1;
892 }
893 }
894
895 if (use_legacy_uart && serial_compat && !strcmp(serial_compat, "uart-1,samsung")) {
896 register_serial_functions(&apple_serial_functions);
897 }
898 #endif /* APPLE_UART */
899
900 if (gPESF == NULL) {
901 return 0;
902 }
903
904 fns = gPESF;
905 while (fns != NULL) {
906 serial_do_transmit = 1;
907 fns->init();
908 if (fns->has_irq) {
909 serial_irq_status |= fns->device; // serial_device_t is one-hot
910 }
911 fns = fns->next;
912 }
913
914 #if HIBERNATION
915 /* hibernation needs to know the UART register addresses since it can't directly use this serial driver */
916 if (dockchannel_uart_base) {
917 gHibernateGlobals.dockChannelRegPhysBase = ml_vtophys(dockchannel_uart_base);
918 gHibernateGlobals.dockChannelRegVirtBase = dockchannel_uart_base;
919 gHibernateGlobals.dockChannelWstatMask = dock_wstat_mask;
920 }
921 if (apple_uart_registers) {
922 gHibernateGlobals.hibUartRegPhysBase = ml_vtophys((vm_offset_t)apple_uart_registers);
923 gHibernateGlobals.hibUartRegVirtBase = (vm_offset_t)apple_uart_registers;
924 }
925 #endif /* HIBERNATION */
926
927 /* Complete. */
928 uart_initted = true;
929 return 1;
930 }
931
932 /**
933 * Forbid or allow transmission over each serial until they receive data.
934 */
935 void
serial_set_on_demand(bool on_demand)936 serial_set_on_demand(bool on_demand)
937 {
938 /* Enable or disable transmission. */
939 serial_do_transmit = !on_demand;
940
941 /* If on-demand is enabled, report it. */
942 if (on_demand) {
943 uart_sends_force_poll(
944 "On-demand serial mode selected.\n"
945 "Waiting for user input to send logs.\n"
946 );
947 }
948 }
949
950 /**
951 * Returns a deadline for the longest time the serial driver should wait for an
952 * interrupt for. This serves as a timeout for the IRQ to allow for the software
953 * drain timer that dockchannels supports.
954 *
955 * @param fns serial functions representing the device to find the deadline for
956 *
957 * @returns absolutetime deadline for this device's IRQ.
958 */
959 static uint64_t
serial_interrupt_deadline(__unused struct pe_serial_functions * fns)960 serial_interrupt_deadline(__unused struct pe_serial_functions *fns)
961 {
962 #if defined(DOCKCHANNEL_UART)
963 if (fns->device == SERIAL_DOCKCHANNEL && use_sw_drain) {
964 return dockchannel_drain_deadline;
965 }
966 #endif
967
968 /**
969 * Default to 1.5ms for all other devices. 1.5ms was chosen as the baudrate
970 * of the AppleSerialDevice is 115200, meaning that it should only take
971 * ~1.5ms to drain the 16 character buffer completely.
972 */
973 uint64_t timeout_interval;
974 nanoseconds_to_absolutetime(1500 * NSEC_PER_USEC, &timeout_interval);
975 return mach_absolute_time() + timeout_interval;
976 }
977
978 /**
979 * Goes to sleep waiting for an interrupt from a specificed serial device.
980 *
981 * @param fns serial functions representing the device to wait for
982 */
983 static void
serial_wait_for_interrupt(struct pe_serial_functions * fns)984 serial_wait_for_interrupt(struct pe_serial_functions *fns)
985 {
986 assert_wait_deadline(fns, THREAD_UNINT, serial_interrupt_deadline(fns));
987 if (!fns->transmit_ready()) {
988 fns->enable_irq();
989 thread_block(THREAD_CONTINUE_NULL);
990 } else {
991 clear_wait(current_thread(), THREAD_AWAKENED);
992 }
993 }
994
995 /**
996 * Output a character onto every registered serial interface.
997 *
998 * @param c The character to output.
999 * @param poll Whether the driver should poll to send the character or if it can
1000 * wait for an interrupt.
1001 * @param force : send even if transmission is disabled on the serial.
1002 */
1003 MARK_AS_HIBERNATE_TEXT void
1004 static
uart_putc_options_(char c,bool poll,bool force)1005 uart_putc_options_(char c, bool poll, bool force)
1006 {
1007 struct pe_serial_functions *fns = gPESF;
1008
1009 while (fns != NULL) {
1010 if (force || serial_do_transmit) {
1011 while (!fns->transmit_ready()) {
1012 if (!uart_hibernation) {
1013 if (!poll && irq_available_and_ready(fns)) {
1014 serial_wait_for_interrupt(fns);
1015 } else {
1016 serial_poll();
1017 }
1018 }
1019 }
1020 fns->transmit_data((uint8_t)c);
1021 }
1022 fns = fns->next;
1023 }
1024 }
1025
1026 /**
1027 * Output a character onto every registered serial interface whose
1028 * transmission is enabled..
1029 *
1030 * @param c The character to output.
1031 * @param poll Whether the driver should poll to send the character or if it can
1032 * wait for an interrupt
1033 */
1034 MARK_AS_HIBERNATE_TEXT void
uart_putc_options(char c,bool poll)1035 uart_putc_options(char c, bool poll)
1036 {
1037 uart_putc_options_(c, poll, 0);
1038 }
1039
1040 /**
1041 * Output a character onto every registered serial interface whose
1042 * transmission is enabled by polling.
1043 *
1044 * @param c The character to output.
1045 */
1046 void
uart_putc(char c)1047 uart_putc(char c)
1048 {
1049 uart_putc_options(c, true);
1050 }
1051
1052 /**
1053 * Output @str onto every registered serial interface by polling.
1054 *
1055 * @param str The stringto output.
1056 */
1057 static void
uart_sends_force_poll(const char * str)1058 uart_sends_force_poll(
1059 const char *str)
1060 {
1061 char c;
1062 while ((c = *(str++))) {
1063 uart_putc_options_(c, true, true);
1064 }
1065 }
1066
1067 /**
1068 * Read a character from the first registered serial interface that has data
1069 * available.
1070 *
1071 * @return The character if any interfaces have data available, otherwise -1.
1072 */
1073 int
uart_getc(void)1074 uart_getc(void)
1075 {
1076 struct pe_serial_functions *fns = gPESF;
1077 while (fns != NULL) {
1078 if (fns->receive_ready()) {
1079 serial_do_transmit = 1;
1080 return (int)fns->receive_data();
1081 }
1082 fns = fns->next;
1083 }
1084 return -1;
1085 }
1086
1087 /**
1088 * Enables IRQs for a specific serial device and returns whether or not IRQs for
1089 * that device where enabled successfully. For a serial driver to have irqs
1090 * enabled, it must have the enable_irq, disable_irq, and acknowledge_irq
1091 * functions defined and the has_irq flag set.
1092 *
1093 * @param device Serial device to enable irqs on
1094 * @note This function should only be called from the AppleSerialShim kext
1095 */
1096 kern_return_t
serial_irq_enable(serial_device_t device)1097 serial_irq_enable(serial_device_t device)
1098 {
1099 struct pe_serial_functions *fns = get_serial_functions(device);
1100
1101 if (!fns || !fns->has_irq || disable_uart_irq) {
1102 return KERN_FAILURE;
1103 }
1104
1105 serial_irq_status &= ~device;
1106
1107 return KERN_SUCCESS;
1108 }
1109
1110 /**
1111 * Performs any actions needed to handle this IRQ. Wakes up the thread waiting
1112 * on the interrupt if one exists.
1113 *
1114 * @param device Serial device that generated the IRQ.
1115 * @note Interrupts will have already been cleared and disabled by serial_irq_filter.
1116 * @note This function should only be called from the AppleSerialShim kext.
1117 */
1118 kern_return_t
serial_irq_action(serial_device_t device)1119 serial_irq_action(serial_device_t device)
1120 {
1121 struct pe_serial_functions *fns = get_serial_functions(device);
1122
1123 if (!fns || !fns->has_irq) {
1124 return KERN_FAILURE;
1125 }
1126
1127 /**
1128 * Because IRQs are enabled only when we know a thread is about to sleep, we
1129 * can call wake up and reasonably expect there to be a thread waiting.
1130 */
1131 thread_wakeup(fns);
1132
1133 return KERN_SUCCESS;
1134 }
1135
1136 /**
1137 * Returns true if the pending IRQ for device is one that can be handled by the
1138 * platform serial driver.
1139 *
1140 * @param device Serial device that generated the IRQ.
1141 * @note This function is called from a primary interrupt context and should be
1142 * kept lightweight.
1143 * @note This function should only be called from the AppleSerialShim kext
1144 */
1145 bool
serial_irq_filter(serial_device_t device)1146 serial_irq_filter(serial_device_t device)
1147 {
1148 struct pe_serial_functions *fns = get_serial_functions(device);
1149
1150 if (!fns || !fns->has_irq) {
1151 return false;
1152 }
1153
1154 /**
1155 * Disable IRQs until next time a thread waits for an interrupt to prevent an interrupt storm.
1156 */
1157 const bool had_irqs_enabled = fns->disable_irq();
1158 const bool was_our_interrupt = fns->acknowledge_irq();
1159
1160 /* Re-enable IRQs if the interrupt wasn't for us. */
1161 if (had_irqs_enabled && !was_our_interrupt) {
1162 fns->enable_irq();
1163 }
1164
1165 return was_our_interrupt;
1166 }
1167
1168 /**
1169 * Prepares all serial devices to go to sleep by draining the hardware FIFOs
1170 * and disabling interrupts.
1171 */
1172 void
serial_go_to_sleep(void)1173 serial_go_to_sleep(void)
1174 {
1175 struct pe_serial_functions *fns = gPESF;
1176 while (fns != NULL) {
1177 if (irq_available_and_ready(fns)) {
1178 fns->disable_irq();
1179 }
1180 fns = fns->next;
1181 }
1182
1183 #ifdef APPLE_UART
1184 /* APPLE_UART needs to drain FIFO before sleeping */
1185 if (get_serial_functions(SERIAL_APPLE_UART)) {
1186 apple_uart_drain_fifo();
1187 }
1188 #endif /* APPLE_UART */
1189 }
1190