1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * This file contains the low-level serial drivers used on ARM/ARM64 devices.
5 * The generic serial console code in osfmk/console/serial_console.c will call
6 * into this code to transmit and receive serial data.
7 *
8 * Logging can be performed on multiple serial interfaces at once through a
9 * method called serial multiplexing. This is implemented by enumerating which
10 * serial interfaces are available on boot and registering them into a linked
11 * list of interfaces pointed to by gPESF. When outputting or receiving
12 * characters, each interface is queried in turn.
13 *
14 * Please view doc/arm_serial.md for an in-depth description of these drivers.
15 */
16 #include <kern/clock.h>
17 #include <kern/debug.h>
18 #include <libkern/OSBase.h>
19 #include <libkern/section_keywords.h>
20 #include <mach/mach_time.h>
21 #include <machine/atomic.h>
22 #include <machine/machine_routines.h>
23 #include <pexpert/pexpert.h>
24 #include <pexpert/protos.h>
25 #include <pexpert/device_tree.h>
26 #include <pexpert/arm/consistent_debug.h>
27 #include <pexpert/arm64/board_config.h>
28 #include <arm64/proc_reg.h>
29 #include <pexpert/arm/protos.h>
30 #include <kern/sched_prim.h>
31 #if HIBERNATION
32 #include <machine/pal_hibernate.h>
33 #endif /* HIBERNATION */
34
35 struct pe_serial_functions {
36 /* Initialize the underlying serial hardware. */
37 void (*init) (void);
38
39 /* Return a non-zero value if the serial interface is ready to send more data. */
40 unsigned int (*transmit_ready) (void);
41
42 /* Write a single byte of data to serial. */
43 void (*transmit_data) (uint8_t c);
44
45 /* Return a non-zero value if there's a byte of data available. */
46 unsigned int (*receive_ready) (void);
47
48 /* Read a single byte from serial. */
49 uint8_t (*receive_data) (void);
50
51 /* Enables IRQs from this device. */
52 void (*enable_irq) (void);
53
54 /* Disables IRQs from this device and reports whether IRQs were enabled. */
55 bool (*disable_irq) (void);
56
57 /* Clears this device's IRQs targeting this agent, returning true if at least one IRQ was cleared. */
58 bool (*acknowledge_irq) (void);
59
60 /**
61 * Whether this serial driver can handle irqs. This value should be set by
62 * querying the device tree to see if the serial device has interrupts
63 * associated with it.
64 *
65 * For a device to support IRQs:
66 * - enable_irq, disable_irq, and acknowledge_irq must be non-null
67 * - The AppleSerialShim kext must be able to match to the serial device
68 * in the IORegistry and call serial_enable_irq with the proper
69 * serial_device_t
70 * - The device tree entry for the serial device should have an interrupt
71 * associated with it.
72 */
73 bool has_irq;
74
75 /* enum identifying which serial device these functions belong to. */
76 serial_device_t device;
77
78 /* Pointer to the next serial interface in the linked-list. */
79 struct pe_serial_functions *next;
80 };
81
82 MARK_AS_HIBERNATE_DATA_CONST_LATE static struct pe_serial_functions* gPESF = NULL;
83
84 /**
85 * Whether uart has been initialized already. This value is kept across a
86 * sleep/wake cycle so we know we need to reinitialize when serial_init is
87 * called again after wake.
88 */
89 MARK_AS_HIBERNATE_DATA static bool uart_initted = false;
90
91 /* Whether uart should run in simple mode that works during hibernation resume. */
92 MARK_AS_HIBERNATE_DATA static bool uart_hibernation = false;
93
94 /**
95 * Used to track if all IRQs have been initialized. Each bit of this variable
96 * represents whether or not a serial device that reports supporting IRQs has
97 * been initialized yet (1 -> not initialized, 0 -> initialized)
98 */
99 static uint32_t serial_irq_status = 0;
100
101 /**
102 * Set by the 'disable-uart-irq' boot-arg to force serial IRQs into polling mode
103 * by preventing the serial driver shim kext from registering itself with
104 * serial_enable_irq.
105 */
106 static bool disable_uart_irq = 0;
107
108 /**
109 * Indicates whether or not a given device's irqs have been set up by calling
110 * serial_enable_irq for that particular device.
111 *
112 * @param device_fns Serial functions for the device that is being checked
113 * @return Whether or not the irqs have been initialized for that device
114 */
115 static bool
irq_initialized(struct pe_serial_functions * device_fns)116 irq_initialized(struct pe_serial_functions *device_fns)
117 {
118 return (serial_irq_status & device_fns->device) == 0;
119 }
120
121 /**
122 * Indicates whether or not a given device supports irqs and if they are ready
123 * to be used.
124 *
125 * @param device_fns Serial functions for the device that is being checked
126 * @return Whether or not the device can and will send IRQs.
127 */
128 static bool
irq_available_and_ready(struct pe_serial_functions * device_fns)129 irq_available_and_ready(struct pe_serial_functions *device_fns)
130 {
131 return device_fns->has_irq && irq_initialized(device_fns);
132 }
133
134 /**
135 * Searches through the global serial functions list and returns the serial function for a particular device
136 *
137 * @param device The device identifier to search for
138 * @return Serial functions for the specified device
139 */
140 static struct pe_serial_functions *
get_serial_functions(serial_device_t device)141 get_serial_functions(serial_device_t device)
142 {
143 struct pe_serial_functions *fns = gPESF;
144 while (fns != NULL) {
145 if (fns->device == device) {
146 return fns;
147 }
148 fns = fns->next;
149 }
150 return NULL;
151 }
152
153 /**
154 * The action to take when polling and waiting for a serial device to be ready
155 * for output. On ARM64, takes a WFE because the WFE timeout will wake us up in
156 * the worst case. On ARMv7 devices, we need to hot poll.
157 */
158 static void
serial_poll(void)159 serial_poll(void)
160 {
161 #if __arm64__
162 __builtin_arm_wfe();
163 #endif
164 }
165
166 /**
167 * This ensures that if we have a future product that supports hibernation, but
168 * doesn't support either UART serial or dock-channels, then hibernation will
169 * gracefully fall back to the serial method that is supported.
170 */
171 #if HIBERNATION || defined(APPLE_UART)
172 MARK_AS_HIBERNATE_DATA static vm_offset_t uart_base = 0;
173 #endif /* HIBERNATION || defined(APPLE_UART) */
174
175 #if HIBERNATION || defined(DOCKCHANNEL_UART)
176 MARK_AS_HIBERNATE_DATA static vm_offset_t dockchannel_uart_base = 0;
177 #endif /* HIBERNATION || defined(DOCKCHANNEL_UART) */
178
179 /*****************************************************************************/
180
181 #ifdef APPLE_UART
182
183 static int32_t dt_pclk = -1;
184 static int32_t dt_sampling = -1;
185 static int32_t dt_ubrdiv = -1;
186
187 static void apple_uart_set_baud_rate(uint32_t baud_rate);
188
189 static void
apple_uart_init(void)190 apple_uart_init(void)
191 {
192 uint32_t ucon0 = 0x405; /* NCLK, No interrupts, No DMA - just polled */
193
194 rULCON0 = 0x03; /* 81N, not IR */
195
196 // Override with pclk dt entry
197 if (dt_pclk != -1) {
198 ucon0 = ucon0 & ~0x400;
199 }
200
201 rUCON0 = ucon0;
202 rUMCON0 = 0x00; /* Clear Flow Control */
203
204 apple_uart_set_baud_rate(115200);
205
206 rUFCON0 = 0x07; /* Clear & Enable FIFOs */
207 rUMCON0 = 0x01; /* Assert RTS on UART0 */
208 }
209
210 static void
apple_uart_enable_irq(void)211 apple_uart_enable_irq(void)
212 {
213 /* sets Tx FIFO watermark to 0 bytes so interrupt is sent when FIFO empty */
214 rUFCON0 &= ~(0xC0);
215
216 /* Enables Tx interrupt */
217 rUCON0 |= 0x2000;
218 }
219
220 static bool
apple_uart_disable_irq(void)221 apple_uart_disable_irq(void)
222 {
223 /* Disables Tx interrupts */
224 const uint32_t ucon0 = rUCON0;
225 const bool irqs_were_enabled = ucon0 & (0x2000);
226
227 if (irqs_were_enabled) {
228 rUCON0 = ucon0 & ~(0x2000);
229 }
230
231 return irqs_were_enabled;
232 }
233
234 static bool
apple_uart_ack_irq(void)235 apple_uart_ack_irq(void)
236 {
237 rUTRSTAT0 |= 0x20;
238 return true;
239 }
240
241 static void
apple_uart_drain_fifo(void)242 apple_uart_drain_fifo(void)
243 {
244 /* wait while Tx FIFO is full or the FIFO count != 0 */
245 while ((rUFSTAT0 & 0x2F0)) {
246 serial_poll();
247 }
248 }
249
250 static void
apple_uart_set_baud_rate(uint32_t baud_rate)251 apple_uart_set_baud_rate(uint32_t baud_rate)
252 {
253 uint32_t div = 0;
254 uint32_t uart_clock = 0;
255 uint32_t sample_rate = 16;
256
257 if (baud_rate < 300) {
258 baud_rate = 9600;
259 }
260
261 if (rUCON0 & 0x400) {
262 // NCLK
263 uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz;
264 } else {
265 // PCLK
266 uart_clock = (uint32_t)gPEClockFrequencyInfo.prf_frequency_hz;
267 }
268
269 if (dt_sampling != -1) {
270 // Use the sampling rate specified in the Device Tree
271 sample_rate = dt_sampling & 0xf;
272 }
273
274 if (dt_ubrdiv != -1) {
275 // Use the ubrdiv specified in the Device Tree
276 div = dt_ubrdiv & 0xffff;
277 } else {
278 // Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1
279 div = uart_clock / (baud_rate * sample_rate);
280
281 uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate);
282 uint32_t baud_low = uart_clock / ((div + 1) * sample_rate);
283
284 // Adjust div to get the closest target baudrate
285 if ((baud_rate - baud_low) > (actual_baud - baud_rate)) {
286 div--;
287 }
288 }
289
290 // Sample Rate [19:16], UBRDIV [15:0]
291 rUBRDIV0 = ((16 - sample_rate) << 16) | div;
292 }
293
294 MARK_AS_HIBERNATE_TEXT static unsigned int
apple_uart_tr0(void)295 apple_uart_tr0(void)
296 {
297 /* UART is ready unless the FIFO is full. */
298 return (rUFSTAT0 & 0x200) == 0;
299 }
300
301 MARK_AS_HIBERNATE_TEXT static void
apple_uart_td0(uint8_t c)302 apple_uart_td0(uint8_t c)
303 {
304 rUTXH0 = c;
305 }
306
307 static unsigned int
apple_uart_rr0(void)308 apple_uart_rr0(void)
309 {
310 /* Receive is ready when there are >0 bytes in the receive FIFO */
311 /* FIFO count is the low 4 bits and the fifo full flag is 1 << 8 */
312 return rUFSTAT0 & ((1 << 8) | 0x0f);
313 }
314
315 static uint8_t
apple_uart_rd0(void)316 apple_uart_rd0(void)
317 {
318 return (uint8_t)rURXH0;
319 }
320
321 MARK_AS_HIBERNATE_DATA_CONST_LATE
322 static struct pe_serial_functions apple_serial_functions =
323 {
324 .init = apple_uart_init,
325 .transmit_ready = apple_uart_tr0,
326 .transmit_data = apple_uart_td0,
327 .receive_ready = apple_uart_rr0,
328 .receive_data = apple_uart_rd0,
329 .enable_irq = apple_uart_enable_irq,
330 .disable_irq = apple_uart_disable_irq,
331 .acknowledge_irq = apple_uart_ack_irq,
332 .device = SERIAL_APPLE_UART
333 };
334
335 #endif /* APPLE_UART */
336
337 /*****************************************************************************/
338
339 #ifdef DOCKCHANNEL_UART
340 #define DOCKCHANNEL_WR_MAX_STALL_US (30*1000)
341
342 static vm_offset_t dock_agent_base;
343 static uint32_t max_dockchannel_drain_period;
344 static uint64_t dockchannel_drain_deadline; // Deadline for external agent to drain before a software drain occurs
345 static bool use_sw_drain;
346 static uint32_t dock_wstat_mask;
347 static uint64_t prev_dockchannel_spaces; // Previous w_stat level of the DockChannel.
348 static uint64_t dockchannel_stall_grace;
349 MARK_AS_HIBERNATE_DATA static bool use_sw_drain;
350 MARK_AS_HIBERNATE_DATA static uint32_t dock_wstat_mask;
351
352 // forward reference
353 static struct pe_serial_functions dockchannel_serial_functions;
354
355 //=======================
356 // Local funtions
357 //=======================
358
359 static int
dockchannel_drain_on_stall()360 dockchannel_drain_on_stall()
361 {
362 // Called when DockChannel runs out of spaces.
363 // Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves.
364 // Return number of bytes drained.
365
366 if (mach_absolute_time() >= dockchannel_drain_deadline) {
367 // It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO
368 // Drop a character.
369 (void)rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
370 os_atomic_inc(&prev_dockchannel_spaces, relaxed);
371 return 1;
372 }
373 return 0;
374 }
375
376 static void
dockchannel_clear_intr(void)377 dockchannel_clear_intr(void)
378 {
379 rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3);
380 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3;
381 rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3);
382 rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3;
383 }
384
385 static bool
dockchannel_disable_irq(void)386 dockchannel_disable_irq(void)
387 {
388 const uint32_t ap_intr_ctrl = rDOCKCHANNELS_AGENT_AP_INTR_CTRL;
389 const bool irqs_were_enabled = ap_intr_ctrl & 0x1;
390 if (irqs_were_enabled) {
391 rDOCKCHANNELS_AGENT_AP_INTR_CTRL = ap_intr_ctrl & ~(0x1);
392 }
393 return irqs_were_enabled;
394 }
395
396 static void
dockchannel_enable_irq(void)397 dockchannel_enable_irq(void)
398 {
399 // set interrupt to be when fifo has 255 empty
400 rDOCKCHANNELS_DEV_WR_WATERMARK(DOCKCHANNEL_UART_CHANNEL) = 0xFF;
401 rDOCKCHANNELS_AGENT_AP_INTR_CTRL |= 0x1;
402 }
403
404 static bool
dockchannel_ack_irq(void)405 dockchannel_ack_irq(void)
406 {
407 /* First check if the IRQ is for the kernel */
408 if (rDOCKCHANNELS_AGENT_AP_INTR_STATUS & 0x1) {
409 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x1;
410 return true;
411 }
412 return false;
413 }
414
415 MARK_AS_HIBERNATE_TEXT static void
dockchannel_transmit_data(uint8_t c)416 dockchannel_transmit_data(uint8_t c)
417 {
418 rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)c;
419
420 if (use_sw_drain && !uart_hibernation) {
421 os_atomic_dec(&prev_dockchannel_spaces, relaxed); // After writing a byte we have one fewer space than previously expected.
422 }
423 }
424
425 static unsigned int
dockchannel_receive_ready(void)426 dockchannel_receive_ready(void)
427 {
428 return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f;
429 }
430
431 static uint8_t
dockchannel_receive_data(void)432 dockchannel_receive_data(void)
433 {
434 return (uint8_t)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL) >> 8) & 0xff);
435 }
436
437 MARK_AS_HIBERNATE_TEXT static unsigned int
dockchannel_transmit_ready(void)438 dockchannel_transmit_ready(void)
439 {
440 uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
441
442 if (!uart_hibernation) {
443 if (use_sw_drain) {
444 if (spaces > prev_dockchannel_spaces) {
445 // More spaces showed up. That can only mean someone read the FIFO.
446 // Note that if the DockFIFO is empty we cannot tell if someone is listening,
447 // we can only give them the benefit of the doubt.
448 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
449 }
450 prev_dockchannel_spaces = spaces;
451 return spaces || dockchannel_drain_on_stall();
452 }
453 }
454
455 return spaces;
456 }
457
458 static void
dockchannel_init(void)459 dockchannel_init(void)
460 {
461 if (use_sw_drain) {
462 nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace);
463 }
464
465 // Clear all interrupt enable and status bits
466 dockchannel_clear_intr();
467
468 // Setup DRAIN timer
469 rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period;
470
471 // Drain timer doesn't get loaded with value from drain period register if fifo
472 // is already full. Drop a character from the fifo.
473 rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
474 }
475
476 MARK_AS_HIBERNATE_DATA_CONST_LATE
477 static struct pe_serial_functions dockchannel_serial_functions =
478 {
479 .init = dockchannel_init,
480 .transmit_ready = dockchannel_transmit_ready,
481 .transmit_data = dockchannel_transmit_data,
482 .receive_ready = dockchannel_receive_ready,
483 .receive_data = dockchannel_receive_data,
484 .enable_irq = dockchannel_enable_irq,
485 .disable_irq = dockchannel_disable_irq,
486 .acknowledge_irq = dockchannel_ack_irq,
487 .device = SERIAL_DOCKCHANNEL
488 };
489
490 #endif /* DOCKCHANNEL_UART */
491
492 /****************************************************************************/
493 #ifdef PI3_UART
494 vm_offset_t pi3_gpio_base_vaddr = 0;
495 vm_offset_t pi3_aux_base_vaddr = 0;
496 static unsigned int
pi3_uart_tr0(void)497 pi3_uart_tr0(void)
498 {
499 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x20;
500 }
501
502 static void
pi3_uart_td0(uint8_t c)503 pi3_uart_td0(uint8_t c)
504 {
505 BCM2837_PUT32(BCM2837_AUX_MU_IO_REG_V, (uint32_t) c);
506 }
507
508 static unsigned int
pi3_uart_rr0(void)509 pi3_uart_rr0(void)
510 {
511 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x01;
512 }
513
514 static uint8_t
pi3_uart_rd0(void)515 pi3_uart_rd0(void)
516 {
517 return (uint8_t) BCM2837_GET32(BCM2837_AUX_MU_IO_REG_V);
518 }
519
520 static void
pi3_uart_init(void)521 pi3_uart_init(void)
522 {
523 // Scratch variable
524 uint32_t i;
525
526 // Reset mini uart registers
527 BCM2837_PUT32(BCM2837_AUX_ENABLES_V, 1);
528 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 0);
529 BCM2837_PUT32(BCM2837_AUX_MU_LCR_REG_V, 3);
530 BCM2837_PUT32(BCM2837_AUX_MU_MCR_REG_V, 0);
531 BCM2837_PUT32(BCM2837_AUX_MU_IER_REG_V, 0);
532 BCM2837_PUT32(BCM2837_AUX_MU_IIR_REG_V, 0xC6);
533 BCM2837_PUT32(BCM2837_AUX_MU_BAUD_REG_V, 270);
534
535 i = (uint32_t)BCM2837_FSEL_REG(14);
536 // Configure GPIOs 14 & 15 for alternate function 5
537 i &= ~(BCM2837_FSEL_MASK(14));
538 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(14));
539 i &= ~(BCM2837_FSEL_MASK(15));
540 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(15));
541
542 BCM2837_PUT32(BCM2837_FSEL_REG(14), i);
543
544 BCM2837_PUT32(BCM2837_GPPUD_V, 0);
545
546 // Barrier before AP spinning for 150 cycles
547 __builtin_arm_isb(ISB_SY);
548
549 for (i = 0; i < 150; i++) {
550 asm volatile ("add x0, x0, xzr");
551 }
552
553 __builtin_arm_isb(ISB_SY);
554
555 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, (1 << 14) | (1 << 15));
556
557 __builtin_arm_isb(ISB_SY);
558
559 for (i = 0; i < 150; i++) {
560 asm volatile ("add x0, x0, xzr");
561 }
562
563 __builtin_arm_isb(ISB_SY);
564
565 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, 0);
566
567 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 3);
568 }
569
570 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) pi3_uart_serial_functions =
571 {
572 .init = pi3_uart_init,
573 .transmit_ready = pi3_uart_tr0,
574 .transmit_data = pi3_uart_td0,
575 .receive_ready = pi3_uart_rr0,
576 .receive_data = pi3_uart_rd0,
577 .device = SERIAL_PI3_UART
578 };
579
580 #endif /* PI3_UART */
581
582 /*****************************************************************************/
583
584 #ifdef VMAPPLE_UART
585
586 static vm_offset_t vmapple_uart0_base_vaddr = 0;
587
588 #define PL011_LCR_WORD_LENGTH_8 0x60u
589 #define PL011_LCR_FIFO_DISABLE 0x00u
590
591 #define PL011_LCR_FIFO_ENABLE 0x10u
592
593 #define PL011_LCR_ONE_STOP_BIT 0x00u
594 #define PL011_LCR_PARITY_DISABLE 0x00u
595 #define PL011_LCR_BREAK_DISABLE 0x00u
596 #define PL011_IBRD_DIV_38400 0x27u
597 #define PL011_FBRD_DIV_38400 0x09u
598 #define PL011_ICR_CLR_ALL_IRQS 0x07ffu
599 #define PL011_CR_UART_ENABLE 0x01u
600 #define PL011_CR_TX_ENABLE 0x100u
601 #define PL011_CR_RX_ENABLE 0x200u
602
603 #define VMAPPLE_UART0_DR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x00))
604 #define VMAPPLE_UART0_ECR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x04))
605 #define VMAPPLE_UART0_FR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x18))
606 #define VMAPPLE_UART0_IBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x24))
607 #define VMAPPLE_UART0_FBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x28))
608 #define VMAPPLE_UART0_LCR_H *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x2c))
609 #define VMAPPLE_UART0_CR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x30))
610 #define VMAPPLE_UART0_TIMSC *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x38))
611 #define VMAPPLE_UART0_ICR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x44))
612
613 static unsigned int
vmapple_uart_transmit_ready(void)614 vmapple_uart_transmit_ready(void)
615 {
616 return (unsigned int) !(VMAPPLE_UART0_FR & 0x20);
617 }
618
619 static void
vmapple_uart_transmit_data(uint8_t c)620 vmapple_uart_transmit_data(uint8_t c)
621 {
622 VMAPPLE_UART0_DR = (uint32_t) c;
623 }
624
625 static unsigned int
vmapple_uart_receive_ready(void)626 vmapple_uart_receive_ready(void)
627 {
628 return (unsigned int) !(VMAPPLE_UART0_FR & 0x10);
629 }
630
631 static uint8_t
vmapple_uart_receive_data(void)632 vmapple_uart_receive_data(void)
633 {
634 return (uint8_t) (VMAPPLE_UART0_DR & 0xff);
635 }
636
637 static void
vmapple_uart_init(void)638 vmapple_uart_init(void)
639 {
640 VMAPPLE_UART0_CR = 0x0;
641 VMAPPLE_UART0_ECR = 0x0;
642 VMAPPLE_UART0_LCR_H = (
643 PL011_LCR_WORD_LENGTH_8 |
644 PL011_LCR_FIFO_ENABLE |
645 PL011_LCR_ONE_STOP_BIT |
646 PL011_LCR_PARITY_DISABLE |
647 PL011_LCR_BREAK_DISABLE
648 );
649 VMAPPLE_UART0_IBRD = PL011_IBRD_DIV_38400;
650 VMAPPLE_UART0_FBRD = PL011_FBRD_DIV_38400;
651 VMAPPLE_UART0_TIMSC = 0x0;
652 VMAPPLE_UART0_ICR = PL011_ICR_CLR_ALL_IRQS;
653 VMAPPLE_UART0_CR = (
654 PL011_CR_UART_ENABLE |
655 PL011_CR_TX_ENABLE |
656 PL011_CR_RX_ENABLE
657 );
658 }
659
660 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) vmapple_uart_serial_functions =
661 {
662 .init = vmapple_uart_init,
663 .transmit_ready = vmapple_uart_transmit_ready,
664 .transmit_data = vmapple_uart_transmit_data,
665 .receive_ready = vmapple_uart_receive_ready,
666 .receive_data = vmapple_uart_receive_data,
667 .device = SERIAL_VMAPPLE_UART
668 };
669
670 #endif /* VMAPPLE_UART */
671
672 /*****************************************************************************/
673
674 static void
register_serial_functions(struct pe_serial_functions * fns)675 register_serial_functions(struct pe_serial_functions *fns)
676 {
677 fns->next = gPESF;
678 gPESF = fns;
679 }
680
681 #if HIBERNATION
682 /**
683 * Transitions the serial driver into a mode that can be run in the hibernation
684 * resume context. In this mode, the serial driver runs at a barebones level
685 * without making sure the serial devices are properly initialized or utilizing
686 * features such as the software drain timer for dockchannels.
687 *
688 * Upon the next call to serial_init (once the hibernation image has been
689 * loaded), this mode is exited and we return to the normal operation of the
690 * driver.
691 */
692 MARK_AS_HIBERNATE_TEXT void
serial_hibernation_init(void)693 serial_hibernation_init(void)
694 {
695 uart_hibernation = true;
696 #if defined(APPLE_UART)
697 uart_base = gHibernateGlobals.hibUartRegPhysBase;
698 #endif /* defined(APPLE_UART) */
699 #if defined(DOCKCHANNEL_UART)
700 dockchannel_uart_base = gHibernateGlobals.dockChannelRegPhysBase;
701 #endif /* defined(DOCKCHANNEL_UART) */
702 }
703
704 /**
705 * Transitions the serial driver back to non-hibernation mode so it can resume
706 * normal operations. Should only be called from serial_init on a hibernation
707 * resume.
708 */
709 MARK_AS_HIBERNATE_TEXT static void
serial_hibernation_cleanup(void)710 serial_hibernation_cleanup(void)
711 {
712 uart_hibernation = false;
713 #if defined(APPLE_UART)
714 uart_base = gHibernateGlobals.hibUartRegVirtBase;
715 #endif /* defined(APPLE_UART) */
716 #if defined(DOCKCHANNEL_UART)
717 dockchannel_uart_base = gHibernateGlobals.dockChannelRegVirtBase;
718 #endif /* defined(DOCKCHANNEL_UART) */
719 }
720 #endif /* HIBERNATION */
721
722 int
serial_init(void)723 serial_init(void)
724 {
725 DTEntry entryP = NULL;
726 uint32_t prop_size;
727 vm_offset_t soc_base;
728 uintptr_t const *reg_prop;
729 uint32_t const *prop_value __unused = NULL;
730
731 struct pe_serial_functions *fns = gPESF;
732
733 /**
734 * Even if the serial devices have already been initialized on cold boot,
735 * when coming out of a sleep/wake, they'll need to be re-initialized. Since
736 * the uart_initted value is kept across a sleep/wake, always re-initialize
737 * to be safe.
738 */
739 if (uart_initted) {
740 #if HIBERNATION
741 if (uart_hibernation) {
742 serial_hibernation_cleanup();
743 }
744 #endif /* HIBERNATION */
745 while (fns != NULL) {
746 fns->init();
747 fns = fns->next;
748 }
749
750 return 1;
751 }
752
753 soc_base = pe_arm_get_soc_base_phys();
754
755 if (soc_base == 0) {
756 return 0;
757 }
758
759 PE_parse_boot_argn("disable-uart-irq", &disable_uart_irq, sizeof(disable_uart_irq));
760
761 #ifdef PI3_UART
762 if (SecureDTFindEntry("name", "gpio", &entryP) == kSuccess) {
763 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
764 pi3_gpio_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
765 }
766 if (SecureDTFindEntry("name", "aux", &entryP) == kSuccess) {
767 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
768 pi3_aux_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
769 }
770 if ((pi3_gpio_base_vaddr != 0) && (pi3_aux_base_vaddr != 0)) {
771 register_serial_functions(&pi3_uart_serial_functions);
772 }
773 #endif /* PI3_UART */
774
775 #ifdef VMAPPLE_UART
776 if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
777 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
778 vmapple_uart0_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
779 }
780
781 if (vmapple_uart0_base_vaddr != 0) {
782 register_serial_functions(&vmapple_uart_serial_functions);
783 }
784 #endif /* VMAPPLE_UART */
785
786 #ifdef DOCKCHANNEL_UART
787 uint32_t no_dockchannel_uart = 0;
788 if (SecureDTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) {
789 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
790 // Should be two reg entries
791 if (prop_size / sizeof(uintptr_t) != 4) {
792 panic("Malformed dockchannel-uart property");
793 }
794 dockchannel_uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
795 dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3));
796 PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart));
797 // Keep the old name for boot-arg
798 if (no_dockchannel_uart == 0) {
799 register_serial_functions(&dockchannel_serial_functions);
800 SecureDTGetProperty(entryP, "max-aop-clk", (void const **)&prop_value, &prop_size);
801 max_dockchannel_drain_period = (uint32_t)((prop_value)? (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD);
802 prop_value = NULL;
803 SecureDTGetProperty(entryP, "enable-sw-drain", (void const **)&prop_value, &prop_size);
804 use_sw_drain = (prop_value)? *prop_value : 0;
805 prop_value = NULL;
806 SecureDTGetProperty(entryP, "dock-wstat-mask", (void const **)&prop_value, &prop_size);
807 dock_wstat_mask = (prop_value)? *prop_value : 0x1ff;
808 prop_value = NULL;
809 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
810 if (prop_value) {
811 dockchannel_serial_functions.has_irq = true;
812 }
813 } else {
814 dockchannel_clear_intr();
815 }
816 // If no dockchannel-uart is found in the device tree, fall back
817 // to looking for the traditional UART serial console.
818 }
819
820 #endif /* DOCKCHANNEL_UART */
821
822 #ifdef APPLE_UART
823 char const *serial_compat = 0;
824 uint32_t use_legacy_uart = 0;
825
826 /* Check if we should enable this deprecated serial device. */
827 PE_parse_boot_argn("use-legacy-uart", &use_legacy_uart, sizeof(use_legacy_uart));
828
829 /*
830 * The boot serial port should have a property named "boot-console".
831 * If we don't find it there, look for "uart0" and "uart1".
832 */
833 if (SecureDTFindEntry("boot-console", NULL, &entryP) == kSuccess) {
834 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
835 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
836 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
837 } else if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
838 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
839 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
840 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
841 } else if (SecureDTFindEntry("name", "uart1", &entryP) == kSuccess) {
842 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
843 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
844 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
845 }
846
847 if (NULL != entryP) {
848 SecureDTGetProperty(entryP, "pclk", (void const **)&prop_value, &prop_size);
849 if (prop_value) {
850 dt_pclk = *prop_value;
851 }
852
853 prop_value = NULL;
854 SecureDTGetProperty(entryP, "sampling", (void const **)&prop_value, &prop_size);
855 if (prop_value) {
856 dt_sampling = *prop_value;
857 }
858
859 prop_value = NULL;
860 SecureDTGetProperty(entryP, "ubrdiv", (void const **)&prop_value, &prop_size);
861 if (prop_value) {
862 dt_ubrdiv = *prop_value;
863 }
864
865 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
866 if (prop_value) {
867 apple_serial_functions.has_irq = true;
868 }
869
870 /* Workaround to enable legacy serial for fastsim targets until clients migrate to dockchannels. */
871 SecureDTGetProperty(entryP, "enable-legacy-serial", (void const **)&prop_value, &prop_size);
872 if (prop_value) {
873 use_legacy_uart = 1;
874 }
875 }
876
877 if (use_legacy_uart && serial_compat && !strcmp(serial_compat, "uart-1,samsung")) {
878 register_serial_functions(&apple_serial_functions);
879 }
880 #endif /* APPLE_UART */
881
882 if (gPESF == NULL) {
883 return 0;
884 }
885
886 fns = gPESF;
887 while (fns != NULL) {
888 fns->init();
889 if (fns->has_irq) {
890 serial_irq_status |= fns->device; // serial_device_t is one-hot
891 }
892 fns = fns->next;
893 }
894
895 #if HIBERNATION
896 /* hibernation needs to know the UART register addresses since it can't directly use this serial driver */
897 if (dockchannel_uart_base) {
898 gHibernateGlobals.dockChannelRegPhysBase = ml_vtophys(dockchannel_uart_base);
899 gHibernateGlobals.dockChannelRegVirtBase = dockchannel_uart_base;
900 gHibernateGlobals.dockChannelWstatMask = dock_wstat_mask;
901 }
902 if (uart_base) {
903 gHibernateGlobals.hibUartRegPhysBase = ml_vtophys(uart_base);
904 gHibernateGlobals.hibUartRegVirtBase = uart_base;
905 }
906 #endif /* HIBERNATION */
907
908 uart_initted = true;
909
910 return 1;
911 }
912
913 /**
914 * Returns a deadline for the longest time the serial driver should wait for an
915 * interrupt for. This serves as a timeout for the IRQ to allow for the software
916 * drain timer that dockchannels supports.
917 *
918 * @param fns serial functions representing the device to find the deadline for
919 *
920 * @returns absolutetime deadline for this device's IRQ.
921 */
922 static uint64_t
serial_interrupt_deadline(__unused struct pe_serial_functions * fns)923 serial_interrupt_deadline(__unused struct pe_serial_functions *fns)
924 {
925 #if defined(DOCKCHANNEL_UART)
926 if (fns->device == SERIAL_DOCKCHANNEL && use_sw_drain) {
927 return dockchannel_drain_deadline;
928 }
929 #endif
930
931 /**
932 * Default to 1.5ms for all other devices. 1.5ms was chosen as the baudrate
933 * of the AppleSerialDevice is 115200, meaning that it should only take
934 * ~1.5ms to drain the 16 character buffer completely.
935 */
936 uint64_t timeout_interval;
937 nanoseconds_to_absolutetime(1500 * NSEC_PER_USEC, &timeout_interval);
938 return mach_absolute_time() + timeout_interval;
939 }
940
941 /**
942 * Goes to sleep waiting for an interrupt from a specificed serial device.
943 *
944 * @param fns serial functions representing the device to wait for
945 */
946 static void
serial_wait_for_interrupt(struct pe_serial_functions * fns)947 serial_wait_for_interrupt(struct pe_serial_functions *fns)
948 {
949 assert_wait_deadline(fns, THREAD_UNINT, serial_interrupt_deadline(fns));
950 if (!fns->transmit_ready()) {
951 fns->enable_irq();
952 thread_block(THREAD_CONTINUE_NULL);
953 } else {
954 clear_wait(current_thread(), THREAD_AWAKENED);
955 }
956 }
957
958 /**
959 * Output a character onto every registered serial interface.
960 *
961 * @param c The character to output.
962 * @param poll Whether the driver should poll to send the character or if it can
963 * wait for an interrupt
964 */
965 MARK_AS_HIBERNATE_TEXT void
uart_putc_options(char c,bool poll)966 uart_putc_options(char c, bool poll)
967 {
968 struct pe_serial_functions *fns = gPESF;
969
970 while (fns != NULL) {
971 while (!fns->transmit_ready()) {
972 if (!uart_hibernation) {
973 if (!poll && irq_available_and_ready(fns)) {
974 serial_wait_for_interrupt(fns);
975 } else {
976 serial_poll();
977 }
978 }
979 }
980 fns->transmit_data((uint8_t)c);
981 fns = fns->next;
982 }
983 }
984
985 /**
986 * Output a character onto every registered serial interface by polling.
987 *
988 * @param c The character to output.
989 */
990 void
uart_putc(char c)991 uart_putc(char c)
992 {
993 uart_putc_options(c, true);
994 }
995
996 /**
997 * Read a character from the first registered serial interface that has data
998 * available.
999 *
1000 * @return The character if any interfaces have data available, otherwise -1.
1001 */
1002 int
uart_getc(void)1003 uart_getc(void)
1004 {
1005 struct pe_serial_functions *fns = gPESF;
1006 while (fns != NULL) {
1007 if (fns->receive_ready()) {
1008 return (int)fns->receive_data();
1009 }
1010 fns = fns->next;
1011 }
1012 return -1;
1013 }
1014
1015 /**
1016 * Enables IRQs for a specific serial device and returns whether or not IRQs for
1017 * that device where enabled successfully. For a serial driver to have irqs
1018 * enabled, it must have the enable_irq, disable_irq, and acknowledge_irq
1019 * functions defined and the has_irq flag set.
1020 *
1021 * @param device Serial device to enable irqs on
1022 * @note This function should only be called from the AppleSerialShim kext
1023 */
1024 kern_return_t
serial_irq_enable(serial_device_t device)1025 serial_irq_enable(serial_device_t device)
1026 {
1027 struct pe_serial_functions *fns = get_serial_functions(device);
1028
1029 if (!fns || !fns->has_irq || disable_uart_irq) {
1030 return KERN_FAILURE;
1031 }
1032
1033 serial_irq_status &= ~device;
1034
1035 return KERN_SUCCESS;
1036 }
1037
1038 /**
1039 * Performs any actions needed to handle this IRQ. Wakes up the thread waiting
1040 * on the interrupt if one exists.
1041 *
1042 * @param device Serial device that generated the IRQ.
1043 * @note Interrupts will have already been cleared and disabled by serial_irq_filter.
1044 * @note This function should only be called from the AppleSerialShim kext.
1045 */
1046 kern_return_t
serial_irq_action(serial_device_t device)1047 serial_irq_action(serial_device_t device)
1048 {
1049 struct pe_serial_functions *fns = get_serial_functions(device);
1050
1051 if (!fns || !fns->has_irq) {
1052 return KERN_FAILURE;
1053 }
1054
1055 /**
1056 * Because IRQs are enabled only when we know a thread is about to sleep, we
1057 * can call wake up and reasonably expect there to be a thread waiting.
1058 */
1059 thread_wakeup(fns);
1060
1061 return KERN_SUCCESS;
1062 }
1063
1064 /**
1065 * Returns true if the pending IRQ for device is one that can be handled by the
1066 * platform serial driver.
1067 *
1068 * @param device Serial device that generated the IRQ.
1069 * @note This function is called from a primary interrupt context and should be
1070 * kept lightweight.
1071 * @note This function should only be called from the AppleSerialShim kext
1072 */
1073 bool
serial_irq_filter(serial_device_t device)1074 serial_irq_filter(serial_device_t device)
1075 {
1076 struct pe_serial_functions *fns = get_serial_functions(device);
1077
1078 if (!fns || !fns->has_irq) {
1079 return false;
1080 }
1081
1082 /**
1083 * Disable IRQs until next time a thread waits for an interrupt to prevent an interrupt storm.
1084 */
1085 const bool had_irqs_enabled = fns->disable_irq();
1086 const bool was_our_interrupt = fns->acknowledge_irq();
1087
1088 /* Re-enable IRQs if the interrupt wasn't for us. */
1089 if (had_irqs_enabled && !was_our_interrupt) {
1090 fns->enable_irq();
1091 }
1092
1093 return was_our_interrupt;
1094 }
1095
1096 /**
1097 * Prepares all serial devices to go to sleep by draining the hardware FIFOs
1098 * and disabling interrupts.
1099 */
1100 void
serial_go_to_sleep(void)1101 serial_go_to_sleep(void)
1102 {
1103 struct pe_serial_functions *fns = gPESF;
1104 while (fns != NULL) {
1105 if (irq_available_and_ready(fns)) {
1106 fns->disable_irq();
1107 }
1108 fns = fns->next;
1109 }
1110
1111 #ifdef APPLE_UART
1112 /* APPLE_UART needs to drain FIFO before sleeping */
1113 if (get_serial_functions(SERIAL_APPLE_UART)) {
1114 apple_uart_drain_fifo();
1115 }
1116 #endif /* APPLE_UART */
1117 }
1118