1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * This file contains the low-level serial drivers used on ARM/ARM64 devices.
5 * The generic serial console code in osfmk/console/serial_console.c will call
6 * into this code to transmit and receive serial data.
7 *
8 * Logging can be performed on multiple serial interfaces at once through a
9 * method called serial multiplexing. This is implemented by enumerating which
10 * serial interfaces are available on boot and registering them into a linked
11 * list of interfaces pointed to by gPESF. When outputting or receiving
12 * characters, each interface is queried in turn.
13 *
14 * Please view doc/arm_serial.md for an in-depth description of these drivers.
15 */
16 #include <kern/clock.h>
17 #include <kern/debug.h>
18 #include <libkern/OSBase.h>
19 #include <libkern/section_keywords.h>
20 #include <mach/mach_time.h>
21 #include <machine/atomic.h>
22 #include <machine/machine_routines.h>
23 #include <pexpert/pexpert.h>
24 #include <pexpert/protos.h>
25 #include <pexpert/device_tree.h>
26 #if defined __arm__
27 #include <arm/caches_internal.h>
28 #include <arm/machine_routines.h>
29 #include <arm/proc_reg.h>
30 #include <pexpert/arm/board_config.h>
31 #include <vm/pmap.h>
32 #elif defined __arm64__
33 #include <pexpert/arm/consistent_debug.h>
34 #include <pexpert/arm64/board_config.h>
35 #include <arm64/proc_reg.h>
36 #endif
37 #include <pexpert/arm/protos.h>
38 #include <kern/sched_prim.h>
39 #if HIBERNATION
40 #include <machine/pal_hibernate.h>
41 #endif /* HIBERNATION */
42
43 struct pe_serial_functions {
44 /* Initialize the underlying serial hardware. */
45 void (*init) (void);
46
47 /* Return a non-zero value if the serial interface is ready to send more data. */
48 unsigned int (*transmit_ready) (void);
49
50 /* Write a single byte of data to serial. */
51 void (*transmit_data) (uint8_t c);
52
53 /* Return a non-zero value if there's a byte of data available. */
54 unsigned int (*receive_ready) (void);
55
56 /* Read a single byte from serial. */
57 uint8_t (*receive_data) (void);
58
59 /* Enables IRQs from this device. */
60 void (*enable_irq) (void);
61
62 /* Disables IRQs from this device. */
63 void (*disable_irq) (void);
64
65 /* Clears IRQs from this device. */
66 bool (*acknowledge_irq) (void);
67
68 /**
69 * Whether this serial driver can handle irqs. This value should be set by
70 * querying the device tree to see if the serial device has interrupts
71 * associated with it.
72 *
73 * For a device to support IRQs:
74 * - enable_irq, disable_irq, and acknowledge_irq must be non-null
75 * - The AppleSerialShim kext must be able to match to the serial device
76 * in the IORegistry and call serial_enable_irq with the proper
77 * serial_device_t
78 * - The device tree entry for the serial device should have an interrupt
79 * associated with it.
80 */
81 bool has_irq;
82
83 /* enum identifying which serial device these functions belong to. */
84 serial_device_t device;
85
86 /* Pointer to the next serial interface in the linked-list. */
87 struct pe_serial_functions *next;
88 };
89
90 MARK_AS_HIBERNATE_DATA_CONST_LATE static struct pe_serial_functions* gPESF = NULL;
91
92 /**
93 * Whether uart has been initialized already. This value is kept across a
94 * sleep/wake cycle so we know we need to reinitialize when serial_init is
95 * called again after wake.
96 */
97 MARK_AS_HIBERNATE_DATA static bool uart_initted = false;
98
99 /* Whether uart should run in simple mode that works during hibernation resume. */
100 MARK_AS_HIBERNATE_DATA static bool uart_hibernation = false;
101
102 /**
103 * Used to track if all IRQs have been initialized. Each bit of this variable
104 * represents whether or not a serial device that reports supporting IRQs has
105 * been initialized yet (1 -> not initialized, 0 -> initialized)
106 */
107 static uint32_t serial_irq_status = 0;
108
109 /**
110 * Set by the 'disable-uart-irq' boot-arg to force serial IRQs into polling mode
111 * by preventing the serial driver shim kext from registering itself with
112 * serial_enable_irq.
113 */
114 static bool disable_uart_irq = 0;
115
116 /**
117 * Indicates whether or not a given device's irqs have been set up by calling
118 * serial_enable_irq for that particular device.
119 *
120 * @param device_fns Serial functions for the device that is being checked
121 * @return Whether or not the irqs have been initialized for that device
122 */
123 static bool
irq_initialized(struct pe_serial_functions * device_fns)124 irq_initialized(struct pe_serial_functions *device_fns)
125 {
126 return (serial_irq_status & device_fns->device) == 0;
127 }
128
129 /**
130 * Indicates whether or not a given device supports irqs and if they are ready
131 * to be used.
132 *
133 * @param device_fns Serial functions for the device that is being checked
134 * @return Whether or not the device can and will send IRQs.
135 */
136 static bool
irq_available_and_ready(struct pe_serial_functions * device_fns)137 irq_available_and_ready(struct pe_serial_functions *device_fns)
138 {
139 return device_fns->has_irq && irq_initialized(device_fns);
140 }
141
142 /**
143 * Searches through the global serial functions list and returns the serial function for a particular device
144 *
145 * @param device The device identifier to search for
146 * @return Serial functions for the specified device
147 */
148 static struct pe_serial_functions *
get_serial_functions(serial_device_t device)149 get_serial_functions(serial_device_t device)
150 {
151 struct pe_serial_functions *fns = gPESF;
152 while (fns != NULL) {
153 if (fns->device == device) {
154 return fns;
155 }
156 fns = fns->next;
157 }
158 return NULL;
159 }
160
161 /**
162 * The action to take when polling and waiting for a serial device to be ready
163 * for output. On ARM64, takes a WFE because the WFE timeout will wake us up in
164 * the worst case. On ARMv7 devices, we need to hot poll.
165 */
166 static void
serial_poll(void)167 serial_poll(void)
168 {
169 #if __arm64__
170 __builtin_arm_wfe();
171 #endif
172 }
173
174 /**
175 * This ensures that if we have a future product that supports hibernation, but
176 * doesn't support either UART serial or dock-channels, then hibernation will
177 * gracefully fall back to the serial method that is supported.
178 */
179 #if HIBERNATION || defined(APPLE_UART)
180 MARK_AS_HIBERNATE_DATA static vm_offset_t uart_base = 0;
181 #endif /* HIBERNATION || defined(APPLE_UART) */
182
183 #if HIBERNATION || defined(DOCKCHANNEL_UART)
184 MARK_AS_HIBERNATE_DATA static vm_offset_t dockchannel_uart_base = 0;
185 #endif /* HIBERNATION || defined(DOCKCHANNEL_UART) */
186
187 /*****************************************************************************/
188
189 #ifdef APPLE_UART
190
191 static int32_t dt_pclk = -1;
192 static int32_t dt_sampling = -1;
193 static int32_t dt_ubrdiv = -1;
194
195 static void apple_uart_set_baud_rate(uint32_t baud_rate);
196
197 static void
apple_uart_init(void)198 apple_uart_init(void)
199 {
200 uint32_t ucon0 = 0x405; /* NCLK, No interrupts, No DMA - just polled */
201
202 rULCON0 = 0x03; /* 81N, not IR */
203
204 // Override with pclk dt entry
205 if (dt_pclk != -1) {
206 ucon0 = ucon0 & ~0x400;
207 }
208
209 rUCON0 = ucon0;
210 rUMCON0 = 0x00; /* Clear Flow Control */
211
212 apple_uart_set_baud_rate(115200);
213
214 rUFCON0 = 0x07; /* Clear & Enable FIFOs */
215 rUMCON0 = 0x01; /* Assert RTS on UART0 */
216 }
217
218 static void
apple_uart_enable_irq(void)219 apple_uart_enable_irq(void)
220 {
221 /* sets Tx FIFO watermark to 0 bytes so interrupt is sent when FIFO empty */
222 rUFCON0 &= ~(0xC0);
223
224 /* Enables Tx interrupt */
225 rUCON0 |= 0x2000;
226 }
227
228 static void
apple_uart_disable_irq(void)229 apple_uart_disable_irq(void)
230 {
231 /* Disables Tx interrupts */
232 rUCON0 &= ~(0x2000);
233 }
234
235 static bool
apple_uart_ack_irq(void)236 apple_uart_ack_irq(void)
237 {
238 rUTRSTAT0 |= 0x20;
239 return true;
240 }
241
242 static void
apple_uart_drain_fifo(void)243 apple_uart_drain_fifo(void)
244 {
245 /* wait while Tx FIFO is full or the FIFO count != 0 */
246 while ((rUFSTAT0 & 0x2F0)) {
247 serial_poll();
248 }
249 }
250
251 static void
apple_uart_set_baud_rate(uint32_t baud_rate)252 apple_uart_set_baud_rate(uint32_t baud_rate)
253 {
254 uint32_t div = 0;
255 uint32_t uart_clock = 0;
256 uint32_t sample_rate = 16;
257
258 if (baud_rate < 300) {
259 baud_rate = 9600;
260 }
261
262 if (rUCON0 & 0x400) {
263 // NCLK
264 uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz;
265 } else {
266 // PCLK
267 uart_clock = (uint32_t)gPEClockFrequencyInfo.prf_frequency_hz;
268 }
269
270 if (dt_sampling != -1) {
271 // Use the sampling rate specified in the Device Tree
272 sample_rate = dt_sampling & 0xf;
273 }
274
275 if (dt_ubrdiv != -1) {
276 // Use the ubrdiv specified in the Device Tree
277 div = dt_ubrdiv & 0xffff;
278 } else {
279 // Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1
280 div = uart_clock / (baud_rate * sample_rate);
281
282 uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate);
283 uint32_t baud_low = uart_clock / ((div + 1) * sample_rate);
284
285 // Adjust div to get the closest target baudrate
286 if ((baud_rate - baud_low) > (actual_baud - baud_rate)) {
287 div--;
288 }
289 }
290
291 // Sample Rate [19:16], UBRDIV [15:0]
292 rUBRDIV0 = ((16 - sample_rate) << 16) | div;
293 }
294
295 MARK_AS_HIBERNATE_TEXT static unsigned int
apple_uart_tr0(void)296 apple_uart_tr0(void)
297 {
298 /* UART is ready unless the FIFO is full. */
299 return (rUFSTAT0 & 0x200) == 0;
300 }
301
302 MARK_AS_HIBERNATE_TEXT static void
apple_uart_td0(uint8_t c)303 apple_uart_td0(uint8_t c)
304 {
305 rUTXH0 = c;
306 }
307
308 static unsigned int
apple_uart_rr0(void)309 apple_uart_rr0(void)
310 {
311 /* Receive is ready when there are >0 bytes in the receive FIFO */
312 return rUFSTAT0 & 0xF;
313 }
314
315 static uint8_t
apple_uart_rd0(void)316 apple_uart_rd0(void)
317 {
318 return (uint8_t)rURXH0;
319 }
320
321 MARK_AS_HIBERNATE_DATA_CONST_LATE
322 static struct pe_serial_functions apple_serial_functions =
323 {
324 .init = apple_uart_init,
325 .transmit_ready = apple_uart_tr0,
326 .transmit_data = apple_uart_td0,
327 .receive_ready = apple_uart_rr0,
328 .receive_data = apple_uart_rd0,
329 .enable_irq = apple_uart_enable_irq,
330 .disable_irq = apple_uart_disable_irq,
331 .acknowledge_irq = apple_uart_ack_irq,
332 .device = SERIAL_APPLE_UART
333 };
334
335 #endif /* APPLE_UART */
336
337 /*****************************************************************************/
338
339 static void
dcc_uart_init(void)340 dcc_uart_init(void)
341 {
342 }
343
344 static uint8_t
read_dtr(void)345 read_dtr(void)
346 {
347 #ifdef __arm__
348 uint8_t c;
349 __asm__ volatile (
350 "mrc p14, 0, %0, c0, c5\n"
351 : "=r"(c));
352 return c;
353 #else
354 panic_unimplemented();
355 return 0;
356 #endif
357 }
358 static void
write_dtr(uint8_t c)359 write_dtr(uint8_t c)
360 {
361 #ifdef __arm__
362 __asm__ volatile (
363 "mcr p14, 0, %0, c0, c5\n"
364 :
365 :"r"(c));
366 #else
367 (void)c;
368 panic_unimplemented();
369 #endif
370 }
371
372 static unsigned int
dcc_tr0(void)373 dcc_tr0(void)
374 {
375 #ifdef __arm__
376 return !(arm_debug_read_dscr() & ARM_DBGDSCR_TXFULL);
377 #else
378 panic_unimplemented();
379 return 0;
380 #endif
381 }
382
383 static void
dcc_td0(uint8_t c)384 dcc_td0(uint8_t c)
385 {
386 write_dtr(c);
387 }
388
389 static unsigned int
dcc_rr0(void)390 dcc_rr0(void)
391 {
392 #ifdef __arm__
393 return arm_debug_read_dscr() & ARM_DBGDSCR_RXFULL;
394 #else
395 panic_unimplemented();
396 return 0;
397 #endif
398 }
399
400 static uint8_t
dcc_rd0(void)401 dcc_rd0(void)
402 {
403 return read_dtr();
404 }
405
406 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) dcc_serial_functions =
407 {
408 .init = dcc_uart_init,
409 .transmit_ready = dcc_tr0,
410 .transmit_data = dcc_td0,
411 .receive_ready = dcc_rr0,
412 .receive_data = dcc_rd0,
413 .device = SERIAL_DCC_UART
414 };
415
416 /*****************************************************************************/
417
418 #ifdef DOCKCHANNEL_UART
419 #define DOCKCHANNEL_WR_MAX_STALL_US (30*1000)
420
421 static vm_offset_t dock_agent_base;
422 static uint32_t max_dockchannel_drain_period;
423 static uint64_t dockchannel_drain_deadline; // Deadline for external agent to drain before a software drain occurs
424 static bool use_sw_drain;
425 static uint32_t dock_wstat_mask;
426 static uint64_t prev_dockchannel_spaces; // Previous w_stat level of the DockChannel.
427 static uint64_t dockchannel_stall_grace;
428 MARK_AS_HIBERNATE_DATA static bool use_sw_drain;
429 MARK_AS_HIBERNATE_DATA static uint32_t dock_wstat_mask;
430
431 // forward reference
432 static struct pe_serial_functions dockchannel_serial_functions;
433
434 //=======================
435 // Local funtions
436 //=======================
437
438 static int
dockchannel_drain_on_stall()439 dockchannel_drain_on_stall()
440 {
441 // Called when DockChannel runs out of spaces.
442 // Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves.
443 // Return number of bytes drained.
444
445 if (mach_absolute_time() >= dockchannel_drain_deadline) {
446 // It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO
447 // Drop a character.
448 (void)rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
449 os_atomic_inc(&prev_dockchannel_spaces, relaxed);
450 return 1;
451 }
452 return 0;
453 }
454
455 static void
dockchannel_clear_intr(void)456 dockchannel_clear_intr(void)
457 {
458 rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3);
459 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3;
460 rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3);
461 rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3;
462 }
463
464 static void
dockchannel_disable_irq(void)465 dockchannel_disable_irq(void)
466 {
467 rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x1);
468 }
469
470 static void
dockchannel_enable_irq(void)471 dockchannel_enable_irq(void)
472 {
473 // set interrupt to be when fifo has 255 empty
474 rDOCKCHANNELS_DEV_WR_WATERMARK(DOCKCHANNEL_UART_CHANNEL) = 0xFF;
475 rDOCKCHANNELS_AGENT_AP_INTR_CTRL |= 0x1;
476 }
477
478 static bool
dockchannel_ack_irq(void)479 dockchannel_ack_irq(void)
480 {
481 /* First check if the IRQ is for the kernel */
482 if (rDOCKCHANNELS_AGENT_AP_INTR_STATUS & 0x1) {
483 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x1;
484 return true;
485 }
486 return false;
487 }
488
489 MARK_AS_HIBERNATE_TEXT static void
dockchannel_transmit_data(uint8_t c)490 dockchannel_transmit_data(uint8_t c)
491 {
492 rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)c;
493
494 if (use_sw_drain && !uart_hibernation) {
495 os_atomic_dec(&prev_dockchannel_spaces, relaxed); // After writing a byte we have one fewer space than previously expected.
496 }
497 }
498
499 static unsigned int
dockchannel_receive_ready(void)500 dockchannel_receive_ready(void)
501 {
502 return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f;
503 }
504
505 static uint8_t
dockchannel_receive_data(void)506 dockchannel_receive_data(void)
507 {
508 return (uint8_t)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL) >> 8) & 0xff);
509 }
510
511 MARK_AS_HIBERNATE_TEXT static unsigned int
dockchannel_transmit_ready(void)512 dockchannel_transmit_ready(void)
513 {
514 uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
515
516 if (!uart_hibernation) {
517 if (use_sw_drain) {
518 if (spaces > prev_dockchannel_spaces) {
519 // More spaces showed up. That can only mean someone read the FIFO.
520 // Note that if the DockFIFO is empty we cannot tell if someone is listening,
521 // we can only give them the benefit of the doubt.
522 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
523 }
524 prev_dockchannel_spaces = spaces;
525 return spaces || dockchannel_drain_on_stall();
526 }
527 }
528
529 return spaces;
530 }
531
532 static void
dockchannel_init(void)533 dockchannel_init(void)
534 {
535 if (use_sw_drain) {
536 nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace);
537 }
538
539 // Clear all interrupt enable and status bits
540 dockchannel_clear_intr();
541
542 // Setup DRAIN timer
543 rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period;
544
545 // Drain timer doesn't get loaded with value from drain period register if fifo
546 // is already full. Drop a character from the fifo.
547 rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
548 }
549
550 MARK_AS_HIBERNATE_DATA_CONST_LATE
551 static struct pe_serial_functions dockchannel_serial_functions =
552 {
553 .init = dockchannel_init,
554 .transmit_ready = dockchannel_transmit_ready,
555 .transmit_data = dockchannel_transmit_data,
556 .receive_ready = dockchannel_receive_ready,
557 .receive_data = dockchannel_receive_data,
558 .enable_irq = dockchannel_enable_irq,
559 .disable_irq = dockchannel_disable_irq,
560 .acknowledge_irq = dockchannel_ack_irq,
561 .device = SERIAL_DOCKCHANNEL
562 };
563
564 #endif /* DOCKCHANNEL_UART */
565
566 /****************************************************************************/
567 #ifdef PI3_UART
568 vm_offset_t pi3_gpio_base_vaddr = 0;
569 vm_offset_t pi3_aux_base_vaddr = 0;
570 static unsigned int
pi3_uart_tr0(void)571 pi3_uart_tr0(void)
572 {
573 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x20;
574 }
575
576 static void
pi3_uart_td0(uint8_t c)577 pi3_uart_td0(uint8_t c)
578 {
579 BCM2837_PUT32(BCM2837_AUX_MU_IO_REG_V, (uint32_t) c);
580 }
581
582 static unsigned int
pi3_uart_rr0(void)583 pi3_uart_rr0(void)
584 {
585 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x01;
586 }
587
588 static uint8_t
pi3_uart_rd0(void)589 pi3_uart_rd0(void)
590 {
591 return (uint8_t) BCM2837_GET32(BCM2837_AUX_MU_IO_REG_V);
592 }
593
594 static void
pi3_uart_init(void)595 pi3_uart_init(void)
596 {
597 // Scratch variable
598 uint32_t i;
599
600 // Reset mini uart registers
601 BCM2837_PUT32(BCM2837_AUX_ENABLES_V, 1);
602 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 0);
603 BCM2837_PUT32(BCM2837_AUX_MU_LCR_REG_V, 3);
604 BCM2837_PUT32(BCM2837_AUX_MU_MCR_REG_V, 0);
605 BCM2837_PUT32(BCM2837_AUX_MU_IER_REG_V, 0);
606 BCM2837_PUT32(BCM2837_AUX_MU_IIR_REG_V, 0xC6);
607 BCM2837_PUT32(BCM2837_AUX_MU_BAUD_REG_V, 270);
608
609 i = (uint32_t)BCM2837_FSEL_REG(14);
610 // Configure GPIOs 14 & 15 for alternate function 5
611 i &= ~(BCM2837_FSEL_MASK(14));
612 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(14));
613 i &= ~(BCM2837_FSEL_MASK(15));
614 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(15));
615
616 BCM2837_PUT32(BCM2837_FSEL_REG(14), i);
617
618 BCM2837_PUT32(BCM2837_GPPUD_V, 0);
619
620 // Barrier before AP spinning for 150 cycles
621 __builtin_arm_isb(ISB_SY);
622
623 for (i = 0; i < 150; i++) {
624 asm volatile ("add x0, x0, xzr");
625 }
626
627 __builtin_arm_isb(ISB_SY);
628
629 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, (1 << 14) | (1 << 15));
630
631 __builtin_arm_isb(ISB_SY);
632
633 for (i = 0; i < 150; i++) {
634 asm volatile ("add x0, x0, xzr");
635 }
636
637 __builtin_arm_isb(ISB_SY);
638
639 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, 0);
640
641 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 3);
642 }
643
644 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) pi3_uart_serial_functions =
645 {
646 .init = pi3_uart_init,
647 .transmit_ready = pi3_uart_tr0,
648 .transmit_data = pi3_uart_td0,
649 .receive_ready = pi3_uart_rr0,
650 .receive_data = pi3_uart_rd0,
651 .device = SERIAL_PI3_UART
652 };
653
654 #endif /* PI3_UART */
655
656 /*****************************************************************************/
657
658 #ifdef VMAPPLE_UART
659
660 static vm_offset_t vmapple_uart0_base_vaddr = 0;
661
662 #define PL011_LCR_WORD_LENGTH_8 0x60u
663 #define PL011_LCR_FIFO_DISABLE 0x00u
664
665 #define PL011_LCR_FIFO_ENABLE 0x10u
666
667 #define PL011_LCR_ONE_STOP_BIT 0x00u
668 #define PL011_LCR_PARITY_DISABLE 0x00u
669 #define PL011_LCR_BREAK_DISABLE 0x00u
670 #define PL011_IBRD_DIV_38400 0x27u
671 #define PL011_FBRD_DIV_38400 0x09u
672 #define PL011_ICR_CLR_ALL_IRQS 0x07ffu
673 #define PL011_CR_UART_ENABLE 0x01u
674 #define PL011_CR_TX_ENABLE 0x100u
675 #define PL011_CR_RX_ENABLE 0x200u
676
677 #define VMAPPLE_UART0_DR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x00))
678 #define VMAPPLE_UART0_ECR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x04))
679 #define VMAPPLE_UART0_FR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x18))
680 #define VMAPPLE_UART0_IBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x24))
681 #define VMAPPLE_UART0_FBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x28))
682 #define VMAPPLE_UART0_LCR_H *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x2c))
683 #define VMAPPLE_UART0_CR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x30))
684 #define VMAPPLE_UART0_TIMSC *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x38))
685 #define VMAPPLE_UART0_ICR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x44))
686
687 static unsigned int
vmapple_uart_transmit_ready(void)688 vmapple_uart_transmit_ready(void)
689 {
690 return (unsigned int) !(VMAPPLE_UART0_FR & 0x20);
691 }
692
693 static void
vmapple_uart_transmit_data(uint8_t c)694 vmapple_uart_transmit_data(uint8_t c)
695 {
696 VMAPPLE_UART0_DR = (uint32_t) c;
697 }
698
699 static unsigned int
vmapple_uart_receive_ready(void)700 vmapple_uart_receive_ready(void)
701 {
702 return (unsigned int) !(VMAPPLE_UART0_FR & 0x10);
703 }
704
705 static uint8_t
vmapple_uart_receive_data(void)706 vmapple_uart_receive_data(void)
707 {
708 return (uint8_t) (VMAPPLE_UART0_DR & 0xff);
709 }
710
711 static void
vmapple_uart_init(void)712 vmapple_uart_init(void)
713 {
714 VMAPPLE_UART0_CR = 0x0;
715 VMAPPLE_UART0_ECR = 0x0;
716 VMAPPLE_UART0_LCR_H = (
717 PL011_LCR_WORD_LENGTH_8 |
718 PL011_LCR_FIFO_ENABLE |
719 PL011_LCR_ONE_STOP_BIT |
720 PL011_LCR_PARITY_DISABLE |
721 PL011_LCR_BREAK_DISABLE
722 );
723 VMAPPLE_UART0_IBRD = PL011_IBRD_DIV_38400;
724 VMAPPLE_UART0_FBRD = PL011_FBRD_DIV_38400;
725 VMAPPLE_UART0_TIMSC = 0x0;
726 VMAPPLE_UART0_ICR = PL011_ICR_CLR_ALL_IRQS;
727 VMAPPLE_UART0_CR = (
728 PL011_CR_UART_ENABLE |
729 PL011_CR_TX_ENABLE |
730 PL011_CR_RX_ENABLE
731 );
732 }
733
734 SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) vmapple_uart_serial_functions =
735 {
736 .init = vmapple_uart_init,
737 .transmit_ready = vmapple_uart_transmit_ready,
738 .transmit_data = vmapple_uart_transmit_data,
739 .receive_ready = vmapple_uart_receive_ready,
740 .receive_data = vmapple_uart_receive_data,
741 .device = SERIAL_VMAPPLE_UART
742 };
743
744 #endif /* VMAPPLE_UART */
745
746 /*****************************************************************************/
747
748 static void
register_serial_functions(struct pe_serial_functions * fns)749 register_serial_functions(struct pe_serial_functions *fns)
750 {
751 fns->next = gPESF;
752 gPESF = fns;
753 }
754
755 #if HIBERNATION
756 /**
757 * Transitions the serial driver into a mode that can be run in the hibernation
758 * resume context. In this mode, the serial driver runs at a barebones level
759 * without making sure the serial devices are properly initialized or utilizing
760 * features such as the software drain timer for dockchannels.
761 *
762 * Upon the next call to serial_init (once the hibernation image has been
763 * loaded), this mode is exited and we return to the normal operation of the
764 * driver.
765 */
766 MARK_AS_HIBERNATE_TEXT void
serial_hibernation_init(void)767 serial_hibernation_init(void)
768 {
769 uart_hibernation = true;
770 #if defined(APPLE_UART)
771 uart_base = gHibernateGlobals.hibUartRegPhysBase;
772 #endif /* defined(APPLE_UART) */
773 #if defined(DOCKCHANNEL_UART)
774 dockchannel_uart_base = gHibernateGlobals.dockChannelRegPhysBase;
775 #endif /* defined(DOCKCHANNEL_UART) */
776 }
777
778 /**
779 * Transitions the serial driver back to non-hibernation mode so it can resume
780 * normal operations. Should only be called from serial_init on a hibernation
781 * resume.
782 */
783 MARK_AS_HIBERNATE_TEXT static void
serial_hibernation_cleanup(void)784 serial_hibernation_cleanup(void)
785 {
786 uart_hibernation = false;
787 #if defined(APPLE_UART)
788 uart_base = gHibernateGlobals.hibUartRegVirtBase;
789 #endif /* defined(APPLE_UART) */
790 #if defined(DOCKCHANNEL_UART)
791 dockchannel_uart_base = gHibernateGlobals.dockChannelRegVirtBase;
792 #endif /* defined(DOCKCHANNEL_UART) */
793 }
794 #endif /* HIBERNATION */
795
796 int
serial_init(void)797 serial_init(void)
798 {
799 DTEntry entryP = NULL;
800 uint32_t prop_size;
801 vm_offset_t soc_base;
802 uintptr_t const *reg_prop;
803 uint32_t const *prop_value __unused = NULL;
804 uint32_t dccmode;
805
806 struct pe_serial_functions *fns = gPESF;
807
808 /**
809 * Even if the serial devices have already been initialized on cold boot,
810 * when coming out of a sleep/wake, they'll need to be re-initialized. Since
811 * the uart_initted value is kept across a sleep/wake, always re-initialize
812 * to be safe.
813 */
814 if (uart_initted) {
815 #if HIBERNATION
816 if (uart_hibernation) {
817 serial_hibernation_cleanup();
818 }
819 #endif /* HIBERNATION */
820 while (fns != NULL) {
821 fns->init();
822 fns = fns->next;
823 }
824
825 return 1;
826 }
827
828 dccmode = 0;
829 if (PE_parse_boot_argn("dcc", &dccmode, sizeof(dccmode))) {
830 register_serial_functions(&dcc_serial_functions);
831 }
832
833 soc_base = pe_arm_get_soc_base_phys();
834
835 if (soc_base == 0) {
836 return 0;
837 }
838
839 PE_parse_boot_argn("disable-uart-irq", &disable_uart_irq, sizeof(disable_uart_irq));
840
841 #ifdef PI3_UART
842 if (SecureDTFindEntry("name", "gpio", &entryP) == kSuccess) {
843 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
844 pi3_gpio_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
845 }
846 if (SecureDTFindEntry("name", "aux", &entryP) == kSuccess) {
847 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
848 pi3_aux_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
849 }
850 if ((pi3_gpio_base_vaddr != 0) && (pi3_aux_base_vaddr != 0)) {
851 register_serial_functions(&pi3_uart_serial_functions);
852 }
853 #endif /* PI3_UART */
854
855 #ifdef VMAPPLE_UART
856 if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
857 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
858 vmapple_uart0_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
859 }
860
861 if (vmapple_uart0_base_vaddr != 0) {
862 register_serial_functions(&vmapple_uart_serial_functions);
863 }
864 #endif /* VMAPPLE_UART */
865
866 #ifdef DOCKCHANNEL_UART
867 uint32_t no_dockchannel_uart = 0;
868 if (SecureDTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) {
869 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
870 // Should be two reg entries
871 if (prop_size / sizeof(uintptr_t) != 4) {
872 panic("Malformed dockchannel-uart property");
873 }
874 dockchannel_uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
875 dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3));
876 PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart));
877 // Keep the old name for boot-arg
878 if (no_dockchannel_uart == 0) {
879 register_serial_functions(&dockchannel_serial_functions);
880 SecureDTGetProperty(entryP, "max-aop-clk", (void const **)&prop_value, &prop_size);
881 max_dockchannel_drain_period = (uint32_t)((prop_value)? (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD);
882 prop_value = NULL;
883 SecureDTGetProperty(entryP, "enable-sw-drain", (void const **)&prop_value, &prop_size);
884 use_sw_drain = (prop_value)? *prop_value : 0;
885 prop_value = NULL;
886 SecureDTGetProperty(entryP, "dock-wstat-mask", (void const **)&prop_value, &prop_size);
887 dock_wstat_mask = (prop_value)? *prop_value : 0x1ff;
888 prop_value = NULL;
889 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
890 if (prop_value) {
891 dockchannel_serial_functions.has_irq = true;
892 }
893 } else {
894 dockchannel_clear_intr();
895 }
896 // If no dockchannel-uart is found in the device tree, fall back
897 // to looking for the traditional UART serial console.
898 }
899
900 #endif /* DOCKCHANNEL_UART */
901
902 #ifdef APPLE_UART
903 char const *serial_compat = 0;
904
905 /*
906 * The boot serial port should have a property named "boot-console".
907 * If we don't find it there, look for "uart0" and "uart1".
908 */
909 if (SecureDTFindEntry("boot-console", NULL, &entryP) == kSuccess) {
910 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
911 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
912 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
913 } else if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
914 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
915 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
916 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
917 } else if (SecureDTFindEntry("name", "uart1", &entryP) == kSuccess) {
918 SecureDTGetProperty(entryP, "reg", (void const **)®_prop, &prop_size);
919 uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
920 SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
921 }
922
923 if (NULL != entryP) {
924 SecureDTGetProperty(entryP, "pclk", (void const **)&prop_value, &prop_size);
925 if (prop_value) {
926 dt_pclk = *prop_value;
927 }
928
929 prop_value = NULL;
930 SecureDTGetProperty(entryP, "sampling", (void const **)&prop_value, &prop_size);
931 if (prop_value) {
932 dt_sampling = *prop_value;
933 }
934
935 prop_value = NULL;
936 SecureDTGetProperty(entryP, "ubrdiv", (void const **)&prop_value, &prop_size);
937 if (prop_value) {
938 dt_ubrdiv = *prop_value;
939 }
940
941 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
942 if (prop_value) {
943 apple_serial_functions.has_irq = true;
944 }
945 }
946
947 if (serial_compat && !strcmp(serial_compat, "uart-1,samsung")) {
948 register_serial_functions(&apple_serial_functions);
949 }
950 #endif /* APPLE_UART */
951
952 if (gPESF == NULL) {
953 return 0;
954 }
955
956 fns = gPESF;
957 while (fns != NULL) {
958 fns->init();
959 if (fns->has_irq) {
960 serial_irq_status |= fns->device; // serial_device_t is one-hot
961 }
962 fns = fns->next;
963 }
964
965 #if HIBERNATION
966 /* hibernation needs to know the UART register addresses since it can't directly use this serial driver */
967 if (dockchannel_uart_base) {
968 gHibernateGlobals.dockChannelRegPhysBase = ml_vtophys(dockchannel_uart_base);
969 gHibernateGlobals.dockChannelRegVirtBase = dockchannel_uart_base;
970 gHibernateGlobals.dockChannelWstatMask = dock_wstat_mask;
971 }
972 if (uart_base) {
973 gHibernateGlobals.hibUartRegPhysBase = ml_vtophys(uart_base);
974 gHibernateGlobals.hibUartRegVirtBase = uart_base;
975 }
976 #endif /* HIBERNATION */
977
978 uart_initted = true;
979
980 return 1;
981 }
982
983 /**
984 * Returns a deadline for the longest time the serial driver should wait for an
985 * interrupt for. This serves as a timeout for the IRQ to allow for the software
986 * drain timer that dockchannels supports.
987 *
988 * @param fns serial functions representing the device to find the deadline for
989 *
990 * @returns absolutetime deadline for this device's IRQ.
991 */
992 static uint64_t
serial_interrupt_deadline(__unused struct pe_serial_functions * fns)993 serial_interrupt_deadline(__unused struct pe_serial_functions *fns)
994 {
995 #if defined(DOCKCHANNEL_UART)
996 if (fns->device == SERIAL_DOCKCHANNEL && use_sw_drain) {
997 return dockchannel_drain_deadline;
998 }
999 #endif
1000
1001 /**
1002 * Default to 1.5ms for all other devices. 1.5ms was chosen as the baudrate
1003 * of the AppleSerialDevice is 115200, meaning that it should only take
1004 * ~1.5ms to drain the 16 character buffer completely.
1005 */
1006 uint64_t timeout_interval;
1007 nanoseconds_to_absolutetime(1500 * NSEC_PER_USEC, &timeout_interval);
1008 return mach_absolute_time() + timeout_interval;
1009 }
1010
1011 /**
1012 * Goes to sleep waiting for an interrupt from a specificed serial device.
1013 *
1014 * @param fns serial functions representing the device to wait for
1015 */
1016 static void
serial_wait_for_interrupt(struct pe_serial_functions * fns)1017 serial_wait_for_interrupt(struct pe_serial_functions *fns)
1018 {
1019 assert_wait_deadline(fns, THREAD_UNINT, serial_interrupt_deadline(fns));
1020 if (!fns->transmit_ready()) {
1021 fns->enable_irq();
1022 thread_block(THREAD_CONTINUE_NULL);
1023 } else {
1024 clear_wait(current_thread(), THREAD_AWAKENED);
1025 }
1026 }
1027
1028 /**
1029 * Output a character onto every registered serial interface.
1030 *
1031 * @param c The character to output.
1032 * @param poll Whether the driver should poll to send the character or if it can
1033 * wait for an interrupt
1034 */
1035 MARK_AS_HIBERNATE_TEXT void
uart_putc_options(char c,bool poll)1036 uart_putc_options(char c, bool poll)
1037 {
1038 struct pe_serial_functions *fns = gPESF;
1039
1040 while (fns != NULL) {
1041 while (!fns->transmit_ready()) {
1042 if (!uart_hibernation) {
1043 if (!poll && irq_available_and_ready(fns)) {
1044 serial_wait_for_interrupt(fns);
1045 } else {
1046 serial_poll();
1047 }
1048 }
1049 }
1050 fns->transmit_data((uint8_t)c);
1051 fns = fns->next;
1052 }
1053 }
1054
1055 /**
1056 * Output a character onto every registered serial interface by polling.
1057 *
1058 * @param c The character to output.
1059 */
1060 void
uart_putc(char c)1061 uart_putc(char c)
1062 {
1063 uart_putc_options(c, true);
1064 }
1065
1066 /**
1067 * Read a character from the first registered serial interface that has data
1068 * available.
1069 *
1070 * @return The character if any interfaces have data available, otherwise -1.
1071 */
1072 int
uart_getc(void)1073 uart_getc(void)
1074 {
1075 struct pe_serial_functions *fns = gPESF;
1076 while (fns != NULL) {
1077 if (fns->receive_ready()) {
1078 return (int)fns->receive_data();
1079 }
1080 fns = fns->next;
1081 }
1082 return -1;
1083 }
1084
1085 /**
1086 * Enables IRQs for a specific serial device and returns whether or not IRQs for
1087 * that device where enabled successfully. For a serial driver to have irqs
1088 * enabled, it must have the enable_irq, disable_irq, and acknowledge_irq
1089 * functions defined and the has_irq flag set.
1090 *
1091 * @param device Serial device to enable irqs on
1092 * @note This function should only be called from the AppleSerialShim kext
1093 */
1094 kern_return_t
serial_enable_irq(serial_device_t device)1095 serial_enable_irq(serial_device_t device)
1096 {
1097 struct pe_serial_functions *fns = get_serial_functions(device);
1098
1099 if (!fns || !fns->has_irq || disable_uart_irq) {
1100 return KERN_FAILURE;
1101 }
1102
1103 serial_irq_status &= ~device;
1104
1105 return KERN_SUCCESS;
1106 }
1107
1108 /**
1109 * Acknowledges an irq for a specific serial device and wakes up the thread
1110 * waiting on the interrupt if one exists.
1111 *
1112 * @param device Serial device to enable irqs for
1113 * @note This function should only be called from the AppleSerialShim kext
1114 */
1115 kern_return_t
serial_ack_irq(serial_device_t device)1116 serial_ack_irq(serial_device_t device)
1117 {
1118 struct pe_serial_functions *fns = get_serial_functions(device);
1119
1120 if (!fns || !fns->has_irq) {
1121 return KERN_FAILURE;
1122 }
1123
1124 /* Disable IRQs until next time a thread waits for an interrupt */
1125 fns->disable_irq();
1126
1127 /**
1128 * Because IRQs are enabled only when we know a thread is about to sleep, we
1129 * can call wake up and reasonably expect there to be a thread waiting.
1130 */
1131 thread_wakeup(fns);
1132
1133 return KERN_SUCCESS;
1134 }
1135
1136 /**
1137 * Returns true if the pending IRQ for device is one that can be handled by the
1138 * platform serial driver.
1139 *
1140 * @param device Serial device to enable irqs for
1141 * @note This function is called from a primary interrupt context and should be
1142 * kept lightweight.
1143 * @note This function should only be called from the AppleSerialShim kext
1144 */
1145 bool
serial_filter_irq(serial_device_t device)1146 serial_filter_irq(serial_device_t device)
1147 {
1148 struct pe_serial_functions *fns = get_serial_functions(device);
1149
1150 if (!fns || !fns->has_irq) {
1151 return false;
1152 }
1153
1154 return fns->acknowledge_irq();
1155 }
1156
1157 /**
1158 * Prepares all serial devices to go to sleep by draining the hardware FIFOs
1159 * and disabling interrupts.
1160 */
1161 void
serial_go_to_sleep(void)1162 serial_go_to_sleep(void)
1163 {
1164 struct pe_serial_functions *fns = gPESF;
1165 while (fns != NULL) {
1166 if (irq_available_and_ready(fns)) {
1167 fns->disable_irq();
1168 }
1169 fns = fns->next;
1170 }
1171
1172 #ifdef APPLE_UART
1173 /* APPLE_UART needs to drain FIFO before sleeping */
1174 if (get_serial_functions(SERIAL_APPLE_UART)) {
1175 apple_uart_drain_fifo();
1176 }
1177 #endif /* APPLE_UART */
1178 }
1179