1 /*
2 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach_assert.h>
30 #include <mach/vm_types.h>
31 #include <mach/mach_time.h>
32 #include <kern/timer.h>
33 #include <kern/clock.h>
34 #include <kern/machine.h>
35 #include <kern/iotrace.h>
36 #include <kern/timeout.h>
37 #include <mach/machine.h>
38 #include <mach/machine/vm_param.h>
39 #include <mach_kdp.h>
40 #include <machine/machine_routines.h>
41 #include <kdp/kdp_udp.h>
42 #if !MACH_KDP
43 #include <kdp/kdp_callout.h>
44 #endif /* !MACH_KDP */
45 #include <arm/cpu_data.h>
46 #include <arm/cpu_data_internal.h>
47 #include <arm/caches_internal.h>
48
49 #include <vm/vm_kern.h>
50 #include <vm/vm_map.h>
51 #include <vm/pmap.h>
52
53 #include <arm/misc_protos.h>
54
55 #include <sys/errno.h>
56
57 #include <libkern/section_keywords.h>
58 #include <libkern/OSDebug.h>
59
60
61 #define INT_SIZE (BYTE_SIZE * sizeof (int))
62
63 #define BCOPY_PHYS_SRC_IS_PHYS(flags) (((flags) & cppvPsrc) != 0)
64 #define BCOPY_PHYS_DST_IS_PHYS(flags) (((flags) & cppvPsnk) != 0)
65 #define BCOPY_PHYS_SRC_IS_USER(flags) (((flags) & (cppvPsrc | cppvKmap)) == 0)
66 #define BCOPY_PHYS_DST_IS_USER(flags) (((flags) & (cppvPsnk | cppvKmap)) == 0)
67
68 static kern_return_t
bcopy_phys_internal(addr64_t src,addr64_t dst,vm_size_t bytes,int flags)69 bcopy_phys_internal(addr64_t src, addr64_t dst, vm_size_t bytes, int flags)
70 {
71 unsigned int src_index;
72 unsigned int dst_index;
73 vm_offset_t src_offset;
74 vm_offset_t dst_offset;
75 unsigned int wimg_bits_src, wimg_bits_dst;
76 unsigned int cpu_num = 0;
77 ppnum_t pn_src;
78 ppnum_t pn_dst;
79 addr64_t end __assert_only;
80 kern_return_t res = KERN_SUCCESS;
81
82 if (!BCOPY_PHYS_SRC_IS_USER(flags)) {
83 assert(!__improbable(os_add_overflow(src, bytes, &end)));
84 }
85 if (!BCOPY_PHYS_DST_IS_USER(flags)) {
86 assert(!__improbable(os_add_overflow(dst, bytes, &end)));
87 }
88
89 while ((bytes > 0) && (res == KERN_SUCCESS)) {
90 src_offset = src & PAGE_MASK;
91 dst_offset = dst & PAGE_MASK;
92 boolean_t use_copy_window_src = FALSE;
93 boolean_t use_copy_window_dst = FALSE;
94 vm_size_t count = bytes;
95 vm_size_t count2 = bytes;
96 if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
97 use_copy_window_src = !pmap_valid_address(src);
98 pn_src = (ppnum_t)(src >> PAGE_SHIFT);
99 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
100 count = PAGE_SIZE - src_offset;
101 wimg_bits_src = pmap_cache_attributes(pn_src);
102 if ((wimg_bits_src & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
103 use_copy_window_src = TRUE;
104 }
105 #else
106 if (use_copy_window_src) {
107 wimg_bits_src = pmap_cache_attributes(pn_src);
108 count = PAGE_SIZE - src_offset;
109 }
110 #endif
111 }
112 if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
113 // write preflighting needed for things like dtrace which may write static read-only mappings
114 use_copy_window_dst = (!pmap_valid_address(dst) || !mmu_kvtop_wpreflight(phystokv((pmap_paddr_t)dst)));
115 pn_dst = (ppnum_t)(dst >> PAGE_SHIFT);
116 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
117 count2 = PAGE_SIZE - dst_offset;
118 wimg_bits_dst = pmap_cache_attributes(pn_dst);
119 if ((wimg_bits_dst & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
120 use_copy_window_dst = TRUE;
121 }
122 #else
123 if (use_copy_window_dst) {
124 wimg_bits_dst = pmap_cache_attributes(pn_dst);
125 count2 = PAGE_SIZE - dst_offset;
126 }
127 #endif
128 }
129
130 char *tmp_src;
131 char *tmp_dst;
132
133 if (use_copy_window_src || use_copy_window_dst) {
134 mp_disable_preemption();
135 cpu_num = cpu_number();
136 }
137
138 if (use_copy_window_src) {
139 src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
140 tmp_src = (char*)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset);
141 } else if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
142 tmp_src = (char*)phystokv_range((pmap_paddr_t)src, &count);
143 } else {
144 tmp_src = (char*)src;
145 }
146 if (use_copy_window_dst) {
147 dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst);
148 tmp_dst = (char*)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset);
149 } else if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
150 tmp_dst = (char*)phystokv_range((pmap_paddr_t)dst, &count2);
151 } else {
152 tmp_dst = (char*)dst;
153 }
154
155 if (count > count2) {
156 count = count2;
157 }
158 if (count > bytes) {
159 count = bytes;
160 }
161
162 if (BCOPY_PHYS_SRC_IS_USER(flags)) {
163 res = copyin((user_addr_t)src, tmp_dst, count);
164 } else if (BCOPY_PHYS_DST_IS_USER(flags)) {
165 res = copyout(tmp_src, (user_addr_t)dst, count);
166 } else {
167 bcopy(tmp_src, tmp_dst, count);
168
169 }
170
171 if (use_copy_window_src) {
172 pmap_unmap_cpu_windows_copy(src_index);
173 }
174 if (use_copy_window_dst) {
175 pmap_unmap_cpu_windows_copy(dst_index);
176 }
177 if (use_copy_window_src || use_copy_window_dst) {
178 mp_enable_preemption();
179 }
180
181 src += count;
182 dst += count;
183 bytes -= count;
184 }
185 return res;
186 }
187
188 void
bcopy_phys(addr64_t src,addr64_t dst,vm_size_t bytes)189 bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
190 {
191 bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk);
192 }
193
194 void
bcopy_phys_with_options(addr64_t src,addr64_t dst,vm_size_t bytes,int options)195 bcopy_phys_with_options(addr64_t src, addr64_t dst, vm_size_t bytes, int options)
196 {
197 bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk | options);
198 }
199
200 extern void *secure_memset(void *, int, size_t);
201
202 static void
bzero_phys_page(vm_offset_t buf)203 bzero_phys_page(vm_offset_t buf)
204 {
205 assert((buf & PAGE_MASK) == 0);
206
207 #pragma clang diagnostic push
208 #pragma clang diagnostic ignored "-Wpass-failed"
209 /*
210 * The unrolling is chosen so that the `add` operands in the codegen
211 * are all immediates and avoid a `mov`
212 */
213 #pragma unroll (4096 / (4 << MMU_CLINE))
214 for (vm_offset_t offset = 0; offset < PAGE_SIZE; offset += (4 << MMU_CLINE)) {
215 asm volatile (
216 "dc zva, %0\n\t"
217 "dc zva, %1\n\t"
218 "dc zva, %2\n\t"
219 "dc zva, %3"
220 :
221 : "r"(buf + offset + (0 << MMU_CLINE))
222 , "r"(buf + offset + (1 << MMU_CLINE))
223 , "r"(buf + offset + (2 << MMU_CLINE))
224 , "r"(buf + offset + (3 << MMU_CLINE))
225 : "memory");
226 }
227 #pragma clang diagnostic pop
228 }
229
230
231 /* Zero bytes starting at a physical address */
232 static void
bzero_phys_internal(addr64_t src,vm_size_t bytes,__unused int options)233 bzero_phys_internal(addr64_t src, vm_size_t bytes, __unused int options)
234 {
235 unsigned int wimg_bits;
236 unsigned int cpu_num = cpu_number();
237 ppnum_t pn;
238 addr64_t end __assert_only;
239
240 assert(!__improbable(os_add_overflow(src, bytes, &end)));
241
242 vm_offset_t offset = src & PAGE_MASK;
243 while (bytes > 0) {
244 vm_size_t count = bytes;
245
246 boolean_t use_copy_window = !pmap_valid_address(src);
247 pn = (ppnum_t)(src >> PAGE_SHIFT);
248 wimg_bits = pmap_cache_attributes(pn);
249 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
250 count = PAGE_SIZE - offset;
251 if ((wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
252 use_copy_window = TRUE;
253 }
254 #else
255 if (use_copy_window) {
256 count = PAGE_SIZE - offset;
257 }
258 #endif
259 char *buf;
260 unsigned int index;
261 if (use_copy_window) {
262 mp_disable_preemption();
263 cpu_num = cpu_number();
264 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
265 buf = (char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset);
266 } else {
267 buf = (char *)phystokv_range((pmap_paddr_t)src, &count);
268 }
269
270 if (count > bytes) {
271 count = bytes;
272 }
273
274 switch (wimg_bits & VM_WIMG_MASK) {
275 case VM_WIMG_DEFAULT:
276 case VM_WIMG_WCOMB:
277 case VM_WIMG_INNERWBACK:
278 case VM_WIMG_WTHRU:
279 #if HAS_UCNORMAL_MEM || APPLEVIRTUALPLATFORM
280 case VM_WIMG_RT:
281 #endif
282 /**
283 * When we are zerofilling a normal page, there are a couple of assumptions that can
284 * be made.
285 *
286 * 1. The destination to be zeroed is page-sized and page-aligned, making the unconditional
287 * 4 stp instructions in bzero redundant.
288 * 2. The dczva loop for zerofilling can be fully unrolled at compile-time thanks to
289 * known size of the destination, reducing instruction fetch overhead caused by
290 * the branch backward in a tight loop.
291 */
292 if (count == PAGE_SIZE) {
293 /**
294 * Thanks to how count is computed above, buf should always be page-size aligned
295 * when count == PAGE_SIZE.
296 */
297 bzero_phys_page((vm_offset_t)buf);
298 } else {
299 bzero(buf, count);
300 }
301 break;
302
303 default:
304 /* 'dc zva' performed by bzero is not safe for device memory */
305 secure_memset((void*)buf, 0, count);
306 }
307
308 if (use_copy_window) {
309 pmap_unmap_cpu_windows_copy(index);
310 mp_enable_preemption();
311 }
312
313 src += count;
314 bytes -= count;
315 offset = 0;
316 }
317 }
318
319 void
bzero_phys_nc(addr64_t src64,vm_size_t bytes)320 bzero_phys_nc(addr64_t src64, vm_size_t bytes)
321 {
322 bzero_phys_internal(src64, bytes, 0);
323 }
324
325 void
bzero_phys(addr64_t src,vm_size_t bytes)326 bzero_phys(addr64_t src, vm_size_t bytes)
327 {
328 bzero_phys_internal(src, bytes, 0);
329 }
330
331 void
bzero_phys_with_options(addr64_t src,vm_size_t bytes,int options)332 bzero_phys_with_options(addr64_t src, vm_size_t bytes, int options)
333 {
334 bzero_phys_internal(src, bytes, options);
335 }
336
337 /*
338 * Read data from a physical address.
339 */
340
341 #if BUILD_QUAD_WORD_FUNCS
342 static inline uint128_t
__read128(vm_address_t addr)343 __read128(vm_address_t addr)
344 {
345 uint64_t hi, lo;
346
347 asm volatile (
348 "ldp %[lo], %[hi], [%[addr]]" "\n"
349 : [lo] "=r"(lo), [hi] "=r"(hi)
350 : [addr] "r"(addr)
351 : "memory"
352 );
353
354 return (((uint128_t)hi) << 64) + lo;
355 }
356 #endif /* BUILD_QUAD_WORD_FUNCS */
357
358 static uint128_t
ml_phys_read_data(pmap_paddr_t paddr,int size)359 ml_phys_read_data(pmap_paddr_t paddr, int size)
360 {
361 vm_address_t addr;
362 ppnum_t pn = atop_kernel(paddr);
363 ppnum_t pn_end = atop_kernel(paddr + size - 1);
364 uint128_t result = 0;
365 uint8_t s1;
366 uint16_t s2;
367 uint32_t s4;
368 uint64_t s8;
369 unsigned int index;
370 bool use_copy_window = true;
371
372 if (__improbable(pn_end != pn)) {
373 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
374 }
375
376 #ifdef ML_IO_TIMEOUTS_ENABLED
377 bool istate, use_timeout = false;
378 kern_timeout_t timeout;
379
380 uint32_t report_phy_read_delay = os_atomic_load(&report_phy_read_delay_to, relaxed);
381 uint32_t const trace_phy_read_delay = os_atomic_load(&trace_phy_read_delay_to, relaxed);
382
383 if (__improbable(report_phy_read_delay != 0)) {
384 istate = ml_set_interrupts_enabled_with_debug(false, false);
385
386 kern_timeout_start(&timeout, TF_NONSPEC_TIMEBASE | TF_SAMPLE_PMC);
387 use_timeout = true;
388
389 mmio_track_t *mmiot = PERCPU_GET(mmio_tracker);
390 mmiot->mmio_start_mt = kern_timeout_start_time(&timeout);
391 mmiot->mmio_paddr = paddr;
392 mmiot->mmio_vaddr = 0;
393 }
394 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
395 if (__improbable(use_timeout && simulate_stretched_io)) {
396 kern_timeout_stretch(&timeout, simulate_stretched_io);
397 }
398 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
399 #endif /* ML_IO_TIMEOUTS_ENABLED */
400
401 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
402 if (pmap_valid_address(paddr)) {
403 addr = phystokv(paddr);
404 use_copy_window = false;
405 }
406 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
407
408 if (use_copy_window) {
409 mp_disable_preemption();
410 unsigned int wimg_bits = pmap_cache_attributes(pn);
411 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
412 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
413 }
414
415 switch (size) {
416 case 1:
417 s1 = *(volatile uint8_t *)addr;
418 result = s1;
419 break;
420 case 2:
421 s2 = *(volatile uint16_t *)addr;
422 result = s2;
423 break;
424 case 4:
425 s4 = *(volatile uint32_t *)addr;
426 result = s4;
427 break;
428 case 8:
429 s8 = *(volatile uint64_t *)addr;
430 result = s8;
431 break;
432 #if BUILD_QUAD_WORD_FUNCS
433 case 16:
434 result = __read128(addr);
435 break;
436 #endif /* BUILD_QUAD_WORD_FUNCS */
437 default:
438 panic("Invalid size %d for ml_phys_read_data", size);
439 break;
440 }
441
442 if (use_copy_window) {
443 pmap_unmap_cpu_windows_copy(index);
444 mp_enable_preemption();
445 }
446
447 #ifdef ML_IO_TIMEOUTS_ENABLED
448 if (__improbable(use_timeout)) {
449 kern_timeout_end(&timeout, TF_NONSPEC_TIMEBASE);
450 uint64_t duration = kern_timeout_gross_duration(&timeout);
451
452 iotrace(IOTRACE_PHYS_READ, 0, addr, size, result, kern_timeout_start_time(&timeout), duration);
453
454 if (__improbable(duration > report_phy_read_delay)) {
455 DTRACE_PHYSLAT4(physread, uint64_t, duration,
456 uint64_t, addr, uint32_t, size, uint64_t, result);
457
458 uint64_t override = 0;
459 override_io_timeouts(0, paddr, &override, NULL);
460
461 if (override != 0) {
462 #if SCHED_HYGIENE_DEBUG
463 /*
464 * The IO timeout was overridden. If we were called in an
465 * interrupt handler context, that can lead to a timeout
466 * panic, so we need to abandon the measurement.
467 */
468 if (interrupt_masked_debug_mode == SCHED_HYGIENE_MODE_PANIC) {
469 ml_irq_debug_abandon();
470 }
471 #endif
472 report_phy_read_delay = override;
473 }
474 }
475
476 if (__improbable(duration > report_phy_read_delay)) {
477 if (phy_read_panic && (machine_timeout_suspended() == FALSE)) {
478 char str[128];
479 const uint64_t hi = (uint64_t)(result >> 64);
480 const uint64_t lo = (uint64_t)(result);
481
482 snprintf(str, sizeof(str),
483 "Read from physical addr 0x%llx (result: 0x%016llx%016llx) timed out:",
484 (unsigned long long)addr, hi, lo);
485 kern_timeout_try_panic(KERN_TIMEOUT_MMIO, paddr, &timeout, str,
486 report_phy_read_delay);
487 }
488 }
489
490 if (__improbable(trace_phy_read_delay > 0 && duration > trace_phy_read_delay)) {
491 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_READ),
492 duration, kern_timeout_start_time(&timeout), addr, result);
493 }
494
495 ml_set_interrupts_enabled_with_debug(istate, false);
496 }
497 #endif /* ML_IO_TIMEOUTS_ENABLED */
498
499 return result;
500 }
501
502 unsigned int
ml_phys_read(vm_offset_t paddr)503 ml_phys_read(vm_offset_t paddr)
504 {
505 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
506 }
507
508 unsigned int
ml_phys_read_word(vm_offset_t paddr)509 ml_phys_read_word(vm_offset_t paddr)
510 {
511 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
512 }
513
514 unsigned int
ml_phys_read_64(addr64_t paddr64)515 ml_phys_read_64(addr64_t paddr64)
516 {
517 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
518 }
519
520 unsigned int
ml_phys_read_word_64(addr64_t paddr64)521 ml_phys_read_word_64(addr64_t paddr64)
522 {
523 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
524 }
525
526 unsigned int
ml_phys_read_half(vm_offset_t paddr)527 ml_phys_read_half(vm_offset_t paddr)
528 {
529 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2);
530 }
531
532 unsigned int
ml_phys_read_half_64(addr64_t paddr64)533 ml_phys_read_half_64(addr64_t paddr64)
534 {
535 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2);
536 }
537
538 unsigned int
ml_phys_read_byte(vm_offset_t paddr)539 ml_phys_read_byte(vm_offset_t paddr)
540 {
541 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1);
542 }
543
544 unsigned int
ml_phys_read_byte_64(addr64_t paddr64)545 ml_phys_read_byte_64(addr64_t paddr64)
546 {
547 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1);
548 }
549
550 unsigned long long
ml_phys_read_double(vm_offset_t paddr)551 ml_phys_read_double(vm_offset_t paddr)
552 {
553 return ml_phys_read_data((pmap_paddr_t)paddr, 8);
554 }
555
556 unsigned long long
ml_phys_read_double_64(addr64_t paddr64)557 ml_phys_read_double_64(addr64_t paddr64)
558 {
559 return ml_phys_read_data((pmap_paddr_t)paddr64, 8);
560 }
561
562 #if BUILD_QUAD_WORD_FUNCS
563 uint128_t
ml_phys_read_quad(vm_offset_t paddr)564 ml_phys_read_quad(vm_offset_t paddr)
565 {
566 return ml_phys_read_data((pmap_paddr_t)paddr, 16);
567 }
568
569 uint128_t
ml_phys_read_quad_64(addr64_t paddr64)570 ml_phys_read_quad_64(addr64_t paddr64)
571 {
572 return ml_phys_read_data((pmap_paddr_t)paddr64, 16);
573 }
574 #endif /* BUILD_QUAD_WORD_FUNCS */
575
576 /*
577 * Write data to a physical address.
578 */
579
580 #if BUILD_QUAD_WORD_FUNCS
581 static inline void
__write128(vm_address_t addr,uint128_t data)582 __write128(vm_address_t addr, uint128_t data)
583 {
584 const uint64_t hi = (uint64_t)(data >> 64);
585 const uint64_t lo = (uint64_t)(data);
586
587 asm volatile (
588 "stp %[lo], %[hi], [%[addr]]" "\n"
589 : /**/
590 : [lo] "r"(lo), [hi] "r"(hi), [addr] "r"(addr)
591 : "memory"
592 );
593 }
594 #endif /* BUILD_QUAD_WORD_FUNCS */
595
596 static void
ml_phys_write_data(pmap_paddr_t paddr,uint128_t data,int size)597 ml_phys_write_data(pmap_paddr_t paddr, uint128_t data, int size)
598 {
599 vm_address_t addr;
600 ppnum_t pn = atop_kernel(paddr);
601 ppnum_t pn_end = atop_kernel(paddr + size - 1);
602 unsigned int index;
603 bool use_copy_window = true;
604
605 if (__improbable(pn_end != pn)) {
606 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
607 }
608
609 #ifdef ML_IO_TIMEOUTS_ENABLED
610 bool istate, use_timeout = false;
611 kern_timeout_t timeout;
612
613 uint32_t report_phy_write_delay = os_atomic_load(&report_phy_write_delay_to, relaxed);
614 uint32_t const trace_phy_write_delay = os_atomic_load(&trace_phy_write_delay_to, relaxed);
615
616 if (__improbable(report_phy_write_delay != 0)) {
617 istate = ml_set_interrupts_enabled_with_debug(false, false);
618
619 kern_timeout_start(&timeout, TF_NONSPEC_TIMEBASE | TF_SAMPLE_PMC);
620 use_timeout = true;
621
622 mmio_track_t *mmiot = PERCPU_GET(mmio_tracker);
623 mmiot->mmio_start_mt = kern_timeout_start_time(&timeout);
624 mmiot->mmio_paddr = paddr;
625 mmiot->mmio_vaddr = 0;
626 }
627 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
628 if (__improbable(use_timeout && simulate_stretched_io)) {
629 kern_timeout_stretch(&timeout, simulate_stretched_io);
630 }
631 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
632 #endif /* ML_IO_TIMEOUTS_ENABLED */
633
634 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
635 if (pmap_valid_address(paddr)) {
636 addr = phystokv(paddr);
637 use_copy_window = false;
638 }
639 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
640
641 if (use_copy_window) {
642 mp_disable_preemption();
643 unsigned int wimg_bits = pmap_cache_attributes(pn);
644 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
645 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
646 }
647
648 switch (size) {
649 case 1:
650 *(volatile uint8_t *)addr = (uint8_t)data;
651 break;
652 case 2:
653 *(volatile uint16_t *)addr = (uint16_t)data;
654 break;
655 case 4:
656 *(volatile uint32_t *)addr = (uint32_t)data;
657 break;
658 case 8:
659 *(volatile uint64_t *)addr = (uint64_t)data;
660 break;
661 #if BUILD_QUAD_WORD_FUNCS
662 case 16:
663 __write128(addr, data);
664 break;
665 #endif /* BUILD_QUAD_WORD_FUNCS */
666 default:
667 panic("Invalid size %d for ml_phys_write_data", size);
668 }
669
670 if (use_copy_window) {
671 pmap_unmap_cpu_windows_copy(index);
672 mp_enable_preemption();
673 }
674
675 #ifdef ML_IO_TIMEOUTS_ENABLED
676 if (__improbable(use_timeout)) {
677 kern_timeout_end(&timeout, TF_NONSPEC_TIMEBASE);
678 uint64_t duration = kern_timeout_gross_duration(&timeout);
679
680 iotrace(IOTRACE_PHYS_WRITE, 0, paddr, size, data, kern_timeout_start_time(&timeout), duration);
681
682 if (__improbable(duration > report_phy_write_delay)) {
683 DTRACE_PHYSLAT4(physwrite, uint64_t, duration,
684 uint64_t, paddr, uint32_t, size, uint64_t, data);
685
686 uint64_t override = 0;
687 override_io_timeouts(0, paddr, NULL, &override);
688
689 if (override != 0) {
690 #if SCHED_HYGIENE_DEBUG
691 /*
692 * The IO timeout was overridden. If we were called in an
693 * interrupt handler context, that can lead to a timeout
694 * panic, so we need to abandon the measurement.
695 */
696 if (interrupt_masked_debug_mode == SCHED_HYGIENE_MODE_PANIC) {
697 ml_irq_debug_abandon();
698 }
699 #endif
700 report_phy_write_delay = override;
701 }
702 }
703
704 if (__improbable(duration > report_phy_write_delay)) {
705 if (phy_write_panic && (machine_timeout_suspended() == FALSE)) {
706 char str[128];
707 const uint64_t hi = (uint64_t)(data >> 64);
708 const uint64_t lo = (uint64_t)(data);
709
710 snprintf(str, sizeof(str),
711 "Write to physical addr 0x%llx (data: 0x%016llx%016llx) timed out:",
712 (unsigned long long)paddr, hi, lo);
713 kern_timeout_try_panic(KERN_TIMEOUT_MMIO, paddr, &timeout, str,
714 report_phy_write_delay);
715 }
716 }
717
718 if (__improbable(trace_phy_write_delay > 0 && duration > trace_phy_write_delay)) {
719 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_WRITE),
720 duration, kern_timeout_start_time(&timeout), paddr, data);
721 }
722
723 ml_set_interrupts_enabled_with_debug(istate, false);
724 }
725 #endif /* ML_IO_TIMEOUTS_ENABLED */
726 }
727
728 void
ml_phys_write_byte(vm_offset_t paddr,unsigned int data)729 ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
730 {
731 ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
732 }
733
734 void
ml_phys_write_byte_64(addr64_t paddr64,unsigned int data)735 ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
736 {
737 ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
738 }
739
740 void
ml_phys_write_half(vm_offset_t paddr,unsigned int data)741 ml_phys_write_half(vm_offset_t paddr, unsigned int data)
742 {
743 ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
744 }
745
746 void
ml_phys_write_half_64(addr64_t paddr64,unsigned int data)747 ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
748 {
749 ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
750 }
751
752 void
ml_phys_write(vm_offset_t paddr,unsigned int data)753 ml_phys_write(vm_offset_t paddr, unsigned int data)
754 {
755 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
756 }
757
758 void
ml_phys_write_64(addr64_t paddr64,unsigned int data)759 ml_phys_write_64(addr64_t paddr64, unsigned int data)
760 {
761 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
762 }
763
764 void
ml_phys_write_word(vm_offset_t paddr,unsigned int data)765 ml_phys_write_word(vm_offset_t paddr, unsigned int data)
766 {
767 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
768 }
769
770 void
ml_phys_write_word_64(addr64_t paddr64,unsigned int data)771 ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
772 {
773 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
774 }
775
776 void
ml_phys_write_double(vm_offset_t paddr,unsigned long long data)777 ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
778 {
779 ml_phys_write_data((pmap_paddr_t)paddr, data, 8);
780 }
781
782 void
ml_phys_write_double_64(addr64_t paddr64,unsigned long long data)783 ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
784 {
785 ml_phys_write_data((pmap_paddr_t)paddr64, data, 8);
786 }
787
788 #if BUILD_QUAD_WORD_FUNCS
789 void
ml_phys_write_quad(vm_offset_t paddr,uint128_t data)790 ml_phys_write_quad(vm_offset_t paddr, uint128_t data)
791 {
792 ml_phys_write_data((pmap_paddr_t)paddr, data, 16);
793 }
794
795 void
ml_phys_write_quad_64(addr64_t paddr64,uint128_t data)796 ml_phys_write_quad_64(addr64_t paddr64, uint128_t data)
797 {
798 ml_phys_write_data((pmap_paddr_t)paddr64, data, 16);
799 }
800 #endif /* BUILD_QUAD_WORD_FUNCS */
801
802 /*
803 * Set indicated bit in bit string.
804 */
805 void
setbit(int bitno,int * s)806 setbit(int bitno, int *s)
807 {
808 s[bitno / INT_SIZE] |= 1U << (bitno % INT_SIZE);
809 }
810
811 /*
812 * Clear indicated bit in bit string.
813 */
814 void
clrbit(int bitno,int * s)815 clrbit(int bitno, int *s)
816 {
817 s[bitno / INT_SIZE] &= ~(1U << (bitno % INT_SIZE));
818 }
819
820 /*
821 * Test if indicated bit is set in bit string.
822 */
823 int
testbit(int bitno,int * s)824 testbit(int bitno, int *s)
825 {
826 return s[bitno / INT_SIZE] & (1U << (bitno % INT_SIZE));
827 }
828
829 /*
830 * Find first bit set in bit string.
831 */
832 int
ffsbit(int * s)833 ffsbit(int *s)
834 {
835 int offset;
836
837 for (offset = 0; !*s; offset += INT_SIZE, ++s) {
838 ;
839 }
840 return offset + __builtin_ctz(*s);
841 }
842
843 int
ffs(unsigned int mask)844 ffs(unsigned int mask)
845 {
846 if (mask == 0) {
847 return 0;
848 }
849
850 /*
851 * NOTE: cannot use __builtin_ffs because it generates a call to
852 * 'ffs'
853 */
854 return 1 + __builtin_ctz(mask);
855 }
856
857 int
ffsll(unsigned long long mask)858 ffsll(unsigned long long mask)
859 {
860 if (mask == 0) {
861 return 0;
862 }
863
864 /*
865 * NOTE: cannot use __builtin_ffsll because it generates a call to
866 * 'ffsll'
867 */
868 return 1 + __builtin_ctzll(mask);
869 }
870
871 /*
872 * Find last bit set in bit string.
873 */
874 int
fls(unsigned int mask)875 fls(unsigned int mask)
876 {
877 if (mask == 0) {
878 return 0;
879 }
880
881 return (sizeof(mask) << 3) - __builtin_clz(mask);
882 }
883
884 int
flsll(unsigned long long mask)885 flsll(unsigned long long mask)
886 {
887 if (mask == 0) {
888 return 0;
889 }
890
891 return (sizeof(mask) << 3) - __builtin_clzll(mask);
892 }
893
894 kern_return_t
copypv(addr64_t source,addr64_t sink,unsigned int size,int which)895 copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
896 {
897 if ((which & (cppvPsrc | cppvPsnk)) == 0) { /* Make sure that only one is virtual */
898 panic("%s: no more than 1 parameter may be virtual", __func__);
899 }
900
901 kern_return_t res = bcopy_phys_internal(source, sink, size, which);
902
903 #ifndef __ARM_COHERENT_IO__
904 if (which & cppvFsrc) {
905 flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
906 }
907
908 if (which & cppvFsnk) {
909 flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
910 }
911 #endif
912
913 return res;
914 }
915
916 int
clr_be_bit(void)917 clr_be_bit(void)
918 {
919 panic("clr_be_bit");
920 return 0;
921 }
922
923 boolean_t
ml_probe_read(__unused vm_offset_t paddr,__unused unsigned int * val)924 ml_probe_read(
925 __unused vm_offset_t paddr,
926 __unused unsigned int *val)
927 {
928 panic("ml_probe_read() unimplemented");
929 return 1;
930 }
931
932 boolean_t
ml_probe_read_64(__unused addr64_t paddr,__unused unsigned int * val)933 ml_probe_read_64(
934 __unused addr64_t paddr,
935 __unused unsigned int *val)
936 {
937 panic("ml_probe_read_64() unimplemented");
938 return 1;
939 }
940
941
942 void
ml_thread_policy(__unused thread_t thread,__unused unsigned policy_id,__unused unsigned policy_info)943 ml_thread_policy(
944 __unused thread_t thread,
945 __unused unsigned policy_id,
946 __unused unsigned policy_info)
947 {
948 // <rdar://problem/7141284>: Reduce print noise
949 // kprintf("ml_thread_policy() unimplemented\n");
950 }
951
952 __dead2
953 void
panic_unimplemented(void)954 panic_unimplemented(void)
955 {
956 panic("Not yet implemented.");
957 }
958
959 /* ARM64_TODO <rdar://problem/9198953> */
960 void abort(void) __dead2;
961
962 void
abort(void)963 abort(void)
964 {
965 panic("Abort.");
966 }
967
968
969 #if !MACH_KDP
970 void
kdp_register_callout(kdp_callout_fn_t fn,void * arg)971 kdp_register_callout(kdp_callout_fn_t fn, void *arg)
972 {
973 #pragma unused(fn,arg)
974 }
975 #endif
976
977 /*
978 * Get a quick virtual mapping of a physical page and run a callback on that
979 * page's virtual address.
980 *
981 * @param dst64 Physical address to access (doesn't need to be page-aligned).
982 * @param bytes Number of bytes to be accessed. This cannot cross page boundaries.
983 * @param func Callback function to call with the page's virtual address.
984 * @param arg Argument passed directly to `func`.
985 *
986 * @return The return value from `func`.
987 */
988 int
apply_func_phys(addr64_t dst64,vm_size_t bytes,int (* func)(void * buffer,vm_size_t bytes,void * arg),void * arg)989 apply_func_phys(
990 addr64_t dst64,
991 vm_size_t bytes,
992 int (*func)(void * buffer, vm_size_t bytes, void * arg),
993 void * arg)
994 {
995 /* The physical aperture is only guaranteed to work with kernel-managed addresses. */
996 if (!pmap_valid_address(dst64)) {
997 panic("%s address error: passed in address (%#llx) not a kernel managed address",
998 __FUNCTION__, dst64);
999 }
1000
1001 /* Ensure we stay within a single page */
1002 if (((((uint32_t)dst64 & (ARM_PGBYTES - 1)) + bytes) > ARM_PGBYTES)) {
1003 panic("%s alignment error: tried accessing addresses spanning more than one page %#llx %#lx",
1004 __FUNCTION__, dst64, bytes);
1005 }
1006
1007 return func((void*)phystokv(dst64), bytes, arg);
1008 }
1009