1 /*
2 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach_assert.h>
30 #include <mach/vm_types.h>
31 #include <mach/mach_time.h>
32 #include <kern/timer.h>
33 #include <kern/clock.h>
34 #include <kern/machine.h>
35 #include <kern/iotrace.h>
36 #include <mach/machine.h>
37 #include <mach/machine/vm_param.h>
38 #include <mach_kdp.h>
39 #include <kdp/kdp_udp.h>
40 #if !MACH_KDP
41 #include <kdp/kdp_callout.h>
42 #endif /* !MACH_KDP */
43 #include <arm/cpu_data.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/caches_internal.h>
46
47 #include <vm/vm_kern.h>
48 #include <vm/vm_map.h>
49 #include <vm/pmap.h>
50
51 #include <arm/misc_protos.h>
52
53 #include <sys/errno.h>
54
55 #include <libkern/section_keywords.h>
56 #include <libkern/OSDebug.h>
57
58
59 #define INT_SIZE (BYTE_SIZE * sizeof (int))
60
61 #define BCOPY_PHYS_SRC_IS_PHYS(flags) (((flags) & cppvPsrc) != 0)
62 #define BCOPY_PHYS_DST_IS_PHYS(flags) (((flags) & cppvPsnk) != 0)
63 #define BCOPY_PHYS_SRC_IS_USER(flags) (((flags) & (cppvPsrc | cppvKmap)) == 0)
64 #define BCOPY_PHYS_DST_IS_USER(flags) (((flags) & (cppvPsnk | cppvKmap)) == 0)
65
66 static kern_return_t
bcopy_phys_internal(addr64_t src,addr64_t dst,vm_size_t bytes,int flags)67 bcopy_phys_internal(addr64_t src, addr64_t dst, vm_size_t bytes, int flags)
68 {
69 unsigned int src_index;
70 unsigned int dst_index;
71 vm_offset_t src_offset;
72 vm_offset_t dst_offset;
73 unsigned int wimg_bits_src, wimg_bits_dst;
74 unsigned int cpu_num = 0;
75 ppnum_t pn_src;
76 ppnum_t pn_dst;
77 addr64_t end __assert_only;
78 kern_return_t res = KERN_SUCCESS;
79
80 if (!BCOPY_PHYS_SRC_IS_USER(flags)) {
81 assert(!__improbable(os_add_overflow(src, bytes, &end)));
82 }
83 if (!BCOPY_PHYS_DST_IS_USER(flags)) {
84 assert(!__improbable(os_add_overflow(dst, bytes, &end)));
85 }
86
87 while ((bytes > 0) && (res == KERN_SUCCESS)) {
88 src_offset = src & PAGE_MASK;
89 dst_offset = dst & PAGE_MASK;
90 boolean_t use_copy_window_src = FALSE;
91 boolean_t use_copy_window_dst = FALSE;
92 vm_size_t count = bytes;
93 vm_size_t count2 = bytes;
94 if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
95 use_copy_window_src = !pmap_valid_address(src);
96 pn_src = (ppnum_t)(src >> PAGE_SHIFT);
97 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
98 count = PAGE_SIZE - src_offset;
99 wimg_bits_src = pmap_cache_attributes(pn_src);
100 if ((wimg_bits_src & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
101 use_copy_window_src = TRUE;
102 }
103 #else
104 if (use_copy_window_src) {
105 wimg_bits_src = pmap_cache_attributes(pn_src);
106 count = PAGE_SIZE - src_offset;
107 }
108 #endif
109 }
110 if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
111 // write preflighting needed for things like dtrace which may write static read-only mappings
112 use_copy_window_dst = (!pmap_valid_address(dst) || !mmu_kvtop_wpreflight(phystokv((pmap_paddr_t)dst)));
113 pn_dst = (ppnum_t)(dst >> PAGE_SHIFT);
114 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
115 count2 = PAGE_SIZE - dst_offset;
116 wimg_bits_dst = pmap_cache_attributes(pn_dst);
117 if ((wimg_bits_dst & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
118 use_copy_window_dst = TRUE;
119 }
120 #else
121 if (use_copy_window_dst) {
122 wimg_bits_dst = pmap_cache_attributes(pn_dst);
123 count2 = PAGE_SIZE - dst_offset;
124 }
125 #endif
126 }
127
128 char *tmp_src;
129 char *tmp_dst;
130
131 if (use_copy_window_src || use_copy_window_dst) {
132 mp_disable_preemption();
133 cpu_num = cpu_number();
134 }
135
136 if (use_copy_window_src) {
137 src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
138 tmp_src = (char*)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset);
139 } else if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
140 tmp_src = (char*)phystokv_range((pmap_paddr_t)src, &count);
141 } else {
142 tmp_src = (char*)src;
143 }
144 if (use_copy_window_dst) {
145 dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst);
146 tmp_dst = (char*)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset);
147 } else if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
148 tmp_dst = (char*)phystokv_range((pmap_paddr_t)dst, &count2);
149 } else {
150 tmp_dst = (char*)dst;
151 }
152
153 if (count > count2) {
154 count = count2;
155 }
156 if (count > bytes) {
157 count = bytes;
158 }
159
160 if (BCOPY_PHYS_SRC_IS_USER(flags)) {
161 res = copyin((user_addr_t)src, tmp_dst, count);
162 } else if (BCOPY_PHYS_DST_IS_USER(flags)) {
163 res = copyout(tmp_src, (user_addr_t)dst, count);
164 } else {
165 bcopy(tmp_src, tmp_dst, count);
166
167 }
168
169 if (use_copy_window_src) {
170 pmap_unmap_cpu_windows_copy(src_index);
171 }
172 if (use_copy_window_dst) {
173 pmap_unmap_cpu_windows_copy(dst_index);
174 }
175 if (use_copy_window_src || use_copy_window_dst) {
176 mp_enable_preemption();
177 }
178
179 src += count;
180 dst += count;
181 bytes -= count;
182 }
183 return res;
184 }
185
186 void
bcopy_phys(addr64_t src,addr64_t dst,vm_size_t bytes)187 bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
188 {
189 bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk);
190 }
191
192 void
bcopy_phys_with_options(addr64_t src,addr64_t dst,vm_size_t bytes,int options)193 bcopy_phys_with_options(addr64_t src, addr64_t dst, vm_size_t bytes, int options)
194 {
195 bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk | options);
196 }
197
198 extern void *secure_memset(void *, int, size_t);
199
200 static void
bzero_phys_page(vm_offset_t buf)201 bzero_phys_page(vm_offset_t buf)
202 {
203 assert((buf & PAGE_MASK) == 0);
204
205 #pragma clang diagnostic push
206 #pragma clang diagnostic ignored "-Wpass-failed"
207 /*
208 * The unrolling is chosen so that the `add` operands in the codegen
209 * are all immediates and avoid a `mov`
210 */
211 #pragma unroll (4096 / (4 << MMU_CLINE))
212 for (vm_offset_t offset = 0; offset < PAGE_SIZE; offset += (4 << MMU_CLINE)) {
213 asm volatile (
214 "dc zva, %0\n\t"
215 "dc zva, %1\n\t"
216 "dc zva, %2\n\t"
217 "dc zva, %3"
218 :
219 : "r"(buf + offset + (0 << MMU_CLINE))
220 , "r"(buf + offset + (1 << MMU_CLINE))
221 , "r"(buf + offset + (2 << MMU_CLINE))
222 , "r"(buf + offset + (3 << MMU_CLINE))
223 : "memory");
224 }
225 #pragma clang diagnostic pop
226 }
227
228
229 /* Zero bytes starting at a physical address */
230 static void
bzero_phys_internal(addr64_t src,vm_size_t bytes,__unused int options)231 bzero_phys_internal(addr64_t src, vm_size_t bytes, __unused int options)
232 {
233 unsigned int wimg_bits;
234 unsigned int cpu_num = cpu_number();
235 ppnum_t pn;
236 addr64_t end __assert_only;
237
238 assert(!__improbable(os_add_overflow(src, bytes, &end)));
239
240 vm_offset_t offset = src & PAGE_MASK;
241 while (bytes > 0) {
242 vm_size_t count = bytes;
243
244 boolean_t use_copy_window = !pmap_valid_address(src);
245 pn = (ppnum_t)(src >> PAGE_SHIFT);
246 wimg_bits = pmap_cache_attributes(pn);
247 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
248 count = PAGE_SIZE - offset;
249 if ((wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
250 use_copy_window = TRUE;
251 }
252 #else
253 if (use_copy_window) {
254 count = PAGE_SIZE - offset;
255 }
256 #endif
257 char *buf;
258 unsigned int index;
259 if (use_copy_window) {
260 mp_disable_preemption();
261 cpu_num = cpu_number();
262 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
263 buf = (char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset);
264 } else {
265 buf = (char *)phystokv_range((pmap_paddr_t)src, &count);
266 }
267
268 if (count > bytes) {
269 count = bytes;
270 }
271
272 switch (wimg_bits & VM_WIMG_MASK) {
273 case VM_WIMG_DEFAULT:
274 case VM_WIMG_WCOMB:
275 case VM_WIMG_INNERWBACK:
276 case VM_WIMG_WTHRU:
277 #if HAS_UCNORMAL_MEM || APPLEVIRTUALPLATFORM
278 case VM_WIMG_RT:
279 #endif
280 /**
281 * When we are zerofilling a normal page, there are a couple of assumptions that can
282 * be made.
283 *
284 * 1. The destination to be zeroed is page-sized and page-aligned, making the unconditional
285 * 4 stp instructions in bzero redundant.
286 * 2. The dczva loop for zerofilling can be fully unrolled at compile-time thanks to
287 * known size of the destination, reducing instruction fetch overhead caused by
288 * the branch backward in a tight loop.
289 */
290 if (count == PAGE_SIZE) {
291 /**
292 * Thanks to how count is computed above, buf should always be page-size aligned
293 * when count == PAGE_SIZE.
294 */
295 bzero_phys_page((vm_offset_t)buf);
296 } else {
297 bzero(buf, count);
298 }
299 break;
300
301 default:
302 /* 'dc zva' performed by bzero is not safe for device memory */
303 secure_memset((void*)buf, 0, count);
304 }
305
306 if (use_copy_window) {
307 pmap_unmap_cpu_windows_copy(index);
308 mp_enable_preemption();
309 }
310
311 src += count;
312 bytes -= count;
313 offset = 0;
314 }
315 }
316
317 void
bzero_phys_nc(addr64_t src64,vm_size_t bytes)318 bzero_phys_nc(addr64_t src64, vm_size_t bytes)
319 {
320 bzero_phys_internal(src64, bytes, 0);
321 }
322
323 void
bzero_phys(addr64_t src,vm_size_t bytes)324 bzero_phys(addr64_t src, vm_size_t bytes)
325 {
326 bzero_phys_internal(src, bytes, 0);
327 }
328
329 void
bzero_phys_with_options(addr64_t src,vm_size_t bytes,int options)330 bzero_phys_with_options(addr64_t src, vm_size_t bytes, int options)
331 {
332 bzero_phys_internal(src, bytes, options);
333 }
334
335 /*
336 * Read data from a physical address.
337 */
338
339 #if BUILD_QUAD_WORD_FUNCS
340 static inline uint128_t
__read128(vm_address_t addr)341 __read128(vm_address_t addr)
342 {
343 uint64_t hi, lo;
344
345 asm volatile (
346 "ldp %[lo], %[hi], [%[addr]]" "\n"
347 : [lo] "=r"(lo), [hi] "=r"(hi)
348 : [addr] "r"(addr)
349 : "memory"
350 );
351
352 return (((uint128_t)hi) << 64) + lo;
353 }
354 #endif /* BUILD_QUAD_WORD_FUNCS */
355
356 static uint128_t
ml_phys_read_data(pmap_paddr_t paddr,int size)357 ml_phys_read_data(pmap_paddr_t paddr, int size)
358 {
359 vm_address_t addr;
360 ppnum_t pn = atop_kernel(paddr);
361 ppnum_t pn_end = atop_kernel(paddr + size - 1);
362 uint128_t result = 0;
363 uint8_t s1;
364 uint16_t s2;
365 uint32_t s4;
366 uint64_t s8;
367 unsigned int index;
368 bool use_copy_window = true;
369
370 if (__improbable(pn_end != pn)) {
371 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
372 }
373
374 #ifdef ML_IO_TIMEOUTS_ENABLED
375 bool istate, timeread = false;
376 uint64_t sabs, eabs;
377
378 uint32_t report_phy_read_delay = os_atomic_load(&report_phy_read_delay_to, relaxed);
379 uint32_t const trace_phy_read_delay = os_atomic_load(&trace_phy_read_delay_to, relaxed);
380
381 if (__improbable(report_phy_read_delay != 0)) {
382 istate = ml_set_interrupts_enabled_with_debug(false, false);
383 sabs = ml_get_timebase();
384 timeread = true;
385 }
386 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
387 if (__improbable(timeread && simulate_stretched_io)) {
388 sabs -= simulate_stretched_io;
389 }
390 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
391 #endif /* ML_IO_TIMEOUTS_ENABLED */
392
393 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
394 if (pmap_valid_address(paddr)) {
395 addr = phystokv(paddr);
396 use_copy_window = false;
397 }
398 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
399
400 if (use_copy_window) {
401 mp_disable_preemption();
402 unsigned int wimg_bits = pmap_cache_attributes(pn);
403 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
404 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
405 }
406
407 switch (size) {
408 case 1:
409 s1 = *(volatile uint8_t *)addr;
410 result = s1;
411 break;
412 case 2:
413 s2 = *(volatile uint16_t *)addr;
414 result = s2;
415 break;
416 case 4:
417 s4 = *(volatile uint32_t *)addr;
418 result = s4;
419 break;
420 case 8:
421 s8 = *(volatile uint64_t *)addr;
422 result = s8;
423 break;
424 #if BUILD_QUAD_WORD_FUNCS
425 case 16:
426 result = __read128(addr);
427 break;
428 #endif /* BUILD_QUAD_WORD_FUNCS */
429 default:
430 panic("Invalid size %d for ml_phys_read_data", size);
431 break;
432 }
433
434 if (use_copy_window) {
435 pmap_unmap_cpu_windows_copy(index);
436 mp_enable_preemption();
437 }
438
439 #ifdef ML_IO_TIMEOUTS_ENABLED
440 if (__improbable(timeread)) {
441 eabs = ml_get_timebase();
442
443 iotrace(IOTRACE_PHYS_READ, 0, addr, size, result, sabs, eabs - sabs);
444
445 if (__improbable((eabs - sabs) > report_phy_read_delay)) {
446 DTRACE_PHYSLAT4(physread, uint64_t, (eabs - sabs),
447 uint64_t, addr, uint32_t, size, uint64_t, result);
448
449 uint64_t override = 0;
450 override_io_timeouts(0, paddr, &override, NULL);
451
452 if (override != 0) {
453 #if SCHED_HYGIENE_DEBUG
454 /*
455 * The IO timeout was overridden. If we were called in an
456 * interrupt handler context, that can lead to a timeout
457 * panic, so we need to abandon the measurement.
458 */
459 if (interrupt_masked_debug_mode == SCHED_HYGIENE_MODE_PANIC) {
460 ml_irq_debug_abandon();
461 }
462 #endif
463 report_phy_read_delay = override;
464 }
465 }
466
467 if (__improbable((eabs - sabs) > report_phy_read_delay)) {
468 if (phy_read_panic && (machine_timeout_suspended() == FALSE)) {
469 const uint64_t hi = (uint64_t)(result >> 64);
470 const uint64_t lo = (uint64_t)(result);
471 uint64_t nsec = 0;
472 absolutetime_to_nanoseconds(eabs - sabs, &nsec);
473 panic("Read from physical addr 0x%llx took %llu ns, "
474 "result: 0x%016llx%016llx (start: %llu, end: %llu), ceiling: %llu",
475 (unsigned long long)addr, nsec, hi, lo, sabs, eabs,
476 (uint64_t)report_phy_read_delay);
477 }
478 }
479
480 if (__improbable(trace_phy_read_delay > 0 && (eabs - sabs) > trace_phy_read_delay)) {
481 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_READ),
482 (eabs - sabs), sabs, addr, result);
483 }
484
485 ml_set_interrupts_enabled_with_debug(istate, false);
486 }
487 #endif /* ML_IO_TIMEOUTS_ENABLED */
488
489 return result;
490 }
491
492 unsigned int
ml_phys_read(vm_offset_t paddr)493 ml_phys_read(vm_offset_t paddr)
494 {
495 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
496 }
497
498 unsigned int
ml_phys_read_word(vm_offset_t paddr)499 ml_phys_read_word(vm_offset_t paddr)
500 {
501 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
502 }
503
504 unsigned int
ml_phys_read_64(addr64_t paddr64)505 ml_phys_read_64(addr64_t paddr64)
506 {
507 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
508 }
509
510 unsigned int
ml_phys_read_word_64(addr64_t paddr64)511 ml_phys_read_word_64(addr64_t paddr64)
512 {
513 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
514 }
515
516 unsigned int
ml_phys_read_half(vm_offset_t paddr)517 ml_phys_read_half(vm_offset_t paddr)
518 {
519 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2);
520 }
521
522 unsigned int
ml_phys_read_half_64(addr64_t paddr64)523 ml_phys_read_half_64(addr64_t paddr64)
524 {
525 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2);
526 }
527
528 unsigned int
ml_phys_read_byte(vm_offset_t paddr)529 ml_phys_read_byte(vm_offset_t paddr)
530 {
531 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1);
532 }
533
534 unsigned int
ml_phys_read_byte_64(addr64_t paddr64)535 ml_phys_read_byte_64(addr64_t paddr64)
536 {
537 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1);
538 }
539
540 unsigned long long
ml_phys_read_double(vm_offset_t paddr)541 ml_phys_read_double(vm_offset_t paddr)
542 {
543 return ml_phys_read_data((pmap_paddr_t)paddr, 8);
544 }
545
546 unsigned long long
ml_phys_read_double_64(addr64_t paddr64)547 ml_phys_read_double_64(addr64_t paddr64)
548 {
549 return ml_phys_read_data((pmap_paddr_t)paddr64, 8);
550 }
551
552 #if BUILD_QUAD_WORD_FUNCS
553 uint128_t
ml_phys_read_quad(vm_offset_t paddr)554 ml_phys_read_quad(vm_offset_t paddr)
555 {
556 return ml_phys_read_data((pmap_paddr_t)paddr, 16);
557 }
558
559 uint128_t
ml_phys_read_quad_64(addr64_t paddr64)560 ml_phys_read_quad_64(addr64_t paddr64)
561 {
562 return ml_phys_read_data((pmap_paddr_t)paddr64, 16);
563 }
564 #endif /* BUILD_QUAD_WORD_FUNCS */
565
566 /*
567 * Write data to a physical address.
568 */
569
570 #if BUILD_QUAD_WORD_FUNCS
571 static inline void
__write128(vm_address_t addr,uint128_t data)572 __write128(vm_address_t addr, uint128_t data)
573 {
574 const uint64_t hi = (uint64_t)(data >> 64);
575 const uint64_t lo = (uint64_t)(data);
576
577 asm volatile (
578 "stp %[lo], %[hi], [%[addr]]" "\n"
579 : /**/
580 : [lo] "r"(lo), [hi] "r"(hi), [addr] "r"(addr)
581 : "memory"
582 );
583 }
584 #endif /* BUILD_QUAD_WORD_FUNCS */
585
586 static void
ml_phys_write_data(pmap_paddr_t paddr,uint128_t data,int size)587 ml_phys_write_data(pmap_paddr_t paddr, uint128_t data, int size)
588 {
589 vm_address_t addr;
590 ppnum_t pn = atop_kernel(paddr);
591 ppnum_t pn_end = atop_kernel(paddr + size - 1);
592 unsigned int index;
593 bool use_copy_window = true;
594
595 if (__improbable(pn_end != pn)) {
596 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
597 }
598
599 #ifdef ML_IO_TIMEOUTS_ENABLED
600 bool istate, timewrite = false;
601 uint64_t sabs, eabs;
602
603 uint32_t report_phy_write_delay = os_atomic_load(&report_phy_write_delay_to, relaxed);
604 uint32_t const trace_phy_write_delay = os_atomic_load(&trace_phy_write_delay_to, relaxed);
605
606 if (__improbable(report_phy_write_delay != 0)) {
607 istate = ml_set_interrupts_enabled_with_debug(false, false);
608 sabs = ml_get_timebase();
609 timewrite = true;
610 }
611 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
612 if (__improbable(timewrite && simulate_stretched_io)) {
613 sabs -= simulate_stretched_io;
614 }
615 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
616 #endif /* ML_IO_TIMEOUTS_ENABLED */
617
618 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
619 if (pmap_valid_address(paddr)) {
620 addr = phystokv(paddr);
621 use_copy_window = false;
622 }
623 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
624
625 if (use_copy_window) {
626 mp_disable_preemption();
627 unsigned int wimg_bits = pmap_cache_attributes(pn);
628 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
629 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
630 }
631
632 switch (size) {
633 case 1:
634 *(volatile uint8_t *)addr = (uint8_t)data;
635 break;
636 case 2:
637 *(volatile uint16_t *)addr = (uint16_t)data;
638 break;
639 case 4:
640 *(volatile uint32_t *)addr = (uint32_t)data;
641 break;
642 case 8:
643 *(volatile uint64_t *)addr = (uint64_t)data;
644 break;
645 #if BUILD_QUAD_WORD_FUNCS
646 case 16:
647 __write128(addr, data);
648 break;
649 #endif /* BUILD_QUAD_WORD_FUNCS */
650 default:
651 panic("Invalid size %d for ml_phys_write_data", size);
652 }
653
654 if (use_copy_window) {
655 pmap_unmap_cpu_windows_copy(index);
656 mp_enable_preemption();
657 }
658
659 #ifdef ML_IO_TIMEOUTS_ENABLED
660 if (__improbable(timewrite)) {
661 eabs = ml_get_timebase();
662
663 iotrace(IOTRACE_PHYS_WRITE, 0, paddr, size, data, sabs, eabs - sabs);
664
665 if (__improbable((eabs - sabs) > report_phy_write_delay)) {
666 DTRACE_PHYSLAT4(physwrite, uint64_t, (eabs - sabs),
667 uint64_t, paddr, uint32_t, size, uint64_t, data);
668
669 uint64_t override = 0;
670 override_io_timeouts(0, paddr, NULL, &override);
671 if (override != 0) {
672 #if SCHED_HYGIENE_DEBUG
673 /*
674 * The IO timeout was overridden. If we were called in an
675 * interrupt handler context, that can lead to a timeout
676 * panic, so we need to abandon the measurement.
677 */
678 if (interrupt_masked_debug_mode == SCHED_HYGIENE_MODE_PANIC) {
679 ml_irq_debug_abandon();
680 }
681 #endif
682 report_phy_write_delay = override;
683 }
684 }
685
686 if (__improbable((eabs - sabs) > report_phy_write_delay)) {
687 if (phy_write_panic && (machine_timeout_suspended() == FALSE)) {
688 const uint64_t hi = (uint64_t)(data >> 64);
689 const uint64_t lo = (uint64_t)(data);
690 uint64_t nsec = 0;
691 absolutetime_to_nanoseconds(eabs - sabs, &nsec);
692 panic("Write from physical addr 0x%llx took %llu ns, "
693 "data: 0x%016llx%016llx (start: %llu, end: %llu), ceiling: %llu",
694 (unsigned long long)paddr, nsec, hi, lo, sabs, eabs,
695 (uint64_t)report_phy_write_delay);
696 }
697 }
698
699 if (__improbable(trace_phy_write_delay > 0 && (eabs - sabs) > trace_phy_write_delay)) {
700 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_WRITE),
701 (eabs - sabs), sabs, paddr, data);
702 }
703
704 ml_set_interrupts_enabled_with_debug(istate, false);
705 }
706 #endif /* ML_IO_TIMEOUTS_ENABLED */
707 }
708
709 void
ml_phys_write_byte(vm_offset_t paddr,unsigned int data)710 ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
711 {
712 ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
713 }
714
715 void
ml_phys_write_byte_64(addr64_t paddr64,unsigned int data)716 ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
717 {
718 ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
719 }
720
721 void
ml_phys_write_half(vm_offset_t paddr,unsigned int data)722 ml_phys_write_half(vm_offset_t paddr, unsigned int data)
723 {
724 ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
725 }
726
727 void
ml_phys_write_half_64(addr64_t paddr64,unsigned int data)728 ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
729 {
730 ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
731 }
732
733 void
ml_phys_write(vm_offset_t paddr,unsigned int data)734 ml_phys_write(vm_offset_t paddr, unsigned int data)
735 {
736 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
737 }
738
739 void
ml_phys_write_64(addr64_t paddr64,unsigned int data)740 ml_phys_write_64(addr64_t paddr64, unsigned int data)
741 {
742 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
743 }
744
745 void
ml_phys_write_word(vm_offset_t paddr,unsigned int data)746 ml_phys_write_word(vm_offset_t paddr, unsigned int data)
747 {
748 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
749 }
750
751 void
ml_phys_write_word_64(addr64_t paddr64,unsigned int data)752 ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
753 {
754 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
755 }
756
757 void
ml_phys_write_double(vm_offset_t paddr,unsigned long long data)758 ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
759 {
760 ml_phys_write_data((pmap_paddr_t)paddr, data, 8);
761 }
762
763 void
ml_phys_write_double_64(addr64_t paddr64,unsigned long long data)764 ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
765 {
766 ml_phys_write_data((pmap_paddr_t)paddr64, data, 8);
767 }
768
769 #if BUILD_QUAD_WORD_FUNCS
770 void
ml_phys_write_quad(vm_offset_t paddr,uint128_t data)771 ml_phys_write_quad(vm_offset_t paddr, uint128_t data)
772 {
773 ml_phys_write_data((pmap_paddr_t)paddr, data, 16);
774 }
775
776 void
ml_phys_write_quad_64(addr64_t paddr64,uint128_t data)777 ml_phys_write_quad_64(addr64_t paddr64, uint128_t data)
778 {
779 ml_phys_write_data((pmap_paddr_t)paddr64, data, 16);
780 }
781 #endif /* BUILD_QUAD_WORD_FUNCS */
782
783 /*
784 * Set indicated bit in bit string.
785 */
786 void
setbit(int bitno,int * s)787 setbit(int bitno, int *s)
788 {
789 s[bitno / INT_SIZE] |= 1U << (bitno % INT_SIZE);
790 }
791
792 /*
793 * Clear indicated bit in bit string.
794 */
795 void
clrbit(int bitno,int * s)796 clrbit(int bitno, int *s)
797 {
798 s[bitno / INT_SIZE] &= ~(1U << (bitno % INT_SIZE));
799 }
800
801 /*
802 * Test if indicated bit is set in bit string.
803 */
804 int
testbit(int bitno,int * s)805 testbit(int bitno, int *s)
806 {
807 return s[bitno / INT_SIZE] & (1U << (bitno % INT_SIZE));
808 }
809
810 /*
811 * Find first bit set in bit string.
812 */
813 int
ffsbit(int * s)814 ffsbit(int *s)
815 {
816 int offset;
817
818 for (offset = 0; !*s; offset += INT_SIZE, ++s) {
819 ;
820 }
821 return offset + __builtin_ctz(*s);
822 }
823
824 int
ffs(unsigned int mask)825 ffs(unsigned int mask)
826 {
827 if (mask == 0) {
828 return 0;
829 }
830
831 /*
832 * NOTE: cannot use __builtin_ffs because it generates a call to
833 * 'ffs'
834 */
835 return 1 + __builtin_ctz(mask);
836 }
837
838 int
ffsll(unsigned long long mask)839 ffsll(unsigned long long mask)
840 {
841 if (mask == 0) {
842 return 0;
843 }
844
845 /*
846 * NOTE: cannot use __builtin_ffsll because it generates a call to
847 * 'ffsll'
848 */
849 return 1 + __builtin_ctzll(mask);
850 }
851
852 /*
853 * Find last bit set in bit string.
854 */
855 int
fls(unsigned int mask)856 fls(unsigned int mask)
857 {
858 if (mask == 0) {
859 return 0;
860 }
861
862 return (sizeof(mask) << 3) - __builtin_clz(mask);
863 }
864
865 int
flsll(unsigned long long mask)866 flsll(unsigned long long mask)
867 {
868 if (mask == 0) {
869 return 0;
870 }
871
872 return (sizeof(mask) << 3) - __builtin_clzll(mask);
873 }
874
875 kern_return_t
copypv(addr64_t source,addr64_t sink,unsigned int size,int which)876 copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
877 {
878 if ((which & (cppvPsrc | cppvPsnk)) == 0) { /* Make sure that only one is virtual */
879 panic("%s: no more than 1 parameter may be virtual", __func__);
880 }
881
882 kern_return_t res = bcopy_phys_internal(source, sink, size, which);
883
884 #ifndef __ARM_COHERENT_IO__
885 if (which & cppvFsrc) {
886 flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
887 }
888
889 if (which & cppvFsnk) {
890 flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
891 }
892 #endif
893
894 return res;
895 }
896
897 int
clr_be_bit(void)898 clr_be_bit(void)
899 {
900 panic("clr_be_bit");
901 return 0;
902 }
903
904 boolean_t
ml_probe_read(__unused vm_offset_t paddr,__unused unsigned int * val)905 ml_probe_read(
906 __unused vm_offset_t paddr,
907 __unused unsigned int *val)
908 {
909 panic("ml_probe_read() unimplemented");
910 return 1;
911 }
912
913 boolean_t
ml_probe_read_64(__unused addr64_t paddr,__unused unsigned int * val)914 ml_probe_read_64(
915 __unused addr64_t paddr,
916 __unused unsigned int *val)
917 {
918 panic("ml_probe_read_64() unimplemented");
919 return 1;
920 }
921
922
923 void
ml_thread_policy(__unused thread_t thread,__unused unsigned policy_id,__unused unsigned policy_info)924 ml_thread_policy(
925 __unused thread_t thread,
926 __unused unsigned policy_id,
927 __unused unsigned policy_info)
928 {
929 // <rdar://problem/7141284>: Reduce print noise
930 // kprintf("ml_thread_policy() unimplemented\n");
931 }
932
933 __dead2
934 void
panic_unimplemented(void)935 panic_unimplemented(void)
936 {
937 panic("Not yet implemented.");
938 }
939
940 /* ARM64_TODO <rdar://problem/9198953> */
941 void abort(void) __dead2;
942
943 void
abort(void)944 abort(void)
945 {
946 panic("Abort.");
947 }
948
949
950 #if !MACH_KDP
951 void
kdp_register_callout(kdp_callout_fn_t fn,void * arg)952 kdp_register_callout(kdp_callout_fn_t fn, void *arg)
953 {
954 #pragma unused(fn,arg)
955 }
956 #endif
957
958 /*
959 * Get a quick virtual mapping of a physical page and run a callback on that
960 * page's virtual address.
961 *
962 * @param dst64 Physical address to access (doesn't need to be page-aligned).
963 * @param bytes Number of bytes to be accessed. This cannot cross page boundaries.
964 * @param func Callback function to call with the page's virtual address.
965 * @param arg Argument passed directly to `func`.
966 *
967 * @return The return value from `func`.
968 */
969 int
apply_func_phys(addr64_t dst64,vm_size_t bytes,int (* func)(void * buffer,vm_size_t bytes,void * arg),void * arg)970 apply_func_phys(
971 addr64_t dst64,
972 vm_size_t bytes,
973 int (*func)(void * buffer, vm_size_t bytes, void * arg),
974 void * arg)
975 {
976 /* The physical aperture is only guaranteed to work with kernel-managed addresses. */
977 if (!pmap_valid_address(dst64)) {
978 panic("%s address error: passed in address (%#llx) not a kernel managed address",
979 __FUNCTION__, dst64);
980 }
981
982 /* Ensure we stay within a single page */
983 if (((((uint32_t)dst64 & (ARM_PGBYTES - 1)) + bytes) > ARM_PGBYTES)) {
984 panic("%s alignment error: tried accessing addresses spanning more than one page %#llx %#lx",
985 __FUNCTION__, dst64, bytes);
986 }
987
988 return func((void*)phystokv(dst64), bytes, arg);
989 }
990