1 /*
2 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach_assert.h>
30 #include <mach/vm_types.h>
31 #include <mach/mach_time.h>
32 #include <kern/timer.h>
33 #include <kern/clock.h>
34 #include <kern/machine.h>
35 #include <kern/iotrace.h>
36 #include <mach/machine.h>
37 #include <mach/machine/vm_param.h>
38 #include <mach_kdp.h>
39 #include <kdp/kdp_udp.h>
40 #if !MACH_KDP
41 #include <kdp/kdp_callout.h>
42 #endif /* !MACH_KDP */
43 #include <arm/cpu_data.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/caches_internal.h>
46
47 #include <vm/vm_kern.h>
48 #include <vm/vm_map.h>
49 #include <vm/pmap.h>
50
51 #include <arm/misc_protos.h>
52
53 #include <sys/errno.h>
54
55 #include <libkern/section_keywords.h>
56 #include <libkern/OSDebug.h>
57
58
59 #define INT_SIZE (BYTE_SIZE * sizeof (int))
60
61 #define BCOPY_PHYS_SRC_IS_PHYS(flags) (((flags) & cppvPsrc) != 0)
62 #define BCOPY_PHYS_DST_IS_PHYS(flags) (((flags) & cppvPsnk) != 0)
63 #define BCOPY_PHYS_SRC_IS_USER(flags) (((flags) & (cppvPsrc | cppvKmap)) == 0)
64 #define BCOPY_PHYS_DST_IS_USER(flags) (((flags) & (cppvPsnk | cppvKmap)) == 0)
65
66
67 static kern_return_t
bcopy_phys_internal(addr64_t src,addr64_t dst,vm_size_t bytes,int flags)68 bcopy_phys_internal(addr64_t src, addr64_t dst, vm_size_t bytes, int flags)
69 {
70 unsigned int src_index;
71 unsigned int dst_index;
72 vm_offset_t src_offset;
73 vm_offset_t dst_offset;
74 unsigned int wimg_bits_src, wimg_bits_dst;
75 unsigned int cpu_num = 0;
76 ppnum_t pn_src;
77 ppnum_t pn_dst;
78 addr64_t end __assert_only;
79 kern_return_t res = KERN_SUCCESS;
80
81
82 if (!BCOPY_PHYS_SRC_IS_USER(flags)) {
83 assert(!__improbable(os_add_overflow(src, bytes, &end)));
84 }
85 if (!BCOPY_PHYS_DST_IS_USER(flags)) {
86 assert(!__improbable(os_add_overflow(dst, bytes, &end)));
87 }
88
89 while ((bytes > 0) && (res == KERN_SUCCESS)) {
90 src_offset = src & PAGE_MASK;
91 dst_offset = dst & PAGE_MASK;
92 boolean_t use_copy_window_src = FALSE;
93 boolean_t use_copy_window_dst = FALSE;
94 vm_size_t count = bytes;
95 vm_size_t count2 = bytes;
96 if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
97 use_copy_window_src = !pmap_valid_address(src);
98 pn_src = (ppnum_t)(src >> PAGE_SHIFT);
99 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
100 count = PAGE_SIZE - src_offset;
101 wimg_bits_src = pmap_cache_attributes(pn_src);
102 if ((wimg_bits_src & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
103 use_copy_window_src = TRUE;
104 }
105 #else
106 if (use_copy_window_src) {
107 wimg_bits_src = pmap_cache_attributes(pn_src);
108 count = PAGE_SIZE - src_offset;
109 }
110 #endif
111 }
112 if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
113 // write preflighting needed for things like dtrace which may write static read-only mappings
114 use_copy_window_dst = (!pmap_valid_address(dst) || !mmu_kvtop_wpreflight(phystokv((pmap_paddr_t)dst)));
115 pn_dst = (ppnum_t)(dst >> PAGE_SHIFT);
116 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
117 count2 = PAGE_SIZE - dst_offset;
118 wimg_bits_dst = pmap_cache_attributes(pn_dst);
119 if ((wimg_bits_dst & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
120 use_copy_window_dst = TRUE;
121 }
122 #else
123 if (use_copy_window_dst) {
124 wimg_bits_dst = pmap_cache_attributes(pn_dst);
125 count2 = PAGE_SIZE - dst_offset;
126 }
127 #endif
128 }
129
130 char *tmp_src;
131 char *tmp_dst;
132
133 if (use_copy_window_src || use_copy_window_dst) {
134 mp_disable_preemption();
135 cpu_num = cpu_number();
136 }
137
138 if (use_copy_window_src) {
139 src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
140 tmp_src = (char*)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset);
141 } else if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
142 tmp_src = (char*)phystokv_range((pmap_paddr_t)src, &count);
143 } else {
144 tmp_src = (char*)src;
145 }
146 if (use_copy_window_dst) {
147 dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst);
148 tmp_dst = (char*)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset);
149 } else if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
150 tmp_dst = (char*)phystokv_range((pmap_paddr_t)dst, &count2);
151 } else {
152 tmp_dst = (char*)dst;
153 }
154
155 if (count > count2) {
156 count = count2;
157 }
158 if (count > bytes) {
159 count = bytes;
160 }
161
162 if (BCOPY_PHYS_SRC_IS_USER(flags)) {
163 res = copyin((user_addr_t)src, tmp_dst, count);
164 } else if (BCOPY_PHYS_DST_IS_USER(flags)) {
165 res = copyout(tmp_src, (user_addr_t)dst, count);
166 } else {
167 bcopy(tmp_src, tmp_dst, count);
168
169 }
170
171 if (use_copy_window_src) {
172 pmap_unmap_cpu_windows_copy(src_index);
173 }
174 if (use_copy_window_dst) {
175 pmap_unmap_cpu_windows_copy(dst_index);
176 }
177 if (use_copy_window_src || use_copy_window_dst) {
178 mp_enable_preemption();
179 }
180
181 src += count;
182 dst += count;
183 bytes -= count;
184 }
185 return res;
186 }
187
188 void
bcopy_phys(addr64_t src,addr64_t dst,vm_size_t bytes)189 bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
190 {
191 bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk);
192 }
193
194 void
bcopy_phys_with_options(addr64_t src,addr64_t dst,vm_size_t bytes,int options)195 bcopy_phys_with_options(addr64_t src, addr64_t dst, vm_size_t bytes, int options)
196 {
197 bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk | options);
198 }
199
200 extern void *secure_memset(void *, int, size_t);
201
202 static void
bzero_phys_page(vm_offset_t buf)203 bzero_phys_page(vm_offset_t buf)
204 {
205 assert((buf & PAGE_MASK) == 0);
206
207 #pragma clang diagnostic push
208 #pragma clang diagnostic ignored "-Wpass-failed"
209 /*
210 * The unrolling is chosen so that the `add` operands in the codegen
211 * are all immediates and avoid a `mov`
212 */
213 #pragma unroll (4096 / (4 << MMU_CLINE))
214 for (vm_offset_t offset = 0; offset < PAGE_SIZE; offset += (4 << MMU_CLINE)) {
215 asm volatile (
216 "dc zva, %0\n\t"
217 "dc zva, %1\n\t"
218 "dc zva, %2\n\t"
219 "dc zva, %3"
220 :
221 : "r"(buf + offset + (0 << MMU_CLINE))
222 , "r"(buf + offset + (1 << MMU_CLINE))
223 , "r"(buf + offset + (2 << MMU_CLINE))
224 , "r"(buf + offset + (3 << MMU_CLINE))
225 : "memory");
226 }
227 #pragma clang diagnostic pop
228 }
229
230
231 /* Zero bytes starting at a physical address */
232 static void
bzero_phys_internal(addr64_t src,vm_size_t bytes,__unused int options)233 bzero_phys_internal(addr64_t src, vm_size_t bytes, __unused int options)
234 {
235 unsigned int wimg_bits;
236 unsigned int cpu_num = cpu_number();
237 ppnum_t pn;
238 addr64_t end __assert_only;
239
240 assert(!__improbable(os_add_overflow(src, bytes, &end)));
241
242 vm_offset_t offset = src & PAGE_MASK;
243 while (bytes > 0) {
244 vm_size_t count = bytes;
245
246 boolean_t use_copy_window = !pmap_valid_address(src);
247 pn = (ppnum_t)(src >> PAGE_SHIFT);
248 wimg_bits = pmap_cache_attributes(pn);
249 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
250 count = PAGE_SIZE - offset;
251 if ((wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
252 use_copy_window = TRUE;
253 }
254 #else
255 if (use_copy_window) {
256 count = PAGE_SIZE - offset;
257 }
258 #endif
259 char *buf;
260 unsigned int index;
261 if (use_copy_window) {
262 mp_disable_preemption();
263 cpu_num = cpu_number();
264 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
265 buf = (char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset);
266 } else {
267 buf = (char *)phystokv_range((pmap_paddr_t)src, &count);
268 }
269
270 if (count > bytes) {
271 count = bytes;
272 }
273
274 switch (wimg_bits & VM_WIMG_MASK) {
275 case VM_WIMG_DEFAULT:
276 case VM_WIMG_WCOMB:
277 case VM_WIMG_INNERWBACK:
278 case VM_WIMG_WTHRU:
279 #if HAS_UCNORMAL_MEM || APPLEVIRTUALPLATFORM
280 case VM_WIMG_RT:
281 #endif
282 /**
283 * When we are zerofilling a normal page, there are a couple of assumptions that can
284 * be made.
285 *
286 * 1. The destination to be zeroed is page-sized and page-aligned, making the unconditional
287 * 4 stp instructions in bzero redundant.
288 * 2. The dczva loop for zerofilling can be fully unrolled at compile-time thanks to
289 * known size of the destination, reducing instruction fetch overhead caused by
290 * the branch backward in a tight loop.
291 */
292 if (count == PAGE_SIZE) {
293 /**
294 * Thanks to how count is computed above, buf should always be page-size aligned
295 * when count == PAGE_SIZE.
296 */
297 bzero_phys_page((vm_offset_t)buf);
298 } else {
299 bzero(buf, count);
300 }
301 break;
302
303 default:
304 /* 'dc zva' performed by bzero is not safe for device memory */
305 secure_memset((void*)buf, 0, count);
306 }
307
308 if (use_copy_window) {
309 pmap_unmap_cpu_windows_copy(index);
310 mp_enable_preemption();
311 }
312
313 src += count;
314 bytes -= count;
315 offset = 0;
316 }
317 }
318
319 void
bzero_phys_nc(addr64_t src64,vm_size_t bytes)320 bzero_phys_nc(addr64_t src64, vm_size_t bytes)
321 {
322 bzero_phys_internal(src64, bytes, 0);
323 }
324
325 void
bzero_phys(addr64_t src,vm_size_t bytes)326 bzero_phys(addr64_t src, vm_size_t bytes)
327 {
328 bzero_phys_internal(src, bytes, 0);
329 }
330
331 void
bzero_phys_with_options(addr64_t src,vm_size_t bytes,int options)332 bzero_phys_with_options(addr64_t src, vm_size_t bytes, int options)
333 {
334 bzero_phys_internal(src, bytes, options);
335 }
336
337 /*
338 * Read data from a physical address.
339 */
340
341 #if BUILD_QUAD_WORD_FUNCS
342 static inline uint128_t
__read128(vm_address_t addr)343 __read128(vm_address_t addr)
344 {
345 uint64_t hi, lo;
346
347 asm volatile (
348 "ldp %[lo], %[hi], [%[addr]]" "\n"
349 : [lo] "=r"(lo), [hi] "=r"(hi)
350 : [addr] "r"(addr)
351 : "memory"
352 );
353
354 return (((uint128_t)hi) << 64) + lo;
355 }
356 #endif /* BUILD_QUAD_WORD_FUNCS */
357
358 static uint128_t
ml_phys_read_data(pmap_paddr_t paddr,int size)359 ml_phys_read_data(pmap_paddr_t paddr, int size)
360 {
361 vm_address_t addr;
362 ppnum_t pn = atop_kernel(paddr);
363 ppnum_t pn_end = atop_kernel(paddr + size - 1);
364 uint128_t result = 0;
365 uint8_t s1;
366 uint16_t s2;
367 uint32_t s4;
368 uint64_t s8;
369 unsigned int index;
370 bool use_copy_window = true;
371
372 if (__improbable(pn_end != pn)) {
373 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
374 }
375
376 #ifdef ML_IO_TIMEOUTS_ENABLED
377 bool istate, timeread = false;
378 uint64_t sabs, eabs;
379
380 uint32_t report_phy_read_delay = os_atomic_load(&report_phy_read_delay_to, relaxed);
381 uint32_t const trace_phy_read_delay = os_atomic_load(&trace_phy_read_delay_to, relaxed);
382
383 if (__improbable(report_phy_read_delay != 0)) {
384 istate = ml_set_interrupts_enabled_with_debug(false, false);
385 sabs = ml_get_timebase();
386 timeread = true;
387 }
388 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
389 if (__improbable(timeread && simulate_stretched_io)) {
390 sabs -= simulate_stretched_io;
391 }
392 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
393 #endif /* ML_IO_TIMEOUTS_ENABLED */
394
395 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
396 if (pmap_valid_address(paddr)) {
397 addr = phystokv(paddr);
398 use_copy_window = false;
399 }
400 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
401
402 if (use_copy_window) {
403 mp_disable_preemption();
404 unsigned int wimg_bits = pmap_cache_attributes(pn);
405 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
406 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
407 }
408
409 switch (size) {
410 case 1:
411 s1 = *(volatile uint8_t *)addr;
412 result = s1;
413 break;
414 case 2:
415 s2 = *(volatile uint16_t *)addr;
416 result = s2;
417 break;
418 case 4:
419 s4 = *(volatile uint32_t *)addr;
420 result = s4;
421 break;
422 case 8:
423 s8 = *(volatile uint64_t *)addr;
424 result = s8;
425 break;
426 #if BUILD_QUAD_WORD_FUNCS
427 case 16:
428 result = __read128(addr);
429 break;
430 #endif /* BUILD_QUAD_WORD_FUNCS */
431 default:
432 panic("Invalid size %d for ml_phys_read_data", size);
433 break;
434 }
435
436 if (use_copy_window) {
437 pmap_unmap_cpu_windows_copy(index);
438 mp_enable_preemption();
439 }
440
441 #ifdef ML_IO_TIMEOUTS_ENABLED
442 if (__improbable(timeread)) {
443 eabs = ml_get_timebase();
444
445 iotrace(IOTRACE_PHYS_READ, 0, addr, size, result, sabs, eabs - sabs);
446
447 if (__improbable((eabs - sabs) > report_phy_read_delay)) {
448 DTRACE_PHYSLAT4(physread, uint64_t, (eabs - sabs),
449 uint64_t, addr, uint32_t, size, uint64_t, result);
450
451 uint64_t override = 0;
452 override_io_timeouts(0, paddr, &override, NULL);
453
454 if (override != 0) {
455 #if SCHED_HYGIENE_DEBUG
456 /*
457 * The IO timeout was overridden. If we were called in an
458 * interrupt handler context, that can lead to a timeout
459 * panic, so we need to abandon the measurement.
460 */
461 if (interrupt_masked_debug_mode == SCHED_HYGIENE_MODE_PANIC) {
462 ml_irq_debug_abandon();
463 }
464 #endif
465 report_phy_read_delay = override;
466 }
467 }
468
469 if (__improbable((eabs - sabs) > report_phy_read_delay)) {
470 if (phy_read_panic && (machine_timeout_suspended() == FALSE)) {
471 const uint64_t hi = (uint64_t)(result >> 64);
472 const uint64_t lo = (uint64_t)(result);
473 uint64_t nsec = 0;
474 absolutetime_to_nanoseconds(eabs - sabs, &nsec);
475 panic("Read from physical addr 0x%llx took %llu ns, "
476 "result: 0x%016llx%016llx (start: %llu, end: %llu), ceiling: %llu",
477 (unsigned long long)addr, nsec, hi, lo, sabs, eabs,
478 (uint64_t)report_phy_read_delay);
479 }
480 }
481
482 if (__improbable(trace_phy_read_delay > 0 && (eabs - sabs) > trace_phy_read_delay)) {
483 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_READ),
484 (eabs - sabs), sabs, addr, result);
485 }
486
487 ml_set_interrupts_enabled_with_debug(istate, false);
488 }
489 #endif /* ML_IO_TIMEOUTS_ENABLED */
490
491 return result;
492 }
493
494 unsigned int
ml_phys_read(vm_offset_t paddr)495 ml_phys_read(vm_offset_t paddr)
496 {
497 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
498 }
499
500 unsigned int
ml_phys_read_word(vm_offset_t paddr)501 ml_phys_read_word(vm_offset_t paddr)
502 {
503 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
504 }
505
506 unsigned int
ml_phys_read_64(addr64_t paddr64)507 ml_phys_read_64(addr64_t paddr64)
508 {
509 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
510 }
511
512 unsigned int
ml_phys_read_word_64(addr64_t paddr64)513 ml_phys_read_word_64(addr64_t paddr64)
514 {
515 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
516 }
517
518 unsigned int
ml_phys_read_half(vm_offset_t paddr)519 ml_phys_read_half(vm_offset_t paddr)
520 {
521 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2);
522 }
523
524 unsigned int
ml_phys_read_half_64(addr64_t paddr64)525 ml_phys_read_half_64(addr64_t paddr64)
526 {
527 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2);
528 }
529
530 unsigned int
ml_phys_read_byte(vm_offset_t paddr)531 ml_phys_read_byte(vm_offset_t paddr)
532 {
533 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1);
534 }
535
536 unsigned int
ml_phys_read_byte_64(addr64_t paddr64)537 ml_phys_read_byte_64(addr64_t paddr64)
538 {
539 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1);
540 }
541
542 unsigned long long
ml_phys_read_double(vm_offset_t paddr)543 ml_phys_read_double(vm_offset_t paddr)
544 {
545 return ml_phys_read_data((pmap_paddr_t)paddr, 8);
546 }
547
548 unsigned long long
ml_phys_read_double_64(addr64_t paddr64)549 ml_phys_read_double_64(addr64_t paddr64)
550 {
551 return ml_phys_read_data((pmap_paddr_t)paddr64, 8);
552 }
553
554 #if BUILD_QUAD_WORD_FUNCS
555 uint128_t
ml_phys_read_quad(vm_offset_t paddr)556 ml_phys_read_quad(vm_offset_t paddr)
557 {
558 return ml_phys_read_data((pmap_paddr_t)paddr, 16);
559 }
560
561 uint128_t
ml_phys_read_quad_64(addr64_t paddr64)562 ml_phys_read_quad_64(addr64_t paddr64)
563 {
564 return ml_phys_read_data((pmap_paddr_t)paddr64, 16);
565 }
566 #endif /* BUILD_QUAD_WORD_FUNCS */
567
568 /*
569 * Write data to a physical address.
570 */
571
572 #if BUILD_QUAD_WORD_FUNCS
573 static inline void
__write128(vm_address_t addr,uint128_t data)574 __write128(vm_address_t addr, uint128_t data)
575 {
576 const uint64_t hi = (uint64_t)(data >> 64);
577 const uint64_t lo = (uint64_t)(data);
578
579 asm volatile (
580 "stp %[lo], %[hi], [%[addr]]" "\n"
581 : /**/
582 : [lo] "r"(lo), [hi] "r"(hi), [addr] "r"(addr)
583 : "memory"
584 );
585 }
586 #endif /* BUILD_QUAD_WORD_FUNCS */
587
588 static void
ml_phys_write_data(pmap_paddr_t paddr,uint128_t data,int size)589 ml_phys_write_data(pmap_paddr_t paddr, uint128_t data, int size)
590 {
591 vm_address_t addr;
592 ppnum_t pn = atop_kernel(paddr);
593 ppnum_t pn_end = atop_kernel(paddr + size - 1);
594 unsigned int index;
595 bool use_copy_window = true;
596
597 if (__improbable(pn_end != pn)) {
598 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
599 }
600
601 #ifdef ML_IO_TIMEOUTS_ENABLED
602 bool istate, timewrite = false;
603 uint64_t sabs, eabs;
604
605 uint32_t report_phy_write_delay = os_atomic_load(&report_phy_write_delay_to, relaxed);
606 uint32_t const trace_phy_write_delay = os_atomic_load(&trace_phy_write_delay_to, relaxed);
607
608 if (__improbable(report_phy_write_delay != 0)) {
609 istate = ml_set_interrupts_enabled_with_debug(false, false);
610 sabs = ml_get_timebase();
611 timewrite = true;
612 }
613 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
614 if (__improbable(timewrite && simulate_stretched_io)) {
615 sabs -= simulate_stretched_io;
616 }
617 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
618 #endif /* ML_IO_TIMEOUTS_ENABLED */
619
620 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
621 if (pmap_valid_address(paddr)) {
622 addr = phystokv(paddr);
623 use_copy_window = false;
624 }
625 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
626
627 if (use_copy_window) {
628 mp_disable_preemption();
629 unsigned int wimg_bits = pmap_cache_attributes(pn);
630 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
631 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
632 }
633
634 switch (size) {
635 case 1:
636 *(volatile uint8_t *)addr = (uint8_t)data;
637 break;
638 case 2:
639 *(volatile uint16_t *)addr = (uint16_t)data;
640 break;
641 case 4:
642 *(volatile uint32_t *)addr = (uint32_t)data;
643 break;
644 case 8:
645 *(volatile uint64_t *)addr = (uint64_t)data;
646 break;
647 #if BUILD_QUAD_WORD_FUNCS
648 case 16:
649 __write128(addr, data);
650 break;
651 #endif /* BUILD_QUAD_WORD_FUNCS */
652 default:
653 panic("Invalid size %d for ml_phys_write_data", size);
654 }
655
656 if (use_copy_window) {
657 pmap_unmap_cpu_windows_copy(index);
658 mp_enable_preemption();
659 }
660
661 #ifdef ML_IO_TIMEOUTS_ENABLED
662 if (__improbable(timewrite)) {
663 eabs = ml_get_timebase();
664
665 iotrace(IOTRACE_PHYS_WRITE, 0, paddr, size, data, sabs, eabs - sabs);
666
667 if (__improbable((eabs - sabs) > report_phy_write_delay)) {
668 DTRACE_PHYSLAT4(physwrite, uint64_t, (eabs - sabs),
669 uint64_t, paddr, uint32_t, size, uint64_t, data);
670
671 uint64_t override = 0;
672 override_io_timeouts(0, paddr, NULL, &override);
673 if (override != 0) {
674 #if SCHED_HYGIENE_DEBUG
675 /*
676 * The IO timeout was overridden. If we were called in an
677 * interrupt handler context, that can lead to a timeout
678 * panic, so we need to abandon the measurement.
679 */
680 if (interrupt_masked_debug_mode == SCHED_HYGIENE_MODE_PANIC) {
681 ml_irq_debug_abandon();
682 }
683 #endif
684 report_phy_write_delay = override;
685 }
686 }
687
688 if (__improbable((eabs - sabs) > report_phy_write_delay)) {
689 if (phy_write_panic && (machine_timeout_suspended() == FALSE)) {
690 const uint64_t hi = (uint64_t)(data >> 64);
691 const uint64_t lo = (uint64_t)(data);
692 uint64_t nsec = 0;
693 absolutetime_to_nanoseconds(eabs - sabs, &nsec);
694 panic("Write from physical addr 0x%llx took %llu ns, "
695 "data: 0x%016llx%016llx (start: %llu, end: %llu), ceiling: %llu",
696 (unsigned long long)paddr, nsec, hi, lo, sabs, eabs,
697 (uint64_t)report_phy_write_delay);
698 }
699 }
700
701 if (__improbable(trace_phy_write_delay > 0 && (eabs - sabs) > trace_phy_write_delay)) {
702 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_WRITE),
703 (eabs - sabs), sabs, paddr, data);
704 }
705
706 ml_set_interrupts_enabled_with_debug(istate, false);
707 }
708 #endif /* ML_IO_TIMEOUTS_ENABLED */
709 }
710
711 void
ml_phys_write_byte(vm_offset_t paddr,unsigned int data)712 ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
713 {
714 ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
715 }
716
717 void
ml_phys_write_byte_64(addr64_t paddr64,unsigned int data)718 ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
719 {
720 ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
721 }
722
723 void
ml_phys_write_half(vm_offset_t paddr,unsigned int data)724 ml_phys_write_half(vm_offset_t paddr, unsigned int data)
725 {
726 ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
727 }
728
729 void
ml_phys_write_half_64(addr64_t paddr64,unsigned int data)730 ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
731 {
732 ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
733 }
734
735 void
ml_phys_write(vm_offset_t paddr,unsigned int data)736 ml_phys_write(vm_offset_t paddr, unsigned int data)
737 {
738 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
739 }
740
741 void
ml_phys_write_64(addr64_t paddr64,unsigned int data)742 ml_phys_write_64(addr64_t paddr64, unsigned int data)
743 {
744 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
745 }
746
747 void
ml_phys_write_word(vm_offset_t paddr,unsigned int data)748 ml_phys_write_word(vm_offset_t paddr, unsigned int data)
749 {
750 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
751 }
752
753 void
ml_phys_write_word_64(addr64_t paddr64,unsigned int data)754 ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
755 {
756 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
757 }
758
759 void
ml_phys_write_double(vm_offset_t paddr,unsigned long long data)760 ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
761 {
762 ml_phys_write_data((pmap_paddr_t)paddr, data, 8);
763 }
764
765 void
ml_phys_write_double_64(addr64_t paddr64,unsigned long long data)766 ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
767 {
768 ml_phys_write_data((pmap_paddr_t)paddr64, data, 8);
769 }
770
771 #if BUILD_QUAD_WORD_FUNCS
772 void
ml_phys_write_quad(vm_offset_t paddr,uint128_t data)773 ml_phys_write_quad(vm_offset_t paddr, uint128_t data)
774 {
775 ml_phys_write_data((pmap_paddr_t)paddr, data, 16);
776 }
777
778 void
ml_phys_write_quad_64(addr64_t paddr64,uint128_t data)779 ml_phys_write_quad_64(addr64_t paddr64, uint128_t data)
780 {
781 ml_phys_write_data((pmap_paddr_t)paddr64, data, 16);
782 }
783 #endif /* BUILD_QUAD_WORD_FUNCS */
784
785 /*
786 * Set indicated bit in bit string.
787 */
788 void
setbit(int bitno,int * s)789 setbit(int bitno, int *s)
790 {
791 s[bitno / INT_SIZE] |= 1U << (bitno % INT_SIZE);
792 }
793
794 /*
795 * Clear indicated bit in bit string.
796 */
797 void
clrbit(int bitno,int * s)798 clrbit(int bitno, int *s)
799 {
800 s[bitno / INT_SIZE] &= ~(1U << (bitno % INT_SIZE));
801 }
802
803 /*
804 * Test if indicated bit is set in bit string.
805 */
806 int
testbit(int bitno,int * s)807 testbit(int bitno, int *s)
808 {
809 return s[bitno / INT_SIZE] & (1U << (bitno % INT_SIZE));
810 }
811
812 /*
813 * Find first bit set in bit string.
814 */
815 int
ffsbit(int * s)816 ffsbit(int *s)
817 {
818 int offset;
819
820 for (offset = 0; !*s; offset += INT_SIZE, ++s) {
821 ;
822 }
823 return offset + __builtin_ctz(*s);
824 }
825
826 int
ffs(unsigned int mask)827 ffs(unsigned int mask)
828 {
829 if (mask == 0) {
830 return 0;
831 }
832
833 /*
834 * NOTE: cannot use __builtin_ffs because it generates a call to
835 * 'ffs'
836 */
837 return 1 + __builtin_ctz(mask);
838 }
839
840 int
ffsll(unsigned long long mask)841 ffsll(unsigned long long mask)
842 {
843 if (mask == 0) {
844 return 0;
845 }
846
847 /*
848 * NOTE: cannot use __builtin_ffsll because it generates a call to
849 * 'ffsll'
850 */
851 return 1 + __builtin_ctzll(mask);
852 }
853
854 /*
855 * Find last bit set in bit string.
856 */
857 int
fls(unsigned int mask)858 fls(unsigned int mask)
859 {
860 if (mask == 0) {
861 return 0;
862 }
863
864 return (sizeof(mask) << 3) - __builtin_clz(mask);
865 }
866
867 int
flsll(unsigned long long mask)868 flsll(unsigned long long mask)
869 {
870 if (mask == 0) {
871 return 0;
872 }
873
874 return (sizeof(mask) << 3) - __builtin_clzll(mask);
875 }
876
877 kern_return_t
copypv(addr64_t source,addr64_t sink,unsigned int size,int which)878 copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
879 {
880 if ((which & (cppvPsrc | cppvPsnk)) == 0) { /* Make sure that only one is virtual */
881 panic("%s: no more than 1 parameter may be virtual", __func__);
882 }
883
884 kern_return_t res = bcopy_phys_internal(source, sink, size, which);
885
886 #ifndef __ARM_COHERENT_IO__
887 if (which & cppvFsrc) {
888 flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
889 }
890
891 if (which & cppvFsnk) {
892 flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
893 }
894 #endif
895
896 return res;
897 }
898
899 int
clr_be_bit(void)900 clr_be_bit(void)
901 {
902 panic("clr_be_bit");
903 return 0;
904 }
905
906 boolean_t
ml_probe_read(__unused vm_offset_t paddr,__unused unsigned int * val)907 ml_probe_read(
908 __unused vm_offset_t paddr,
909 __unused unsigned int *val)
910 {
911 panic("ml_probe_read() unimplemented");
912 return 1;
913 }
914
915 boolean_t
ml_probe_read_64(__unused addr64_t paddr,__unused unsigned int * val)916 ml_probe_read_64(
917 __unused addr64_t paddr,
918 __unused unsigned int *val)
919 {
920 panic("ml_probe_read_64() unimplemented");
921 return 1;
922 }
923
924
925 void
ml_thread_policy(__unused thread_t thread,__unused unsigned policy_id,__unused unsigned policy_info)926 ml_thread_policy(
927 __unused thread_t thread,
928 __unused unsigned policy_id,
929 __unused unsigned policy_info)
930 {
931 // <rdar://problem/7141284>: Reduce print noise
932 // kprintf("ml_thread_policy() unimplemented\n");
933 }
934
935 __dead2
936 void
panic_unimplemented(void)937 panic_unimplemented(void)
938 {
939 panic("Not yet implemented.");
940 }
941
942 /* ARM64_TODO <rdar://problem/9198953> */
943 void abort(void) __dead2;
944
945 void
abort(void)946 abort(void)
947 {
948 panic("Abort.");
949 }
950
951
952 #if !MACH_KDP
953 void
kdp_register_callout(kdp_callout_fn_t fn,void * arg)954 kdp_register_callout(kdp_callout_fn_t fn, void *arg)
955 {
956 #pragma unused(fn,arg)
957 }
958 #endif
959
960 /*
961 * Get a quick virtual mapping of a physical page and run a callback on that
962 * page's virtual address.
963 *
964 * @param dst64 Physical address to access (doesn't need to be page-aligned).
965 * @param bytes Number of bytes to be accessed. This cannot cross page boundaries.
966 * @param func Callback function to call with the page's virtual address.
967 * @param arg Argument passed directly to `func`.
968 *
969 * @return The return value from `func`.
970 */
971 int
apply_func_phys(addr64_t dst64,vm_size_t bytes,int (* func)(void * buffer,vm_size_t bytes,void * arg),void * arg)972 apply_func_phys(
973 addr64_t dst64,
974 vm_size_t bytes,
975 int (*func)(void * buffer, vm_size_t bytes, void * arg),
976 void * arg)
977 {
978 /* The physical aperture is only guaranteed to work with kernel-managed addresses. */
979 if (!pmap_valid_address(dst64)) {
980 panic("%s address error: passed in address (%#llx) not a kernel managed address",
981 __FUNCTION__, dst64);
982 }
983
984 /* Ensure we stay within a single page */
985 if (((((uint32_t)dst64 & (ARM_PGBYTES - 1)) + bytes) > ARM_PGBYTES)) {
986 panic("%s alignment error: tried accessing addresses spanning more than one page %#llx %#lx",
987 __FUNCTION__, dst64, bytes);
988 }
989
990 return func((void*)phystokv(dst64), bytes, arg);
991 }
992