1 /*
2 * Copyright (c) 2007-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <mach_assert.h>
30 #include <mach/vm_types.h>
31 #include <mach/mach_time.h>
32 #include <kern/timer.h>
33 #include <kern/clock.h>
34 #include <kern/machine.h>
35 #include <kern/iotrace.h>
36 #include <mach/machine.h>
37 #include <mach/machine/vm_param.h>
38 #include <mach_kdp.h>
39 #include <kdp/kdp_udp.h>
40 #if !MACH_KDP
41 #include <kdp/kdp_callout.h>
42 #endif /* !MACH_KDP */
43 #include <arm/cpu_data.h>
44 #include <arm/cpu_data_internal.h>
45 #include <arm/caches_internal.h>
46
47 #include <vm/vm_kern.h>
48 #include <vm/vm_map.h>
49 #include <vm/pmap.h>
50
51 #include <arm/misc_protos.h>
52
53 #include <sys/errno.h>
54
55 #include <libkern/section_keywords.h>
56 #include <libkern/OSDebug.h>
57
58 #define INT_SIZE (BYTE_SIZE * sizeof (int))
59
60 #define BCOPY_PHYS_SRC_IS_PHYS(flags) (((flags) & cppvPsrc) != 0)
61 #define BCOPY_PHYS_DST_IS_PHYS(flags) (((flags) & cppvPsnk) != 0)
62 #define BCOPY_PHYS_SRC_IS_USER(flags) (((flags) & (cppvPsrc | cppvKmap)) == 0)
63 #define BCOPY_PHYS_DST_IS_USER(flags) (((flags) & (cppvPsnk | cppvKmap)) == 0)
64
65 static kern_return_t
bcopy_phys_internal(addr64_t src,addr64_t dst,vm_size_t bytes,int flags)66 bcopy_phys_internal(addr64_t src, addr64_t dst, vm_size_t bytes, int flags)
67 {
68 unsigned int src_index;
69 unsigned int dst_index;
70 vm_offset_t src_offset;
71 vm_offset_t dst_offset;
72 unsigned int wimg_bits_src, wimg_bits_dst;
73 unsigned int cpu_num = 0;
74 ppnum_t pn_src;
75 ppnum_t pn_dst;
76 addr64_t end __assert_only;
77 kern_return_t res = KERN_SUCCESS;
78
79 if (!BCOPY_PHYS_SRC_IS_USER(flags)) {
80 assert(!__improbable(os_add_overflow(src, bytes, &end)));
81 }
82 if (!BCOPY_PHYS_DST_IS_USER(flags)) {
83 assert(!__improbable(os_add_overflow(dst, bytes, &end)));
84 }
85
86 while ((bytes > 0) && (res == KERN_SUCCESS)) {
87 src_offset = src & PAGE_MASK;
88 dst_offset = dst & PAGE_MASK;
89 boolean_t use_copy_window_src = FALSE;
90 boolean_t use_copy_window_dst = FALSE;
91 vm_size_t count = bytes;
92 vm_size_t count2 = bytes;
93 if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
94 use_copy_window_src = !pmap_valid_address(src);
95 pn_src = (ppnum_t)(src >> PAGE_SHIFT);
96 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
97 count = PAGE_SIZE - src_offset;
98 wimg_bits_src = pmap_cache_attributes(pn_src);
99 if ((wimg_bits_src & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
100 use_copy_window_src = TRUE;
101 }
102 #else
103 if (use_copy_window_src) {
104 wimg_bits_src = pmap_cache_attributes(pn_src);
105 count = PAGE_SIZE - src_offset;
106 }
107 #endif
108 }
109 if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
110 // write preflighting needed for things like dtrace which may write static read-only mappings
111 use_copy_window_dst = (!pmap_valid_address(dst) || !mmu_kvtop_wpreflight(phystokv((pmap_paddr_t)dst)));
112 pn_dst = (ppnum_t)(dst >> PAGE_SHIFT);
113 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
114 count2 = PAGE_SIZE - dst_offset;
115 wimg_bits_dst = pmap_cache_attributes(pn_dst);
116 if ((wimg_bits_dst & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
117 use_copy_window_dst = TRUE;
118 }
119 #else
120 if (use_copy_window_dst) {
121 wimg_bits_dst = pmap_cache_attributes(pn_dst);
122 count2 = PAGE_SIZE - dst_offset;
123 }
124 #endif
125 }
126
127 char *tmp_src;
128 char *tmp_dst;
129
130 if (use_copy_window_src || use_copy_window_dst) {
131 mp_disable_preemption();
132 cpu_num = cpu_number();
133 }
134
135 if (use_copy_window_src) {
136 src_index = pmap_map_cpu_windows_copy(pn_src, VM_PROT_READ, wimg_bits_src);
137 tmp_src = (char*)(pmap_cpu_windows_copy_addr(cpu_num, src_index) + src_offset);
138 } else if (BCOPY_PHYS_SRC_IS_PHYS(flags)) {
139 tmp_src = (char*)phystokv_range((pmap_paddr_t)src, &count);
140 } else {
141 tmp_src = (char*)src;
142 }
143 if (use_copy_window_dst) {
144 dst_index = pmap_map_cpu_windows_copy(pn_dst, VM_PROT_READ | VM_PROT_WRITE, wimg_bits_dst);
145 tmp_dst = (char*)(pmap_cpu_windows_copy_addr(cpu_num, dst_index) + dst_offset);
146 } else if (BCOPY_PHYS_DST_IS_PHYS(flags)) {
147 tmp_dst = (char*)phystokv_range((pmap_paddr_t)dst, &count2);
148 } else {
149 tmp_dst = (char*)dst;
150 }
151
152 if (count > count2) {
153 count = count2;
154 }
155 if (count > bytes) {
156 count = bytes;
157 }
158
159 if (BCOPY_PHYS_SRC_IS_USER(flags)) {
160 res = copyin((user_addr_t)src, tmp_dst, count);
161 } else if (BCOPY_PHYS_DST_IS_USER(flags)) {
162 res = copyout(tmp_src, (user_addr_t)dst, count);
163 } else {
164 bcopy(tmp_src, tmp_dst, count);
165 }
166
167 if (use_copy_window_src) {
168 pmap_unmap_cpu_windows_copy(src_index);
169 }
170 if (use_copy_window_dst) {
171 pmap_unmap_cpu_windows_copy(dst_index);
172 }
173 if (use_copy_window_src || use_copy_window_dst) {
174 mp_enable_preemption();
175 }
176
177 src += count;
178 dst += count;
179 bytes -= count;
180 }
181 return res;
182 }
183
184 void
bcopy_phys(addr64_t src,addr64_t dst,vm_size_t bytes)185 bcopy_phys(addr64_t src, addr64_t dst, vm_size_t bytes)
186 {
187 bcopy_phys_internal(src, dst, bytes, cppvPsrc | cppvPsnk);
188 }
189
190 void
bzero_phys_nc(addr64_t src64,vm_size_t bytes)191 bzero_phys_nc(addr64_t src64, vm_size_t bytes)
192 {
193 bzero_phys(src64, bytes);
194 }
195
196 extern void *secure_memset(void *, int, size_t);
197
198 /* Zero bytes starting at a physical address */
199 void
bzero_phys(addr64_t src,vm_size_t bytes)200 bzero_phys(addr64_t src, vm_size_t bytes)
201 {
202 unsigned int wimg_bits;
203 unsigned int cpu_num = cpu_number();
204 ppnum_t pn;
205 addr64_t end __assert_only;
206
207 assert(!__improbable(os_add_overflow(src, bytes, &end)));
208
209 vm_offset_t offset = src & PAGE_MASK;
210 while (bytes > 0) {
211 vm_size_t count = bytes;
212
213 boolean_t use_copy_window = !pmap_valid_address(src);
214 pn = (ppnum_t)(src >> PAGE_SHIFT);
215 wimg_bits = pmap_cache_attributes(pn);
216 #if !defined(__ARM_COHERENT_IO__) && !__ARM_PTE_PHYSMAP__
217 count = PAGE_SIZE - offset;
218 if ((wimg_bits & VM_WIMG_MASK) != VM_WIMG_DEFAULT) {
219 use_copy_window = TRUE;
220 }
221 #else
222 if (use_copy_window) {
223 count = PAGE_SIZE - offset;
224 }
225 #endif
226 char *buf;
227 unsigned int index;
228 if (use_copy_window) {
229 mp_disable_preemption();
230 cpu_num = cpu_number();
231 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
232 buf = (char *)(pmap_cpu_windows_copy_addr(cpu_num, index) + offset);
233 } else {
234 buf = (char *)phystokv_range((pmap_paddr_t)src, &count);
235 }
236
237 if (count > bytes) {
238 count = bytes;
239 }
240
241 switch (wimg_bits & VM_WIMG_MASK) {
242 case VM_WIMG_DEFAULT:
243 case VM_WIMG_WCOMB:
244 case VM_WIMG_INNERWBACK:
245 case VM_WIMG_WTHRU:
246 #if HAS_UCNORMAL_MEM
247 case VM_WIMG_RT:
248 #endif
249 bzero(buf, count);
250 break;
251 default:
252 /* 'dc zva' performed by bzero is not safe for device memory */
253 secure_memset((void*)buf, 0, count);
254 }
255
256 if (use_copy_window) {
257 pmap_unmap_cpu_windows_copy(index);
258 mp_enable_preemption();
259 }
260
261 src += count;
262 bytes -= count;
263 offset = 0;
264 }
265 }
266
267 /*
268 * Read data from a physical address.
269 */
270
271
272 static uint64_t
ml_phys_read_data(pmap_paddr_t paddr,int size)273 ml_phys_read_data(pmap_paddr_t paddr, int size)
274 {
275 vm_address_t addr;
276 ppnum_t pn = atop_kernel(paddr);
277 ppnum_t pn_end = atop_kernel(paddr + size - 1);
278 uint64_t result = 0;
279 uint8_t s1;
280 uint16_t s2;
281 uint32_t s4;
282 unsigned int index;
283 bool use_copy_window = true;
284
285 if (__improbable(pn_end != pn)) {
286 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
287 }
288
289 #ifdef ML_IO_TIMEOUTS_ENABLED
290 bool istate, timeread = false;
291 uint64_t sabs, eabs;
292
293 uint32_t const report_phy_read_delay = os_atomic_load(&report_phy_read_delay_to, relaxed);
294 uint32_t const trace_phy_read_delay = os_atomic_load(&trace_phy_read_delay_to, relaxed);
295
296 if (__improbable(report_phy_read_delay != 0)) {
297 istate = ml_set_interrupts_enabled(FALSE);
298 sabs = mach_absolute_time();
299 timeread = true;
300 }
301 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
302 if (__improbable(timeread && simulate_stretched_io)) {
303 sabs -= simulate_stretched_io;
304 }
305 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
306 #endif /* ML_IO_TIMEOUTS_ENABLED */
307
308 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
309 if (pmap_valid_address(paddr)) {
310 addr = phystokv(paddr);
311 use_copy_window = false;
312 }
313 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
314
315 if (use_copy_window) {
316 mp_disable_preemption();
317 unsigned int wimg_bits = pmap_cache_attributes(pn);
318 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ, wimg_bits);
319 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
320 }
321
322 switch (size) {
323 case 1:
324 s1 = *(volatile uint8_t *)addr;
325 result = s1;
326 break;
327 case 2:
328 s2 = *(volatile uint16_t *)addr;
329 result = s2;
330 break;
331 case 4:
332 s4 = *(volatile uint32_t *)addr;
333 result = s4;
334 break;
335 case 8:
336 result = *(volatile uint64_t *)addr;
337 break;
338 default:
339 panic("Invalid size %d for ml_phys_read_data", size);
340 break;
341 }
342
343 if (use_copy_window) {
344 pmap_unmap_cpu_windows_copy(index);
345 mp_enable_preemption();
346 }
347
348 #ifdef ML_IO_TIMEOUTS_ENABLED
349 if (__improbable(timeread)) {
350 eabs = mach_absolute_time();
351
352 iotrace(IOTRACE_PHYS_READ, 0, addr, size, result, sabs, eabs - sabs);
353
354 if (__improbable((eabs - sabs) > report_phy_read_delay)) {
355 ml_set_interrupts_enabled(istate);
356
357 if (phy_read_panic && (machine_timeout_suspended() == FALSE)) {
358 panic("Read from physical addr 0x%llx took %llu ns, "
359 "result: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
360 (unsigned long long)addr, (eabs - sabs), result, sabs, eabs,
361 (uint64_t)report_phy_read_delay);
362 }
363
364 if (report_phy_read_osbt) {
365 OSReportWithBacktrace("ml_phys_read_data took %llu us",
366 (eabs - sabs) / NSEC_PER_USEC);
367 }
368 #if CONFIG_DTRACE
369 DTRACE_PHYSLAT4(physread, uint64_t, (eabs - sabs),
370 uint64_t, addr, uint32_t, size, uint64_t, result);
371 #endif /* CONFIG_DTRACE */
372 } else if (__improbable(trace_phy_read_delay > 0 && (eabs - sabs) > trace_phy_read_delay)) {
373 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_READ),
374 (eabs - sabs), sabs, addr, result);
375
376 ml_set_interrupts_enabled(istate);
377 } else {
378 ml_set_interrupts_enabled(istate);
379 }
380 }
381 #endif /* ML_IO_TIMEOUTS_ENABLED */
382
383 return result;
384 }
385
386 unsigned int
ml_phys_read(vm_offset_t paddr)387 ml_phys_read(vm_offset_t paddr)
388 {
389 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
390 }
391
392 unsigned int
ml_phys_read_word(vm_offset_t paddr)393 ml_phys_read_word(vm_offset_t paddr)
394 {
395 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 4);
396 }
397
398 unsigned int
ml_phys_read_64(addr64_t paddr64)399 ml_phys_read_64(addr64_t paddr64)
400 {
401 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
402 }
403
404 unsigned int
ml_phys_read_word_64(addr64_t paddr64)405 ml_phys_read_word_64(addr64_t paddr64)
406 {
407 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 4);
408 }
409
410 unsigned int
ml_phys_read_half(vm_offset_t paddr)411 ml_phys_read_half(vm_offset_t paddr)
412 {
413 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 2);
414 }
415
416 unsigned int
ml_phys_read_half_64(addr64_t paddr64)417 ml_phys_read_half_64(addr64_t paddr64)
418 {
419 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 2);
420 }
421
422 unsigned int
ml_phys_read_byte(vm_offset_t paddr)423 ml_phys_read_byte(vm_offset_t paddr)
424 {
425 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr, 1);
426 }
427
428 unsigned int
ml_phys_read_byte_64(addr64_t paddr64)429 ml_phys_read_byte_64(addr64_t paddr64)
430 {
431 return (unsigned int)ml_phys_read_data((pmap_paddr_t)paddr64, 1);
432 }
433
434 unsigned long long
ml_phys_read_double(vm_offset_t paddr)435 ml_phys_read_double(vm_offset_t paddr)
436 {
437 return ml_phys_read_data((pmap_paddr_t)paddr, 8);
438 }
439
440 unsigned long long
ml_phys_read_double_64(addr64_t paddr64)441 ml_phys_read_double_64(addr64_t paddr64)
442 {
443 return ml_phys_read_data((pmap_paddr_t)paddr64, 8);
444 }
445
446
447
448 /*
449 * Write data to a physical address.
450 */
451
452 static void
ml_phys_write_data(pmap_paddr_t paddr,uint64_t data,int size)453 ml_phys_write_data(pmap_paddr_t paddr, uint64_t data, int size)
454 {
455 vm_address_t addr;
456 ppnum_t pn = atop_kernel(paddr);
457 ppnum_t pn_end = atop_kernel(paddr + size - 1);
458 unsigned int index;
459 bool use_copy_window = true;
460
461 if (__improbable(pn_end != pn)) {
462 panic("%s: paddr 0x%llx spans a page boundary", __func__, (uint64_t)paddr);
463 }
464
465 #ifdef ML_IO_TIMEOUTS_ENABLED
466 bool istate, timewrite = false;
467 uint64_t sabs, eabs;
468
469 uint32_t const report_phy_write_delay = os_atomic_load(&report_phy_write_delay_to, relaxed);
470 uint32_t const trace_phy_write_delay = os_atomic_load(&trace_phy_write_delay_to, relaxed);
471
472 if (__improbable(report_phy_write_delay != 0)) {
473 istate = ml_set_interrupts_enabled(FALSE);
474 sabs = mach_absolute_time();
475 timewrite = true;
476 }
477 #ifdef ML_IO_SIMULATE_STRETCHED_ENABLED
478 if (__improbable(timewrite && simulate_stretched_io)) {
479 sabs -= simulate_stretched_io;
480 }
481 #endif /* ML_IO_SIMULATE_STRETCHED_ENABLED */
482 #endif /* ML_IO_TIMEOUTS_ENABLED */
483
484 #if defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__
485 if (pmap_valid_address(paddr)) {
486 addr = phystokv(paddr);
487 use_copy_window = false;
488 }
489 #endif /* defined(__ARM_COHERENT_IO__) || __ARM_PTE_PHYSMAP__ */
490
491 if (use_copy_window) {
492 mp_disable_preemption();
493 unsigned int wimg_bits = pmap_cache_attributes(pn);
494 index = pmap_map_cpu_windows_copy(pn, VM_PROT_READ | VM_PROT_WRITE, wimg_bits);
495 addr = pmap_cpu_windows_copy_addr(cpu_number(), index) | ((uint32_t)paddr & PAGE_MASK);
496 }
497
498 switch (size) {
499 case 1:
500 *(volatile uint8_t *)addr = (uint8_t)data;
501 break;
502 case 2:
503 *(volatile uint16_t *)addr = (uint16_t)data;
504 break;
505 case 4:
506 *(volatile uint32_t *)addr = (uint32_t)data;
507 break;
508 case 8:
509 *(volatile uint64_t *)addr = data;
510 break;
511 default:
512 panic("Invalid size %d for ml_phys_write_data", size);
513 }
514
515 if (use_copy_window) {
516 pmap_unmap_cpu_windows_copy(index);
517 mp_enable_preemption();
518 }
519
520 #ifdef ML_IO_TIMEOUTS_ENABLED
521 if (__improbable(timewrite)) {
522 eabs = mach_absolute_time();
523
524 iotrace(IOTRACE_PHYS_WRITE, 0, paddr, size, data, sabs, eabs - sabs);
525
526 if (__improbable((eabs - sabs) > report_phy_write_delay)) {
527 ml_set_interrupts_enabled(istate);
528
529 if (phy_write_panic && (machine_timeout_suspended() == FALSE)) {
530 panic("Write from physical addr 0x%llx took %llu ns, "
531 "data: 0x%llx (start: %llu, end: %llu), ceiling: %llu",
532 (unsigned long long)paddr, (eabs - sabs), data, sabs, eabs,
533 (uint64_t)report_phy_write_delay);
534 }
535
536 if (report_phy_write_osbt) {
537 OSReportWithBacktrace("ml_phys_write_data took %llu us",
538 (eabs - sabs) / NSEC_PER_USEC);
539 }
540 #if CONFIG_DTRACE
541 DTRACE_PHYSLAT4(physwrite, uint64_t, (eabs - sabs),
542 uint64_t, paddr, uint32_t, size, uint64_t, data);
543 #endif /* CONFIG_DTRACE */
544 } else if (__improbable(trace_phy_write_delay > 0 && (eabs - sabs) > trace_phy_write_delay)) {
545 KDBG(MACHDBG_CODE(DBG_MACH_IO, DBC_MACH_IO_PHYS_WRITE),
546 (eabs - sabs), sabs, paddr, data);
547
548 ml_set_interrupts_enabled(istate);
549 } else {
550 ml_set_interrupts_enabled(istate);
551 }
552 }
553 #endif /* ML_IO_TIMEOUTS_ENABLED */
554 }
555
556 void
ml_phys_write_byte(vm_offset_t paddr,unsigned int data)557 ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
558 {
559 ml_phys_write_data((pmap_paddr_t)paddr, data, 1);
560 }
561
562 void
ml_phys_write_byte_64(addr64_t paddr64,unsigned int data)563 ml_phys_write_byte_64(addr64_t paddr64, unsigned int data)
564 {
565 ml_phys_write_data((pmap_paddr_t)paddr64, data, 1);
566 }
567
568 void
ml_phys_write_half(vm_offset_t paddr,unsigned int data)569 ml_phys_write_half(vm_offset_t paddr, unsigned int data)
570 {
571 ml_phys_write_data((pmap_paddr_t)paddr, data, 2);
572 }
573
574 void
ml_phys_write_half_64(addr64_t paddr64,unsigned int data)575 ml_phys_write_half_64(addr64_t paddr64, unsigned int data)
576 {
577 ml_phys_write_data((pmap_paddr_t)paddr64, data, 2);
578 }
579
580 void
ml_phys_write(vm_offset_t paddr,unsigned int data)581 ml_phys_write(vm_offset_t paddr, unsigned int data)
582 {
583 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
584 }
585
586 void
ml_phys_write_64(addr64_t paddr64,unsigned int data)587 ml_phys_write_64(addr64_t paddr64, unsigned int data)
588 {
589 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
590 }
591
592 void
ml_phys_write_word(vm_offset_t paddr,unsigned int data)593 ml_phys_write_word(vm_offset_t paddr, unsigned int data)
594 {
595 ml_phys_write_data((pmap_paddr_t)paddr, data, 4);
596 }
597
598 void
ml_phys_write_word_64(addr64_t paddr64,unsigned int data)599 ml_phys_write_word_64(addr64_t paddr64, unsigned int data)
600 {
601 ml_phys_write_data((pmap_paddr_t)paddr64, data, 4);
602 }
603
604 void
ml_phys_write_double(vm_offset_t paddr,unsigned long long data)605 ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
606 {
607 ml_phys_write_data((pmap_paddr_t)paddr, data, 8);
608 }
609
610 void
ml_phys_write_double_64(addr64_t paddr64,unsigned long long data)611 ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
612 {
613 ml_phys_write_data((pmap_paddr_t)paddr64, data, 8);
614 }
615
616
617 /*
618 * Set indicated bit in bit string.
619 */
620 void
setbit(int bitno,int * s)621 setbit(int bitno, int *s)
622 {
623 s[bitno / INT_SIZE] |= 1U << (bitno % INT_SIZE);
624 }
625
626 /*
627 * Clear indicated bit in bit string.
628 */
629 void
clrbit(int bitno,int * s)630 clrbit(int bitno, int *s)
631 {
632 s[bitno / INT_SIZE] &= ~(1U << (bitno % INT_SIZE));
633 }
634
635 /*
636 * Test if indicated bit is set in bit string.
637 */
638 int
testbit(int bitno,int * s)639 testbit(int bitno, int *s)
640 {
641 return s[bitno / INT_SIZE] & (1U << (bitno % INT_SIZE));
642 }
643
644 /*
645 * Find first bit set in bit string.
646 */
647 int
ffsbit(int * s)648 ffsbit(int *s)
649 {
650 int offset;
651
652 for (offset = 0; !*s; offset += INT_SIZE, ++s) {
653 ;
654 }
655 return offset + __builtin_ctz(*s);
656 }
657
658 int
ffs(unsigned int mask)659 ffs(unsigned int mask)
660 {
661 if (mask == 0) {
662 return 0;
663 }
664
665 /*
666 * NOTE: cannot use __builtin_ffs because it generates a call to
667 * 'ffs'
668 */
669 return 1 + __builtin_ctz(mask);
670 }
671
672 int
ffsll(unsigned long long mask)673 ffsll(unsigned long long mask)
674 {
675 if (mask == 0) {
676 return 0;
677 }
678
679 /*
680 * NOTE: cannot use __builtin_ffsll because it generates a call to
681 * 'ffsll'
682 */
683 return 1 + __builtin_ctzll(mask);
684 }
685
686 /*
687 * Find last bit set in bit string.
688 */
689 int
fls(unsigned int mask)690 fls(unsigned int mask)
691 {
692 if (mask == 0) {
693 return 0;
694 }
695
696 return (sizeof(mask) << 3) - __builtin_clz(mask);
697 }
698
699 int
flsll(unsigned long long mask)700 flsll(unsigned long long mask)
701 {
702 if (mask == 0) {
703 return 0;
704 }
705
706 return (sizeof(mask) << 3) - __builtin_clzll(mask);
707 }
708
709 kern_return_t
copypv(addr64_t source,addr64_t sink,unsigned int size,int which)710 copypv(addr64_t source, addr64_t sink, unsigned int size, int which)
711 {
712 if ((which & (cppvPsrc | cppvPsnk)) == 0) { /* Make sure that only one is virtual */
713 panic("%s: no more than 1 parameter may be virtual", __func__);
714 }
715
716 kern_return_t res = bcopy_phys_internal(source, sink, size, which);
717
718 #ifndef __ARM_COHERENT_IO__
719 if (which & cppvFsrc) {
720 flush_dcache64(source, size, ((which & cppvPsrc) == cppvPsrc));
721 }
722
723 if (which & cppvFsnk) {
724 flush_dcache64(sink, size, ((which & cppvPsnk) == cppvPsnk));
725 }
726 #endif
727
728 return res;
729 }
730
731 int
clr_be_bit(void)732 clr_be_bit(void)
733 {
734 panic("clr_be_bit");
735 return 0;
736 }
737
738 boolean_t
ml_probe_read(__unused vm_offset_t paddr,__unused unsigned int * val)739 ml_probe_read(
740 __unused vm_offset_t paddr,
741 __unused unsigned int *val)
742 {
743 panic("ml_probe_read() unimplemented");
744 return 1;
745 }
746
747 boolean_t
ml_probe_read_64(__unused addr64_t paddr,__unused unsigned int * val)748 ml_probe_read_64(
749 __unused addr64_t paddr,
750 __unused unsigned int *val)
751 {
752 panic("ml_probe_read_64() unimplemented");
753 return 1;
754 }
755
756
757 void
ml_thread_policy(__unused thread_t thread,__unused unsigned policy_id,__unused unsigned policy_info)758 ml_thread_policy(
759 __unused thread_t thread,
760 __unused unsigned policy_id,
761 __unused unsigned policy_info)
762 {
763 // <rdar://problem/7141284>: Reduce print noise
764 // kprintf("ml_thread_policy() unimplemented\n");
765 }
766
767 __dead2
768 void
panic_unimplemented(void)769 panic_unimplemented(void)
770 {
771 panic("Not yet implemented.");
772 }
773
774 /* ARM64_TODO <rdar://problem/9198953> */
775 void abort(void) __dead2;
776
777 void
abort(void)778 abort(void)
779 {
780 panic("Abort.");
781 }
782
783
784 #if !MACH_KDP
785 void
kdp_register_callout(kdp_callout_fn_t fn,void * arg)786 kdp_register_callout(kdp_callout_fn_t fn, void *arg)
787 {
788 #pragma unused(fn,arg)
789 }
790 #endif
791
792 /*
793 * Get a quick virtual mapping of a physical page and run a callback on that
794 * page's virtual address.
795 *
796 * @param dst64 Physical address to access (doesn't need to be page-aligned).
797 * @param bytes Number of bytes to be accessed. This cannot cross page boundaries.
798 * @param func Callback function to call with the page's virtual address.
799 * @param arg Argument passed directly to `func`.
800 *
801 * @return The return value from `func`.
802 */
803 int
apply_func_phys(addr64_t dst64,vm_size_t bytes,int (* func)(void * buffer,vm_size_t bytes,void * arg),void * arg)804 apply_func_phys(
805 addr64_t dst64,
806 vm_size_t bytes,
807 int (*func)(void * buffer, vm_size_t bytes, void * arg),
808 void * arg)
809 {
810 /* The physical aperture is only guaranteed to work with kernel-managed addresses. */
811 if (!pmap_valid_address(dst64)) {
812 panic("%s address error: passed in address (%#llx) not a kernel managed address",
813 __FUNCTION__, dst64);
814 }
815
816 /* Ensure we stay within a single page */
817 if (((((uint32_t)dst64 & (ARM_PGBYTES - 1)) + bytes) > ARM_PGBYTES)) {
818 panic("%s alignment error: tried accessing addresses spanning more than one page %#llx %#lx",
819 __FUNCTION__, dst64, bytes);
820 }
821
822 return func((void*)phystokv(dst64), bytes, arg);
823 }
824