1 /*
2 * Copyright (c) 2003-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57
58 #include <mach/i386/vm_param.h>
59
60 #include <string.h>
61 #include <stdint.h>
62 #include <mach/vm_param.h>
63 #include <mach/vm_prot.h>
64 #include <mach/machine.h>
65 #include <mach/time_value.h>
66 #include <kern/spl.h>
67 #include <kern/assert.h>
68 #include <kern/debug.h>
69 #include <kern/misc_protos.h>
70 #include <kern/startup.h>
71 #include <kern/clock.h>
72 #include <kern/pms.h>
73 #include <kern/cpu_data.h>
74 #include <kern/processor.h>
75 #include <kern/monotonic.h>
76 #include <sys/kdebug.h>
77 #include <console/serial_protos.h>
78 #include <vm/vm_page.h>
79 #include <vm/pmap.h>
80 #include <vm/vm_kern.h>
81 #include <machine/pal_routines.h>
82 #include <i386/fpu.h>
83 #include <i386/pmap.h>
84 #include <i386/misc_protos.h>
85 #include <i386/cpu_threads.h>
86 #include <i386/cpuid.h>
87 #include <i386/lapic.h>
88 #include <i386/mp.h>
89 #include <i386/mp_desc.h>
90 #if CONFIG_MTRR
91 #include <i386/mtrr.h>
92 #endif
93 #include <i386/machine_routines.h>
94 #if CONFIG_MCA
95 #include <i386/machine_check.h>
96 #endif
97 #include <i386/ucode.h>
98 #include <i386/postcode.h>
99 #include <i386/Diagnostics.h>
100 #include <i386/pmCPU.h>
101 #include <i386/tsc.h>
102 #include <i386/acpi.h>
103 #if DEBUG
104 #include <machine/pal_routines.h>
105 #endif
106 extern void xcpm_bootstrap(void);
107 #if DEVELOPMENT || DEBUG
108 #include <i386/trap_internal.h>
109 #endif
110
111 #if KPERF
112 #include <kperf/kptimer.h>
113 #endif /* KPERF */
114
115 #include <san/kasan.h>
116
117 #if DEBUG || DEVELOPMENT
118 #define DBG(x, ...) kprintf(x, ##__VA_ARGS__)
119 #define dyldLogFunc(x, ...) kprintf(x, ##__VA_ARGS__)
120 #else
121 #define DBG(x ...)
122 #endif
123
124 #include <libkern/kernel_mach_header.h>
125 #include <mach/dyld_kernel_fixups.h>
126
127
128 int debug_task;
129
130 int early_boot = 1;
131
132 bool serial_console_enabled = false;
133
134 static boot_args *kernelBootArgs;
135
136 extern int disableConsoleOutput;
137 extern const char version[];
138 extern const char version_variant[];
139 extern int nx_enabled;
140
141 /*
142 * Set initial values so that ml_phys_* routines can use the booter's ID mapping
143 * to touch physical space before the kernel's physical aperture exists.
144 */
145 uint64_t physmap_base = 0;
146 uint64_t physmap_max = 4 * GB;
147
148 pd_entry_t *KPTphys;
149 pd_entry_t *IdlePTD;
150 pdpt_entry_t *IdlePDPT;
151 pml4_entry_t *IdlePML4;
152
153 int kernPhysPML4Index;
154 int kernPhysPML4EntryCount;
155
156 /*
157 * These are 4K mapping page table pages from KPTphys[] that we wound
158 * up not using. They get ml_static_mfree()'d once the VM is initialized.
159 */
160 ppnum_t released_PT_ppn = 0;
161 uint32_t released_PT_cnt = 0;
162
163 #if DEVELOPMENT || DEBUG
164 int panic_on_cacheline_mismatch = -1;
165 char panic_on_trap_procname[64];
166 uint32_t panic_on_trap_mask;
167 #endif
168 lbr_modes_t last_branch_enabled_modes;
169 int insn_copyin_count;
170 #if DEVELOPMENT || DEBUG
171 #define DEFAULT_INSN_COPYIN_COUNT x86_INSTRUCTION_STATE_MAX_INSN_BYTES
172 #else
173 #define DEFAULT_INSN_COPYIN_COUNT 192
174 #endif
175
176 char *physfree;
177 void idt64_remap(void);
178
179 /*
180 * Note: ALLOCPAGES() can only be used safely within Idle_PTs_init()
181 * due to the mutation of physfree.
182 */
183 static void *
ALLOCPAGES(int npages)184 ALLOCPAGES(int npages)
185 {
186 uintptr_t tmp = (uintptr_t)physfree;
187 bzero(physfree, npages * PAGE_SIZE);
188 physfree += npages * PAGE_SIZE;
189 tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK;
190 return (void *)tmp;
191 }
192
193 static void
fillkpt(pt_entry_t * base,int prot,uintptr_t src,int index,int count)194 fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count)
195 {
196 int i;
197 for (i = 0; i < count; i++) {
198 base[index] = src | prot | INTEL_PTE_VALID;
199 src += PAGE_SIZE;
200 index++;
201 }
202 }
203
204 extern pmap_paddr_t first_avail;
205
206 int break_kprintf = 0;
207
208 uint64_t
x86_64_pre_sleep(void)209 x86_64_pre_sleep(void)
210 {
211 IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX];
212 uint64_t oldcr3 = get_cr3_raw();
213 set_cr3_raw((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4));
214 return oldcr3;
215 }
216
217 void
x86_64_post_sleep(uint64_t new_cr3)218 x86_64_post_sleep(uint64_t new_cr3)
219 {
220 IdlePML4[0] = 0;
221 set_cr3_raw((uint32_t) new_cr3);
222 }
223
224
225
226
227 // Set up the physical mapping - NPHYSMAP GB of memory mapped at a high address
228 // NPHYSMAP is determined by the maximum supported RAM size plus 4GB to account
229 // the PCI hole (which is less 4GB but not more).
230
231 static int
physmap_init_L2(uint64_t * physStart,pt_entry_t ** l2ptep)232 physmap_init_L2(uint64_t *physStart, pt_entry_t **l2ptep)
233 {
234 unsigned i;
235 pt_entry_t *physmapL2 = ALLOCPAGES(1);
236
237 if (physmapL2 == NULL) {
238 DBG("physmap_init_L2 page alloc failed when initting L2 for physAddr 0x%llx.\n", *physStart);
239 *l2ptep = NULL;
240 return -1;
241 }
242
243 for (i = 0; i < NPDPG; i++) {
244 physmapL2[i] = *physStart
245 | INTEL_PTE_PS
246 | INTEL_PTE_VALID
247 | INTEL_PTE_NX
248 | INTEL_PTE_WRITE;
249
250 *physStart += NBPD;
251 }
252 *l2ptep = physmapL2;
253 return 0;
254 }
255
256 static int
physmap_init_L3(int startIndex,uint64_t highest_phys,uint64_t * physStart,pt_entry_t ** l3ptep)257 physmap_init_L3(int startIndex, uint64_t highest_phys, uint64_t *physStart, pt_entry_t **l3ptep)
258 {
259 unsigned i;
260 int ret;
261 pt_entry_t *l2pte;
262 pt_entry_t *physmapL3 = ALLOCPAGES(1); /* ALLOCPAGES bzeroes the memory */
263
264 if (physmapL3 == NULL) {
265 DBG("physmap_init_L3 page alloc failed when initting L3 for physAddr 0x%llx.\n", *physStart);
266 *l3ptep = NULL;
267 return -1;
268 }
269
270 for (i = startIndex; i < NPDPTPG && *physStart < highest_phys; i++) {
271 if ((ret = physmap_init_L2(physStart, &l2pte)) < 0) {
272 return ret;
273 }
274
275 physmapL3[i] = ((uintptr_t)ID_MAP_VTOP(l2pte))
276 | INTEL_PTE_VALID
277 | INTEL_PTE_NX
278 | INTEL_PTE_WRITE;
279 }
280
281 *l3ptep = physmapL3;
282
283 return 0;
284 }
285
286 static void
physmap_init(uint8_t phys_random_L3,uint64_t * new_physmap_base,uint64_t * new_physmap_max)287 physmap_init(uint8_t phys_random_L3, uint64_t *new_physmap_base, uint64_t *new_physmap_max)
288 {
289 pt_entry_t *l3pte;
290 int pml4_index, i;
291 int L3_start_index;
292 uint64_t physAddr = 0;
293 uint64_t highest_physaddr;
294 unsigned pdpte_count;
295
296 #if DEVELOPMENT || DEBUG
297 if (kernelBootArgs->PhysicalMemorySize > K64_MAXMEM) {
298 panic("Installed physical memory exceeds configured maximum.");
299 }
300 #endif
301
302 /*
303 * Add 4GB to the loader-provided physical memory size to account for MMIO space
304 * XXX in a perfect world, we'd scan PCI buses and count the max memory requested in BARs by
305 * XXX all enumerated device, then add more for hot-pluggable devices.
306 */
307 highest_physaddr = kernelBootArgs->PhysicalMemorySize + 4 * GB;
308
309 /*
310 * Calculate the number of PML4 entries we'll need. The total number of entries is
311 * pdpte_count = (((highest_physaddr) >> PDPT_SHIFT) + entropy_value +
312 * ((highest_physaddr & PDPT_MASK) == 0 ? 0 : 1))
313 * pml4e_count = pdpte_count >> (PML4_SHIFT - PDPT_SHIFT)
314 */
315 assert(highest_physaddr < (UINT64_MAX - PDPTMASK));
316 pdpte_count = (unsigned) (((highest_physaddr + PDPTMASK) >> PDPTSHIFT) + phys_random_L3);
317 kernPhysPML4EntryCount = (pdpte_count + ((1U << (PML4SHIFT - PDPTSHIFT)) - 1)) >> (PML4SHIFT - PDPTSHIFT);
318 if (kernPhysPML4EntryCount == 0) {
319 kernPhysPML4EntryCount = 1;
320 }
321 if (kernPhysPML4EntryCount > KERNEL_PHYSMAP_PML4_COUNT_MAX) {
322 #if DEVELOPMENT || DEBUG
323 panic("physmap too large");
324 #else
325 kprintf("[pmap] Limiting physmap to %d PML4s (was %d)\n", KERNEL_PHYSMAP_PML4_COUNT_MAX,
326 kernPhysPML4EntryCount);
327 kernPhysPML4EntryCount = KERNEL_PHYSMAP_PML4_COUNT_MAX;
328 #endif
329 }
330
331 kernPhysPML4Index = KERNEL_KEXTS_INDEX - kernPhysPML4EntryCount; /* utb: KERNEL_PHYSMAP_PML4_INDEX */
332
333 /*
334 * XXX: Make sure that the addresses returned for physmapL3 and physmapL2 plus their extents
335 * are in the system-available memory range
336 */
337
338
339 /* We assume NX support. Mark all levels of the PHYSMAP NX
340 * to avoid granting executability via a single bit flip.
341 */
342 #if DEVELOPMENT || DEBUG
343 uint32_t reg[4];
344 do_cpuid(0x80000000, reg);
345 if (reg[eax] >= 0x80000001) {
346 do_cpuid(0x80000001, reg);
347 assert(reg[edx] & CPUID_EXTFEATURE_XD);
348 }
349 #endif /* DEVELOPMENT || DEBUG */
350
351 L3_start_index = phys_random_L3;
352
353 for (pml4_index = kernPhysPML4Index;
354 pml4_index < (kernPhysPML4Index + kernPhysPML4EntryCount) && physAddr < highest_physaddr;
355 pml4_index++) {
356 if (physmap_init_L3(L3_start_index, highest_physaddr, &physAddr, &l3pte) < 0) {
357 panic("Physmap page table initialization failed");
358 /* NOTREACHED */
359 }
360
361 L3_start_index = 0;
362
363 IdlePML4[pml4_index] = ((uintptr_t)ID_MAP_VTOP(l3pte))
364 | INTEL_PTE_VALID
365 | INTEL_PTE_NX
366 | INTEL_PTE_WRITE;
367 }
368
369 *new_physmap_base = KVADDR(kernPhysPML4Index, phys_random_L3, 0, 0);
370 /*
371 * physAddr contains the last-mapped physical address, so that's what we
372 * add to physmap_base to derive the ending VA for the physmap.
373 */
374 *new_physmap_max = *new_physmap_base + physAddr;
375
376 DBG("Physical address map base: 0x%qx\n", *new_physmap_base);
377 for (i = kernPhysPML4Index; i < (kernPhysPML4Index + kernPhysPML4EntryCount); i++) {
378 DBG("Physical map idlepml4[%d]: 0x%llx\n", i, IdlePML4[i]);
379 }
380 }
381
382 void doublemap_init(uint8_t);
383
384 static void
Idle_PTs_init(void)385 Idle_PTs_init(void)
386 {
387 uint64_t rand64;
388 uint64_t new_physmap_base, new_physmap_max;
389
390 /* Allocate the "idle" kernel page tables: */
391 KPTphys = ALLOCPAGES(NKPT); /* level 1 */
392 IdlePTD = ALLOCPAGES(NPGPTD); /* level 2 */
393 IdlePDPT = ALLOCPAGES(1); /* level 3 */
394 IdlePML4 = ALLOCPAGES(1); /* level 4 */
395
396 // Fill the lowest level with everything up to physfree
397 fillkpt(KPTphys,
398 INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
399
400 /* IdlePTD */
401 fillkpt(IdlePTD,
402 INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
403
404 // IdlePDPT entries
405 fillkpt(IdlePDPT,
406 INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
407
408 // IdlePML4 single entry for kernel space.
409 fillkpt(IdlePML4 + KERNEL_PML4_INDEX,
410 INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1);
411
412 postcode(VSTART_PHYSMAP_INIT);
413
414 /*
415 * early_random() cannot be called more than one time before the cpu's
416 * gsbase is initialized, so use the full 64-bit value to extract the
417 * two 8-bit entropy values needed for address randomization.
418 */
419 rand64 = early_random();
420 physmap_init(rand64 & 0xFF, &new_physmap_base, &new_physmap_max);
421 doublemap_init((rand64 >> 8) & 0xFF);
422 idt64_remap();
423
424 postcode(VSTART_SET_CR3);
425
426 /*
427 * Switch to the page tables. We set physmap_base and physmap_max just
428 * before switching to the new page tables to avoid someone calling
429 * kprintf() or otherwise using physical memory in between.
430 * This is needed because kprintf() writes to physical memory using
431 * ml_phys_read_data and PHYSMAP_PTOV, which requires physmap_base to be
432 * set correctly.
433 */
434 physmap_base = new_physmap_base;
435 physmap_max = new_physmap_max;
436 set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
437 }
438
439 /*
440 * Release any still unused, preallocated boot kernel page tables.
441 * start..end is the VA range currently unused.
442 */
443 void
Idle_PTs_release(vm_offset_t start,vm_offset_t end)444 Idle_PTs_release(vm_offset_t start, vm_offset_t end)
445 {
446 uint32_t i;
447 uint32_t index_start;
448 uint32_t index_limit;
449 ppnum_t pn_first;
450 ppnum_t pn;
451 uint32_t cnt;
452
453 /*
454 * Align start to the next large page boundary
455 */
456 start = ((start + I386_LPGMASK) & ~I386_LPGMASK);
457
458 /*
459 * convert start into an index in KPTphys[]
460 */
461 index_start = (uint32_t)((start - KERNEL_BASE) >> PAGE_SHIFT);
462
463 /*
464 * Find the ending index in KPTphys[]
465 */
466 index_limit = (uint32_t)((end - KERNEL_BASE) >> PAGE_SHIFT);
467
468 if (index_limit > NKPT * PTE_PER_PAGE) {
469 index_limit = NKPT * PTE_PER_PAGE;
470 }
471
472 /*
473 * Make sure all the 4K page tables are empty.
474 * If not, panic a development/debug kernel.
475 * On a production kernel, since this would stop us from booting,
476 * just abort the operation.
477 */
478 for (i = index_start; i < index_limit; ++i) {
479 assert(KPTphys[i] == 0);
480 if (KPTphys[i] != 0) {
481 return;
482 }
483 }
484
485 /*
486 * Now figure out the indices into the 2nd level page tables, IdlePTD[].
487 */
488 index_start >>= PTPGSHIFT;
489 index_limit >>= PTPGSHIFT;
490 if (index_limit > NPGPTD * PTE_PER_PAGE) {
491 index_limit = NPGPTD * PTE_PER_PAGE;
492 }
493
494 if (index_limit <= index_start) {
495 return;
496 }
497
498
499 /*
500 * Now check the pages referenced from Level 2 tables.
501 * They should be contiguous, assert fail if not on development/debug.
502 * In production, just fail the removal to allow the system to boot.
503 */
504 pn_first = 0;
505 cnt = 0;
506 for (i = index_start; i < index_limit; ++i) {
507 assert(IdlePTD[i] != 0);
508 if (IdlePTD[i] == 0) {
509 return;
510 }
511
512 pn = (ppnum_t)((PG_FRAME & IdlePTD[i]) >> PTSHIFT);
513 if (cnt == 0) {
514 pn_first = pn;
515 } else {
516 assert(pn == pn_first + cnt);
517 if (pn != pn_first + cnt) {
518 return;
519 }
520 }
521 ++cnt;
522 }
523
524 /*
525 * Good to go, clear the level 2 entries and invalidate the TLB
526 */
527 for (i = index_start; i < index_limit; ++i) {
528 IdlePTD[i] = 0;
529 }
530 set_cr3_raw(get_cr3_raw());
531
532 /*
533 * Remember these PFNs to be released later in pmap_lowmem_finalize()
534 */
535 released_PT_ppn = pn_first;
536 released_PT_cnt = cnt;
537 #if DEVELOPMENT || DEBUG
538 printf("Idle_PTs_release %d pages from PFN 0x%x\n", released_PT_cnt, released_PT_ppn);
539 #endif
540 }
541
542 extern void vstart_trap_handler;
543
544 #define BOOT_TRAP_VECTOR(t) \
545 [t] = { \
546 (uintptr_t) &vstart_trap_handler, \
547 KERNEL64_CS, \
548 0, \
549 ACC_P|ACC_PL_K|ACC_INTR_GATE, \
550 0 \
551 },
552
553 /* Recursive macro to iterate 0..31 */
554 #define L0(x, n) x(n)
555 #define L1(x, n) L0(x,n-1) L0(x,n)
556 #define L2(x, n) L1(x,n-2) L1(x,n)
557 #define L3(x, n) L2(x,n-4) L2(x,n)
558 #define L4(x, n) L3(x,n-8) L3(x,n)
559 #define L5(x, n) L4(x,n-16) L4(x,n)
560 #define FOR_0_TO_31(x) L5(x,31)
561
562 /*
563 * Bootstrap IDT. Active only during early startup.
564 * Only the trap vectors are defined since interrupts are masked.
565 * All traps point to a common handler.
566 */
567 struct fake_descriptor64 master_boot_idt64[IDTSZ]
568 __attribute__((section("__HIB,__desc")))
569 __attribute__((aligned(PAGE_SIZE))) = {
570 FOR_0_TO_31(BOOT_TRAP_VECTOR)
571 };
572
573 static void
vstart_idt_init(boolean_t master)574 vstart_idt_init(boolean_t master)
575 {
576 x86_64_desc_register_t vstart_idt = {
577 sizeof(master_boot_idt64),
578 master_boot_idt64
579 };
580
581 if (master) {
582 fix_desc64(master_boot_idt64, 32);
583 }
584 lidt((void *)&vstart_idt);
585 }
586
587 extern void *collection_base_pointers[KCNumKinds];
588
589 kern_return_t
i386_slide_individual_kext(kernel_mach_header_t * mh,uintptr_t slide)590 i386_slide_individual_kext(kernel_mach_header_t *mh, uintptr_t slide)
591 {
592 int ret = kernel_collection_slide(mh, (const void **) (void *)collection_base_pointers);
593 if (ret != 0) {
594 printf("Sliding pageable kc was stopped\n");
595 return KERN_FAILURE;
596 }
597
598 kernel_collection_adjust_fileset_entry_addrs(mh, slide);
599 return KERN_SUCCESS;
600 }
601
602 kern_return_t
i386_slide_kext_collection_mh_addrs(kernel_mach_header_t * mh,uintptr_t slide,bool adjust_mach_headers)603 i386_slide_kext_collection_mh_addrs(kernel_mach_header_t *mh, uintptr_t slide, bool adjust_mach_headers)
604 {
605 int ret = kernel_collection_slide(mh, (const void **) (void *)collection_base_pointers);
606 if (ret != KERN_SUCCESS) {
607 printf("Kernel Collection slide was stopped with value %d\n", ret);
608 return KERN_FAILURE;
609 }
610
611 kernel_collection_adjust_mh_addrs(mh, slide, adjust_mach_headers,
612 NULL, NULL, NULL, NULL, NULL, NULL, NULL);
613
614 return KERN_SUCCESS;
615 }
616
617 static void
i386_slide_and_rebase_image(uintptr_t kstart_addr)618 i386_slide_and_rebase_image(uintptr_t kstart_addr)
619 {
620 extern uintptr_t kc_highest_nonlinkedit_vmaddr;
621 kernel_mach_header_t *k_mh, *kc_mh = NULL;
622 kernel_segment_command_t *seg;
623 uintptr_t slide;
624
625 k_mh = &_mh_execute_header;
626 /*
627 * If we're not booting, an MH_FILESET, we don't need to slide
628 * anything because EFI has done that for us. When booting an
629 * MH_FILESET, EFI will slide the kernel proper, but not the kexts.
630 * Below, we infer the slide by comparing the slid address of the
631 * kernel's mach-o header and the unslid vmaddr of the first segment
632 * of the mach-o (which is assumed to always point to the mach-o
633 * header).
634 */
635 if (!kernel_mach_header_is_in_fileset(k_mh)) {
636 DBG("[MH] kcgen-style KC\n");
637 return;
638 }
639
640 /*
641 * The kernel is part of a MH_FILESET kernel collection: determine slide
642 * based on first segment's mach-o vmaddr.
643 */
644 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
645 assert(seg->cmd == LC_SEGMENT_KERNEL);
646 slide = (uintptr_t)k_mh - seg->vmaddr;
647 DBG("[MH] Sliding new-style KC: %llu\n", (unsigned long long)slide);
648
649 /*
650 * The kernel collection mach-o header should be the start address
651 * passed to us by EFI.
652 */
653 kc_mh = (kernel_mach_header_t *)(kstart_addr);
654 assert(kc_mh->filetype == MH_FILESET);
655
656 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
657
658 /*
659 * rebase/slide all the kexts in the collection
660 * (EFI should have already rebased the kernel)
661 */
662 kernel_collection_slide(kc_mh, (const void **) (void *)collection_base_pointers);
663
664
665 /*
666 * Now adjust the vmaddr fields of all mach-o headers
667 * and symbols in this MH_FILESET
668 */
669 kernel_collection_adjust_mh_addrs(kc_mh, slide, false,
670 NULL, NULL, NULL, NULL, NULL, NULL, &kc_highest_nonlinkedit_vmaddr);
671 }
672
673 /*
674 * vstart() is called in the natural mode (64bit for K64, 32 for K32)
675 * on a set of bootstrap pagetables which use large, 2MB pages to map
676 * all of physical memory in both. See idle_pt.c for details.
677 *
678 * In K64 this identity mapping is mirrored the top and bottom 512GB
679 * slots of PML4.
680 *
681 * The bootstrap processor called with argument boot_args_start pointing to
682 * the boot-args block. The kernel's (4K page) page tables are allocated and
683 * initialized before switching to these.
684 *
685 * Non-bootstrap processors are called with argument boot_args_start NULL.
686 * These processors switch immediately to the existing kernel page tables.
687 */
688 __attribute__((noreturn))
689 void
vstart(vm_offset_t boot_args_start)690 vstart(vm_offset_t boot_args_start)
691 {
692 boolean_t is_boot_cpu = !(boot_args_start == 0);
693 int cpu = 0;
694 uint32_t lphysfree;
695 #if DEBUG
696 uint64_t gsbase;
697 #endif
698
699
700 postcode(VSTART_ENTRY);
701
702 /*
703 * Set-up temporary trap handlers during page-table set-up.
704 */
705
706 if (is_boot_cpu) {
707 vstart_idt_init(TRUE);
708 postcode(VSTART_IDT_INIT);
709
710 /*
711 * Ensure that any %gs-relative access results in an immediate fault
712 * until gsbase is properly initialized below
713 */
714 wrmsr64(MSR_IA32_GS_BASE, EARLY_GSBASE_MAGIC);
715
716 /*
717 * Get startup parameters.
718 */
719 kernelBootArgs = (boot_args *)boot_args_start;
720 lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
721 physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1));
722
723 pal_serial_init();
724
725 DBG("revision 0x%x\n", kernelBootArgs->Revision);
726 DBG("version 0x%x\n", kernelBootArgs->Version);
727 DBG("command line %s\n", kernelBootArgs->CommandLine);
728 DBG("memory map 0x%x\n", kernelBootArgs->MemoryMap);
729 DBG("memory map sz 0x%x\n", kernelBootArgs->MemoryMapSize);
730 DBG("kaddr 0x%x\n", kernelBootArgs->kaddr);
731 DBG("ksize 0x%x\n", kernelBootArgs->ksize);
732 DBG("physfree %p\n", physfree);
733 DBG("bootargs: %p, &ksize: %p &kaddr: %p\n",
734 kernelBootArgs,
735 &kernelBootArgs->ksize,
736 &kernelBootArgs->kaddr);
737 DBG("SMBIOS mem sz 0x%llx\n", kernelBootArgs->PhysicalMemorySize);
738 DBG("KC_hdrs_vaddr %p\n", (void *)kernelBootArgs->KC_hdrs_vaddr);
739
740 if (kernelBootArgs->Version >= 2 && kernelBootArgs->Revision >= 1 &&
741 kernelBootArgs->KC_hdrs_vaddr != 0) {
742 /*
743 * slide the header addresses in all mach-o segments and sections, and
744 * perform any new-style chained-fixup sliding for kexts, as necessary.
745 * Note that efiboot has already loaded the kernel and all LC_SEGMENT_64s
746 * that correspond to the kexts present in the primary KC, into slid addresses.
747 */
748 i386_slide_and_rebase_image((uintptr_t)ml_static_ptovirt(kernelBootArgs->KC_hdrs_vaddr));
749 }
750
751 /*
752 * Setup boot args given the physical start address.
753 * Note: PE_init_platform needs to be called before Idle_PTs_init
754 * because access to the DeviceTree is required to read the
755 * random seed before generating a random physical map slide.
756 */
757 kernelBootArgs = (boot_args *)
758 ml_static_ptovirt(boot_args_start);
759 DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
760 (unsigned long)boot_args_start, kernelBootArgs);
761
762 #if KASAN
763 kasan_reserve_memory(kernelBootArgs);
764 #endif
765
766 PE_init_platform(FALSE, kernelBootArgs);
767 postcode(PE_INIT_PLATFORM_D);
768
769 Idle_PTs_init();
770 postcode(VSTART_IDLE_PTS_INIT);
771
772 #if KASAN
773 /* Init kasan and map whatever was stolen from physfree */
774 kasan_init();
775 kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree));
776 #endif
777
778 #if CONFIG_CPU_COUNTERS
779 mt_early_init();
780 #endif /* CONFIG_CPU_COUNTERS */
781
782 first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
783
784 cpu_data_alloc(TRUE);
785
786 cpu_desc_init(cpu_datap(0));
787 postcode(VSTART_CPU_DESC_INIT);
788 cpu_desc_load(cpu_datap(0));
789
790 postcode(VSTART_CPU_MODE_INIT);
791 cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be
792 * invoked on the APs
793 * via i386_init_slave()
794 */
795 } else {
796 /* Slave CPUs should use the basic IDT until i386_init_slave() */
797 vstart_idt_init(FALSE);
798
799 /* Switch to kernel's page tables (from the Boot PTs) */
800 set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
801
802 /* Find our logical cpu number */
803 cpu = lapic_to_cpu[lapic_safe_apicid()];
804 #if DEBUG
805 gsbase = rdmsr64(MSR_IA32_GS_BASE);
806 #endif
807 cpu_desc_load(cpu_datap(cpu));
808 #if DEBUG
809 DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, (unsigned long long)gsbase);
810 #endif
811
812 /*
813 * Before we can discover our local APIC ID, we need to potentially
814 * initialize X2APIC, if it's enabled and firmware started us with
815 * the APIC in legacy mode.
816 */
817 lapic_init_slave();
818 }
819
820 early_boot = 0;
821 postcode(VSTART_EXIT);
822 x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
823 : (uintptr_t) i386_init_slave,
824 cpu_datap(cpu)->cpu_int_stack_top);
825 }
826
827 void
pstate_trace(void)828 pstate_trace(void)
829 {
830 }
831
832 /*
833 * Cpu initialization. Running virtual, but without MACH VM
834 * set up.
835 */
836 void
i386_init(void)837 i386_init(void)
838 {
839 unsigned int maxmem;
840 uint64_t maxmemtouse;
841 unsigned int cpus = 0;
842 boolean_t fidn;
843 boolean_t IA32e = TRUE;
844
845 postcode(I386_INIT_ENTRY);
846
847 pal_i386_init();
848 tsc_init();
849 rtclock_early_init(); /* mach_absolute_time() now functional */
850
851 kernel_debug_string_early("i386_init");
852 pstate_trace();
853
854 #if CONFIG_MCA
855 /* Initialize machine-check handling */
856 mca_cpu_init();
857 #endif
858
859 master_cpu = 0;
860
861 kernel_debug_string_early("kernel_startup_bootstrap");
862 kernel_startup_bootstrap();
863
864 /*
865 * Initialize the timer callout world
866 */
867 timer_call_init();
868
869 cpu_init();
870
871 postcode(CPU_INIT_D);
872
873 /* setup debugging output if one has been chosen */
874 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
875 kprintf("kprintf initialized\n");
876
877 if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof(dgWork.dgFlags))) {
878 dgWork.dgFlags = 0;
879 }
880
881 if (PE_parse_boot_argn("insn_capcnt", &insn_copyin_count, sizeof(insn_copyin_count))) {
882 /*
883 * Enforce max and min values (allowing 0 to disable copying completely)
884 * for the instruction copyin count
885 */
886 if (insn_copyin_count > x86_INSTRUCTION_STATE_MAX_INSN_BYTES ||
887 (insn_copyin_count != 0 && insn_copyin_count < 64)) {
888 insn_copyin_count = DEFAULT_INSN_COPYIN_COUNT;
889 }
890 } else {
891 insn_copyin_count = DEFAULT_INSN_COPYIN_COUNT;
892 }
893
894 #if DEVELOPMENT || DEBUG
895 if (!PE_parse_boot_argn("panic_clmismatch", &panic_on_cacheline_mismatch,
896 sizeof(panic_on_cacheline_mismatch))) {
897 panic_on_cacheline_mismatch = 0;
898 }
899
900 if (!PE_parse_boot_argn("panic_on_trap_procname", &panic_on_trap_procname[0],
901 sizeof(panic_on_trap_procname))) {
902 panic_on_trap_procname[0] = 0;
903 }
904
905 if (!PE_parse_boot_argn("panic_on_trap_mask", &panic_on_trap_mask,
906 sizeof(panic_on_trap_mask))) {
907 if (panic_on_trap_procname[0] != 0) {
908 panic_on_trap_mask = DEFAULT_PANIC_ON_TRAP_MASK;
909 } else {
910 panic_on_trap_mask = 0;
911 }
912 }
913 #endif
914 /* But allow that to be overridden via boot-arg: */
915 if (!PE_parse_boot_argn("lbr_support", &last_branch_enabled_modes,
916 sizeof(last_branch_enabled_modes))) {
917 /* Disable LBR support by default due to its high context switch overhead */
918 last_branch_enabled_modes = LBR_ENABLED_NONE;
919 }
920
921 serialmode = 0;
922 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
923 /* We want a serial keyboard and/or console */
924 kprintf("Serial mode specified: %08X\n", serialmode);
925 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
926 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
927 enable_dklog_serial_output = (serialmode & SERIALMODE_DKLOG) != 0;
928 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
929 if (force_sync) {
930 serialmode |= SERIALMODE_SYNCDRAIN;
931 kprintf(
932 "WARNING: Forcing uart driver to output synchronously."
933 "printf()s/IOLogs will impact kernel performance.\n"
934 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
935 }
936 }
937 }
938 if (serialmode & SERIALMODE_OUTPUT) {
939 serial_console_enabled = true;
940 (void)switch_to_serial_console();
941 disableConsoleOutput = FALSE; /* Allow printfs to happen */
942 }
943
944 /* setup console output */
945 kernel_debug_string_early("PE_init_printf");
946 PE_init_printf(FALSE);
947
948 kprintf("version_variant = %s\n", version_variant);
949 kprintf("version = %s\n", version);
950
951 if (!PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
952 maxmemtouse = 0;
953 } else {
954 maxmemtouse = ((uint64_t)maxmem) * MB;
955 }
956
957 max_cpus_from_firmware = acpi_count_enabled_logical_processors();
958
959 if (PE_parse_boot_argn("cpus", &cpus, sizeof(cpus))) {
960 if ((0 < cpus) && (cpus < max_ncpus)) {
961 max_ncpus = cpus;
962 }
963 }
964
965 /*
966 * debug support for > 4G systems
967 */
968 PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof(vm_himemory_mode));
969 if (!vm_himemory_mode) {
970 kprintf("himemory_mode disabled\n");
971 }
972
973 if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof(fidn))) {
974 force_immediate_debugger_NMI = FALSE;
975 } else {
976 force_immediate_debugger_NMI = fidn;
977 }
978
979 #if DEBUG
980 nanoseconds_to_absolutetime(URGENCY_NOTIFICATION_ASSERT_NS, &urgency_notification_assert_abstime_threshold);
981 #endif
982 PE_parse_boot_argn("urgency_notification_abstime",
983 &urgency_notification_assert_abstime_threshold,
984 sizeof(urgency_notification_assert_abstime_threshold));
985
986 if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD)) {
987 nx_enabled = 0;
988 }
989
990 /*
991 * VM initialization, after this we're using page tables...
992 * Thn maximum number of cpus must be set beforehand.
993 */
994 kernel_debug_string_early("i386_vm_init");
995 i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
996
997 /* create the console for verbose or pretty mode */
998 /* Note: doing this prior to tsc_init() allows for graceful panic! */
999 PE_init_platform(TRUE, kernelBootArgs);
1000 PE_create_console();
1001
1002 /* set %gs early so that power management can use locks */
1003 thread_t thread = thread_bootstrap();
1004 machine_set_current_thread(thread);
1005
1006 kernel_debug_string_early("power_management_init");
1007 power_management_init();
1008 xcpm_bootstrap();
1009
1010 #if CONFIG_CPU_COUNTERS
1011 mt_cpu_up(cpu_datap(0));
1012 #endif /* CONFIG_CPU_COUNTERS */
1013
1014 processor_bootstrap();
1015
1016 pstate_trace();
1017 kernel_debug_string_early("machine_startup");
1018 machine_startup();
1019 pstate_trace();
1020 }
1021
1022 static void __dead2
do_init_slave(boolean_t fast_restart)1023 do_init_slave(boolean_t fast_restart)
1024 {
1025 void *init_param = FULL_SLAVE_INIT;
1026
1027 postcode(I386_INIT_SLAVE);
1028
1029 if (!fast_restart) {
1030 /* Ensure that caching and write-through are enabled */
1031 set_cr0(get_cr0() & ~(CR0_NW | CR0_CD));
1032
1033 DBG("i386_init_slave() CPU%d: phys (%d) active.\n",
1034 get_cpu_number(), get_cpu_phys_number());
1035
1036 assert(!ml_get_interrupts_enabled());
1037
1038 cpu_syscall_init(current_cpu_datap());
1039 pmap_cpu_init();
1040
1041 #if CONFIG_MCA
1042 mca_cpu_init();
1043 #endif
1044
1045 LAPIC_INIT();
1046 /*
1047 * Note that the true argument here does not necessarily mean we're
1048 * here from a resume (this code path is also executed on boot).
1049 * The implementation of lapic_configure checks to see if the
1050 * state variable has been initialized, as it would be before
1051 * sleep. If it has not been, it's construed as an indicator of
1052 * first boot.
1053 */
1054 lapic_configure(true);
1055 LAPIC_DUMP();
1056 LAPIC_CPU_MAP_DUMP();
1057
1058 init_fpu();
1059
1060 #if CONFIG_MTRR
1061 mtrr_update_cpu();
1062 #endif
1063 /* update CPU microcode and apply CPU workarounds */
1064 ucode_update_wake_and_apply_cpu_was();
1065
1066 /* Enable LBRs on non-boot CPUs */
1067 i386_lbr_init(cpuid_info(), false);
1068 } else {
1069 init_param = FAST_SLAVE_INIT;
1070 }
1071
1072 #if CONFIG_VMX
1073 /* resume VT operation */
1074 vmx_resume(FALSE);
1075 #endif
1076
1077 #if CONFIG_MTRR
1078 if (!fast_restart) {
1079 pat_init();
1080 }
1081 #endif
1082
1083 cpu_thread_init(); /* not strictly necessary */
1084
1085 cpu_init(); /* Sets cpu_running which starter cpu waits for */
1086
1087
1088 #if CONFIG_CPU_COUNTERS
1089 mt_cpu_up(current_cpu_datap());
1090 #endif /* CONFIG_CPU_COUNTERS */
1091
1092 #if KPERF
1093 /*
1094 * We can only directly invoke kptimer_curcpu_up() when there is already an
1095 * active thread (that is, that this CPU has already been started at some point),
1096 * otherwise the ktrace calls within the kptimer operations will try to deref
1097 * the current thread and will instead cause a system reset.
1098 * If this is the first time the CPU is being started, we don't need to call
1099 * kptimer_curcpu_up().
1100 */
1101 if (current_processor()->active_thread != THREAD_NULL) {
1102 kptimer_curcpu_up();
1103 }
1104 #endif /* KPERF */
1105
1106 slave_main(init_param);
1107
1108 panic("do_init_slave() returned from slave_main()");
1109 }
1110
1111 /*
1112 * i386_init_slave() is called from pstart.
1113 * We're in the cpu's interrupt stack with interrupts disabled.
1114 * At this point we are in legacy mode. We need to switch on IA32e
1115 * if the mode is set to 64-bits.
1116 */
1117 void
i386_init_slave(void)1118 i386_init_slave(void)
1119 {
1120 do_init_slave(FALSE);
1121 }
1122
1123 /*
1124 * i386_init_slave_fast() is called from pmCPUHalt.
1125 * We're running on the idle thread and need to fix up
1126 * some accounting and get it so that the scheduler sees this
1127 * CPU again.
1128 */
1129 void
i386_init_slave_fast(void)1130 i386_init_slave_fast(void)
1131 {
1132 do_init_slave(TRUE);
1133 }
1134
1135
1136 /* TODO: Evaluate global PTEs for the double-mapped translations */
1137
1138 uint64_t dblmap_base, dblmap_max;
1139 kernel_segment_command_t *hdescseg;
1140
1141 pt_entry_t *dblmapL3;
1142 unsigned int dblallocs;
1143 uint64_t dblmap_dist;
1144 extern uint64_t idt64_hndl_table0[];
1145
1146
1147 void
doublemap_init(uint8_t randL3)1148 doublemap_init(uint8_t randL3)
1149 {
1150 dblmapL3 = ALLOCPAGES(1); // for 512 1GiB entries
1151 dblallocs++;
1152
1153 struct {
1154 pt_entry_t entries[PTE_PER_PAGE];
1155 } * dblmapL2 = ALLOCPAGES(1); // for 512 2MiB entries
1156 dblallocs++;
1157
1158 dblmapL3[randL3] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0]))
1159 | INTEL_PTE_VALID
1160 | INTEL_PTE_WRITE;
1161
1162 hdescseg = getsegbynamefromheader(&_mh_execute_header, "__HIB");
1163
1164 vm_offset_t hdescb = hdescseg->vmaddr;
1165 unsigned long hdescsz = hdescseg->vmsize;
1166 unsigned long hdescszr = round_page_64(hdescsz);
1167 vm_offset_t hdescc = hdescb, hdesce = hdescb + hdescszr;
1168
1169 kernel_section_t *thdescsect = getsectbynamefromheader(&_mh_execute_header, "__HIB", "__text");
1170 vm_offset_t thdescb = thdescsect->addr;
1171 unsigned long thdescsz = thdescsect->size;
1172 unsigned long thdescszr = round_page_64(thdescsz);
1173 vm_offset_t thdesce = thdescb + thdescszr;
1174
1175 assert((hdescb & 0xFFF) == 0);
1176 /* Mirror HIB translations into the double-mapped pagetable subtree*/
1177 for (int i = 0; hdescc < hdesce; i++) {
1178 struct {
1179 pt_entry_t entries[PTE_PER_PAGE];
1180 } * dblmapL1 = ALLOCPAGES(1);
1181 dblallocs++;
1182 dblmapL2[0].entries[i] = ((uintptr_t)ID_MAP_VTOP(&dblmapL1[0])) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
1183 int hdescn = (int) ((hdesce - hdescc) / PAGE_SIZE);
1184 for (int j = 0; j < MIN(PTE_PER_PAGE, hdescn); j++) {
1185 uint64_t template = INTEL_PTE_VALID;
1186 if ((hdescc >= thdescb) && (hdescc < thdesce)) {
1187 /* executable */
1188 } else {
1189 template |= INTEL_PTE_WRITE | INTEL_PTE_NX; /* Writeable, NX */
1190 }
1191 dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template;
1192 hdescc += PAGE_SIZE;
1193 }
1194 }
1195
1196 IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
1197
1198 dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, randL3, 0, 0);
1199 dblmap_max = dblmap_base + hdescszr;
1200 /* Calculate the double-map distance, which accounts for the current
1201 * KASLR slide
1202 */
1203
1204 dblmap_dist = dblmap_base - hdescb;
1205 idt64_hndl_table0[1] = DBLMAP(idt64_hndl_table0[1]); /* 64-bit exit trampoline */
1206 idt64_hndl_table0[3] = DBLMAP(idt64_hndl_table0[3]); /* 32-bit exit trampoline */
1207 idt64_hndl_table0[6] = (uint64_t)(uintptr_t)&kernel_stack_mask;
1208
1209 extern cpu_data_t cpshadows[], scdatas[];
1210 uintptr_t cd1 = (uintptr_t) &cpshadows[0];
1211 uintptr_t cd2 = (uintptr_t) &scdatas[0];
1212 /* Record the displacement from the kernel's per-CPU data pointer, eventually
1213 * programmed into GSBASE, to the "shadows" in the doublemapped
1214 * region. These are not aliases, but separate physical allocations
1215 * containing data required in the doublemapped trampolines.
1216 */
1217 idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2;
1218
1219 DBG("Double map base: 0x%qx\n", dblmap_base);
1220 DBG("double map idlepml4[%d]: 0x%llx\n", KERNEL_DBLMAP_PML4_INDEX, IdlePML4[KERNEL_DBLMAP_PML4_INDEX]);
1221 assert(LDTSZ > LDTSZ_MIN);
1222 }
1223
1224 vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t);
1225
1226 #include <i386/pmap_internal.h>
1227
1228 /* Use of this routine is expected to be synchronized by callers
1229 * Creates non-executable aliases.
1230 */
1231 vm_offset_t
dyn_dblmap(vm_offset_t cva,vm_offset_t sz)1232 dyn_dblmap(vm_offset_t cva, vm_offset_t sz)
1233 {
1234 vm_offset_t ava = dblmap_max;
1235
1236 assert((sz & PAGE_MASK) == 0);
1237 assert(cva != 0);
1238
1239 pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP);
1240 dblmap_max += sz;
1241 return ava - cva;
1242 }
1243 /* Adjust offsets interior to the bootstrap interrupt descriptor table to redirect
1244 * control to the double-mapped interrupt vectors. The IDTR proper will be
1245 * programmed via cpu_desc_load()
1246 */
1247 void
idt64_remap(void)1248 idt64_remap(void)
1249 {
1250 for (int i = 0; i < IDTSZ; i++) {
1251 master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64);
1252 }
1253 }
1254