1 /*
2 * Copyright (c) 2003-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57
58 #include <mach/i386/vm_param.h>
59
60 #include <string.h>
61 #include <stdint.h>
62 #include <mach/vm_param.h>
63 #include <mach/vm_prot.h>
64 #include <mach/machine.h>
65 #include <mach/time_value.h>
66 #include <kern/spl.h>
67 #include <kern/assert.h>
68 #include <kern/debug.h>
69 #include <kern/misc_protos.h>
70 #include <kern/startup.h>
71 #include <kern/clock.h>
72 #include <kern/pms.h>
73 #include <kern/cpu_data.h>
74 #include <kern/processor.h>
75 #include <kern/monotonic.h>
76 #include <sys/kdebug.h>
77 #include <console/serial_protos.h>
78 #include <vm/vm_page.h>
79 #include <vm/pmap.h>
80 #include <vm/vm_kern.h>
81 #include <machine/pal_routines.h>
82 #include <i386/fpu.h>
83 #include <i386/pmap.h>
84 #include <i386/misc_protos.h>
85 #include <i386/cpu_threads.h>
86 #include <i386/cpuid.h>
87 #include <i386/lapic.h>
88 #include <i386/mp.h>
89 #include <i386/mp_desc.h>
90 #if CONFIG_MTRR
91 #include <i386/mtrr.h>
92 #endif
93 #include <i386/machine_routines.h>
94 #if CONFIG_MCA
95 #include <i386/machine_check.h>
96 #endif
97 #include <i386/ucode.h>
98 #include <i386/postcode.h>
99 #include <i386/Diagnostics.h>
100 #include <i386/pmCPU.h>
101 #include <i386/tsc.h>
102 #include <i386/acpi.h>
103 #if DEBUG
104 #include <machine/pal_routines.h>
105 #endif
106 extern void xcpm_bootstrap(void);
107 #if DEVELOPMENT || DEBUG
108 #include <i386/trap_internal.h>
109 #endif
110
111 #if KPERF
112 #include <kperf/kptimer.h>
113 #endif /* KPERF */
114
115 #include <san/kasan.h>
116
117 #if DEBUG || DEVELOPMENT
118 #define DBG(x, ...) kprintf(x, ##__VA_ARGS__)
119 #define dyldLogFunc(x, ...) kprintf(x, ##__VA_ARGS__)
120 #else
121 #define DBG(x ...)
122 #endif
123
124 #include <libkern/kernel_mach_header.h>
125 #include <mach/dyld_kernel_fixups.h>
126
127
128 int debug_task;
129
130 int early_boot = 1;
131
132 bool serial_console_enabled = false;
133
134 static boot_args *kernelBootArgs;
135
136 extern int disableConsoleOutput;
137 extern const char version[];
138 extern const char version_variant[];
139 extern int nx_enabled;
140
141 /*
142 * Set initial values so that ml_phys_* routines can use the booter's ID mapping
143 * to touch physical space before the kernel's physical aperture exists.
144 */
145 uint64_t physmap_base = 0;
146 uint64_t physmap_max = 4 * GB;
147
148 pd_entry_t *KPTphys;
149 pd_entry_t *IdlePTD;
150 pdpt_entry_t *IdlePDPT;
151 pml4_entry_t *IdlePML4;
152
153 int kernPhysPML4Index;
154 int kernPhysPML4EntryCount;
155
156 /*
157 * These are 4K mapping page table pages from KPTphys[] that we wound
158 * up not using. They get ml_static_mfree()'d once the VM is initialized.
159 */
160 ppnum_t released_PT_ppn = 0;
161 uint32_t released_PT_cnt = 0;
162
163 #if DEVELOPMENT || DEBUG
164 int panic_on_cacheline_mismatch = -1;
165 char panic_on_trap_procname[64];
166 uint32_t panic_on_trap_mask;
167 #endif
168 lbr_modes_t last_branch_enabled_modes;
169 int insn_copyin_count;
170 #if DEVELOPMENT || DEBUG
171 #define DEFAULT_INSN_COPYIN_COUNT x86_INSTRUCTION_STATE_MAX_INSN_BYTES
172 #else
173 #define DEFAULT_INSN_COPYIN_COUNT 192
174 #endif
175
176 char *physfree;
177 void idt64_remap(void);
178
179 TUNABLE(bool, restore_boot, "-restore", false);
180
181 /*
182 * Note: ALLOCPAGES() can only be used safely within Idle_PTs_init()
183 * due to the mutation of physfree.
184 */
185 static void *
ALLOCPAGES(int npages)186 ALLOCPAGES(int npages)
187 {
188 uintptr_t tmp = (uintptr_t)physfree;
189 bzero(physfree, npages * PAGE_SIZE);
190 physfree += npages * PAGE_SIZE;
191 tmp += VM_MIN_KERNEL_ADDRESS & ~LOW_4GB_MASK;
192 return (void *)tmp;
193 }
194
195 static void
fillkpt(pt_entry_t * base,int prot,uintptr_t src,int index,int count)196 fillkpt(pt_entry_t *base, int prot, uintptr_t src, int index, int count)
197 {
198 int i;
199 for (i = 0; i < count; i++) {
200 base[index] = src | prot | INTEL_PTE_VALID;
201 src += PAGE_SIZE;
202 index++;
203 }
204 }
205
206 extern pmap_paddr_t first_avail;
207
208 int break_kprintf = 0;
209
210 uint64_t
x86_64_pre_sleep(void)211 x86_64_pre_sleep(void)
212 {
213 IdlePML4[0] = IdlePML4[KERNEL_PML4_INDEX];
214 uint64_t oldcr3 = get_cr3_raw();
215 set_cr3_raw((uint32_t) (uintptr_t)ID_MAP_VTOP(IdlePML4));
216 return oldcr3;
217 }
218
219 void
x86_64_post_sleep(uint64_t new_cr3)220 x86_64_post_sleep(uint64_t new_cr3)
221 {
222 IdlePML4[0] = 0;
223 set_cr3_raw((uint32_t) new_cr3);
224 }
225
226
227
228
229 // Set up the physical mapping - NPHYSMAP GB of memory mapped at a high address
230 // NPHYSMAP is determined by the maximum supported RAM size plus 4GB to account
231 // the PCI hole (which is less 4GB but not more).
232
233 static int
physmap_init_L2(uint64_t * physStart,pt_entry_t ** l2ptep)234 physmap_init_L2(uint64_t *physStart, pt_entry_t **l2ptep)
235 {
236 unsigned i;
237 pt_entry_t *physmapL2 = ALLOCPAGES(1);
238
239 if (physmapL2 == NULL) {
240 DBG("physmap_init_L2 page alloc failed when initting L2 for physAddr 0x%llx.\n", *physStart);
241 *l2ptep = NULL;
242 return -1;
243 }
244
245 for (i = 0; i < NPDPG; i++) {
246 physmapL2[i] = *physStart
247 | INTEL_PTE_PS
248 | INTEL_PTE_VALID
249 | INTEL_PTE_NX
250 | INTEL_PTE_WRITE;
251
252 *physStart += NBPD;
253 }
254 *l2ptep = physmapL2;
255 return 0;
256 }
257
258 static int
physmap_init_L3(int startIndex,uint64_t highest_phys,uint64_t * physStart,pt_entry_t ** l3ptep)259 physmap_init_L3(int startIndex, uint64_t highest_phys, uint64_t *physStart, pt_entry_t **l3ptep)
260 {
261 unsigned i;
262 int ret;
263 pt_entry_t *l2pte;
264 pt_entry_t *physmapL3 = ALLOCPAGES(1); /* ALLOCPAGES bzeroes the memory */
265
266 if (physmapL3 == NULL) {
267 DBG("physmap_init_L3 page alloc failed when initting L3 for physAddr 0x%llx.\n", *physStart);
268 *l3ptep = NULL;
269 return -1;
270 }
271
272 for (i = startIndex; i < NPDPTPG && *physStart < highest_phys; i++) {
273 if ((ret = physmap_init_L2(physStart, &l2pte)) < 0) {
274 return ret;
275 }
276
277 physmapL3[i] = ((uintptr_t)ID_MAP_VTOP(l2pte))
278 | INTEL_PTE_VALID
279 | INTEL_PTE_NX
280 | INTEL_PTE_WRITE;
281 }
282
283 *l3ptep = physmapL3;
284
285 return 0;
286 }
287
288 static void
physmap_init(uint8_t phys_random_L3,uint64_t * new_physmap_base,uint64_t * new_physmap_max)289 physmap_init(uint8_t phys_random_L3, uint64_t *new_physmap_base, uint64_t *new_physmap_max)
290 {
291 pt_entry_t *l3pte;
292 int pml4_index, i;
293 int L3_start_index;
294 uint64_t physAddr = 0;
295 uint64_t highest_physaddr;
296 unsigned pdpte_count;
297
298 #if DEVELOPMENT || DEBUG
299 if (kernelBootArgs->PhysicalMemorySize > K64_MAXMEM) {
300 panic("Installed physical memory exceeds configured maximum.");
301 }
302 #endif
303
304 /*
305 * Add 4GB to the loader-provided physical memory size to account for MMIO space
306 * XXX in a perfect world, we'd scan PCI buses and count the max memory requested in BARs by
307 * XXX all enumerated device, then add more for hot-pluggable devices.
308 */
309 highest_physaddr = kernelBootArgs->PhysicalMemorySize + 4 * GB;
310
311 /*
312 * Calculate the number of PML4 entries we'll need. The total number of entries is
313 * pdpte_count = (((highest_physaddr) >> PDPT_SHIFT) + entropy_value +
314 * ((highest_physaddr & PDPT_MASK) == 0 ? 0 : 1))
315 * pml4e_count = pdpte_count >> (PML4_SHIFT - PDPT_SHIFT)
316 */
317 assert(highest_physaddr < (UINT64_MAX - PDPTMASK));
318 pdpte_count = (unsigned) (((highest_physaddr + PDPTMASK) >> PDPTSHIFT) + phys_random_L3);
319 kernPhysPML4EntryCount = (pdpte_count + ((1U << (PML4SHIFT - PDPTSHIFT)) - 1)) >> (PML4SHIFT - PDPTSHIFT);
320 if (kernPhysPML4EntryCount == 0) {
321 kernPhysPML4EntryCount = 1;
322 }
323 if (kernPhysPML4EntryCount > KERNEL_PHYSMAP_PML4_COUNT_MAX) {
324 #if DEVELOPMENT || DEBUG
325 panic("physmap too large");
326 #else
327 kprintf("[pmap] Limiting physmap to %d PML4s (was %d)\n", KERNEL_PHYSMAP_PML4_COUNT_MAX,
328 kernPhysPML4EntryCount);
329 kernPhysPML4EntryCount = KERNEL_PHYSMAP_PML4_COUNT_MAX;
330 #endif
331 }
332
333 kernPhysPML4Index = KERNEL_KEXTS_INDEX - kernPhysPML4EntryCount; /* utb: KERNEL_PHYSMAP_PML4_INDEX */
334
335 /*
336 * XXX: Make sure that the addresses returned for physmapL3 and physmapL2 plus their extents
337 * are in the system-available memory range
338 */
339
340
341 /* We assume NX support. Mark all levels of the PHYSMAP NX
342 * to avoid granting executability via a single bit flip.
343 */
344 #if DEVELOPMENT || DEBUG
345 uint32_t reg[4];
346 do_cpuid(0x80000000, reg);
347 if (reg[eax] >= 0x80000001) {
348 do_cpuid(0x80000001, reg);
349 assert(reg[edx] & CPUID_EXTFEATURE_XD);
350 }
351 #endif /* DEVELOPMENT || DEBUG */
352
353 L3_start_index = phys_random_L3;
354
355 for (pml4_index = kernPhysPML4Index;
356 pml4_index < (kernPhysPML4Index + kernPhysPML4EntryCount) && physAddr < highest_physaddr;
357 pml4_index++) {
358 if (physmap_init_L3(L3_start_index, highest_physaddr, &physAddr, &l3pte) < 0) {
359 panic("Physmap page table initialization failed");
360 /* NOTREACHED */
361 }
362
363 L3_start_index = 0;
364
365 IdlePML4[pml4_index] = ((uintptr_t)ID_MAP_VTOP(l3pte))
366 | INTEL_PTE_VALID
367 | INTEL_PTE_NX
368 | INTEL_PTE_WRITE;
369 }
370
371 *new_physmap_base = KVADDR(kernPhysPML4Index, phys_random_L3, 0, 0);
372 /*
373 * physAddr contains the last-mapped physical address, so that's what we
374 * add to physmap_base to derive the ending VA for the physmap.
375 */
376 *new_physmap_max = *new_physmap_base + physAddr;
377
378 DBG("Physical address map base: 0x%qx\n", *new_physmap_base);
379 for (i = kernPhysPML4Index; i < (kernPhysPML4Index + kernPhysPML4EntryCount); i++) {
380 DBG("Physical map idlepml4[%d]: 0x%llx\n", i, IdlePML4[i]);
381 }
382 }
383
384 void doublemap_init(uint8_t);
385
386 static void
Idle_PTs_init(void)387 Idle_PTs_init(void)
388 {
389 uint64_t rand64;
390 uint64_t new_physmap_base, new_physmap_max;
391
392 /* Allocate the "idle" kernel page tables: */
393 KPTphys = ALLOCPAGES(NKPT); /* level 1 */
394 IdlePTD = ALLOCPAGES(NPGPTD); /* level 2 */
395 IdlePDPT = ALLOCPAGES(1); /* level 3 */
396 IdlePML4 = ALLOCPAGES(1); /* level 4 */
397
398 // Fill the lowest level with everything up to physfree
399 fillkpt(KPTphys,
400 INTEL_PTE_WRITE, 0, 0, (int)(((uintptr_t)physfree) >> PAGE_SHIFT));
401
402 /* IdlePTD */
403 fillkpt(IdlePTD,
404 INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(KPTphys), 0, NKPT);
405
406 // IdlePDPT entries
407 fillkpt(IdlePDPT,
408 INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePTD), 0, NPGPTD);
409
410 // IdlePML4 single entry for kernel space.
411 fillkpt(IdlePML4 + KERNEL_PML4_INDEX,
412 INTEL_PTE_WRITE, (uintptr_t)ID_MAP_VTOP(IdlePDPT), 0, 1);
413
414 postcode(VSTART_PHYSMAP_INIT);
415
416 /*
417 * early_random() cannot be called more than one time before the cpu's
418 * gsbase is initialized, so use the full 64-bit value to extract the
419 * two 8-bit entropy values needed for address randomization.
420 */
421 rand64 = early_random();
422 physmap_init(rand64 & 0xFF, &new_physmap_base, &new_physmap_max);
423 doublemap_init((rand64 >> 8) & 0xFF);
424 idt64_remap();
425
426 postcode(VSTART_SET_CR3);
427
428 /*
429 * Switch to the page tables. We set physmap_base and physmap_max just
430 * before switching to the new page tables to avoid someone calling
431 * kprintf() or otherwise using physical memory in between.
432 * This is needed because kprintf() writes to physical memory using
433 * ml_phys_read_data and PHYSMAP_PTOV, which requires physmap_base to be
434 * set correctly.
435 */
436 physmap_base = new_physmap_base;
437 physmap_max = new_physmap_max;
438 set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
439 }
440
441 /*
442 * Release any still unused, preallocated boot kernel page tables.
443 * start..end is the VA range currently unused.
444 */
445 void
Idle_PTs_release(vm_offset_t start,vm_offset_t end)446 Idle_PTs_release(vm_offset_t start, vm_offset_t end)
447 {
448 uint32_t i;
449 uint32_t index_start;
450 uint32_t index_limit;
451 ppnum_t pn_first;
452 ppnum_t pn;
453 uint32_t cnt;
454
455 /*
456 * Align start to the next large page boundary
457 */
458 start = ((start + I386_LPGMASK) & ~I386_LPGMASK);
459
460 /*
461 * convert start into an index in KPTphys[]
462 */
463 index_start = (uint32_t)((start - KERNEL_BASE) >> PAGE_SHIFT);
464
465 /*
466 * Find the ending index in KPTphys[]
467 */
468 index_limit = (uint32_t)((end - KERNEL_BASE) >> PAGE_SHIFT);
469
470 if (index_limit > NKPT * PTE_PER_PAGE) {
471 index_limit = NKPT * PTE_PER_PAGE;
472 }
473
474 /*
475 * Make sure all the 4K page tables are empty.
476 * If not, panic a development/debug kernel.
477 * On a production kernel, since this would stop us from booting,
478 * just abort the operation.
479 */
480 for (i = index_start; i < index_limit; ++i) {
481 assert(KPTphys[i] == 0);
482 if (KPTphys[i] != 0) {
483 return;
484 }
485 }
486
487 /*
488 * Now figure out the indices into the 2nd level page tables, IdlePTD[].
489 */
490 index_start >>= PTPGSHIFT;
491 index_limit >>= PTPGSHIFT;
492 if (index_limit > NPGPTD * PTE_PER_PAGE) {
493 index_limit = NPGPTD * PTE_PER_PAGE;
494 }
495
496 if (index_limit <= index_start) {
497 return;
498 }
499
500
501 /*
502 * Now check the pages referenced from Level 2 tables.
503 * They should be contiguous, assert fail if not on development/debug.
504 * In production, just fail the removal to allow the system to boot.
505 */
506 pn_first = 0;
507 cnt = 0;
508 for (i = index_start; i < index_limit; ++i) {
509 assert(IdlePTD[i] != 0);
510 if (IdlePTD[i] == 0) {
511 return;
512 }
513
514 pn = (ppnum_t)((PG_FRAME & IdlePTD[i]) >> PTSHIFT);
515 if (cnt == 0) {
516 pn_first = pn;
517 } else {
518 assert(pn == pn_first + cnt);
519 if (pn != pn_first + cnt) {
520 return;
521 }
522 }
523 ++cnt;
524 }
525
526 /*
527 * Good to go, clear the level 2 entries and invalidate the TLB
528 */
529 for (i = index_start; i < index_limit; ++i) {
530 IdlePTD[i] = 0;
531 }
532 set_cr3_raw(get_cr3_raw());
533
534 /*
535 * Remember these PFNs to be released later in pmap_lowmem_finalize()
536 */
537 released_PT_ppn = pn_first;
538 released_PT_cnt = cnt;
539 #if DEVELOPMENT || DEBUG
540 printf("Idle_PTs_release %d pages from PFN 0x%x\n", released_PT_cnt, released_PT_ppn);
541 #endif
542 }
543
544 extern void vstart_trap_handler;
545
546 #define BOOT_TRAP_VECTOR(t) \
547 [t] = { \
548 (uintptr_t) &vstart_trap_handler, \
549 KERNEL64_CS, \
550 0, \
551 ACC_P|ACC_PL_K|ACC_INTR_GATE, \
552 0 \
553 },
554
555 /* Recursive macro to iterate 0..31 */
556 #define L0(x, n) x(n)
557 #define L1(x, n) L0(x,n-1) L0(x,n)
558 #define L2(x, n) L1(x,n-2) L1(x,n)
559 #define L3(x, n) L2(x,n-4) L2(x,n)
560 #define L4(x, n) L3(x,n-8) L3(x,n)
561 #define L5(x, n) L4(x,n-16) L4(x,n)
562 #define FOR_0_TO_31(x) L5(x,31)
563
564 /*
565 * Bootstrap IDT. Active only during early startup.
566 * Only the trap vectors are defined since interrupts are masked.
567 * All traps point to a common handler.
568 */
569 struct fake_descriptor64 master_boot_idt64[IDTSZ]
570 __attribute__((section("__HIB,__desc")))
571 __attribute__((aligned(PAGE_SIZE))) = {
572 FOR_0_TO_31(BOOT_TRAP_VECTOR)
573 };
574
575 static void
vstart_idt_init(boolean_t master)576 vstart_idt_init(boolean_t master)
577 {
578 x86_64_desc_register_t vstart_idt = {
579 sizeof(master_boot_idt64),
580 master_boot_idt64
581 };
582
583 if (master) {
584 fix_desc64(master_boot_idt64, 32);
585 }
586 lidt((void *)&vstart_idt);
587 }
588
589 extern void *collection_base_pointers[KCNumKinds];
590
591 kern_return_t
i386_slide_individual_kext(kernel_mach_header_t * mh,uintptr_t slide)592 i386_slide_individual_kext(kernel_mach_header_t *mh, uintptr_t slide)
593 {
594 int ret = kernel_collection_slide(mh, (const void **) (void *)collection_base_pointers);
595 if (ret != 0) {
596 printf("Sliding pageable kc was stopped\n");
597 return KERN_FAILURE;
598 }
599
600 kernel_collection_adjust_fileset_entry_addrs(mh, slide);
601 return KERN_SUCCESS;
602 }
603
604 kern_return_t
i386_slide_kext_collection_mh_addrs(kernel_mach_header_t * mh,uintptr_t slide,bool adjust_mach_headers)605 i386_slide_kext_collection_mh_addrs(kernel_mach_header_t *mh, uintptr_t slide, bool adjust_mach_headers)
606 {
607 int ret = kernel_collection_slide(mh, (const void **) (void *)collection_base_pointers);
608 if (ret != KERN_SUCCESS) {
609 printf("Kernel Collection slide was stopped with value %d\n", ret);
610 return KERN_FAILURE;
611 }
612
613 kernel_collection_adjust_mh_addrs(mh, slide, adjust_mach_headers,
614 NULL, NULL, NULL, NULL, NULL, NULL, NULL);
615
616 return KERN_SUCCESS;
617 }
618
619 static void
i386_slide_and_rebase_image(uintptr_t kstart_addr)620 i386_slide_and_rebase_image(uintptr_t kstart_addr)
621 {
622 extern uintptr_t kc_highest_nonlinkedit_vmaddr;
623 kernel_mach_header_t *k_mh, *kc_mh = NULL;
624 kernel_segment_command_t *seg;
625 uintptr_t slide;
626
627 k_mh = &_mh_execute_header;
628 /*
629 * If we're not booting, an MH_FILESET, we don't need to slide
630 * anything because EFI has done that for us. When booting an
631 * MH_FILESET, EFI will slide the kernel proper, but not the kexts.
632 * Below, we infer the slide by comparing the slid address of the
633 * kernel's mach-o header and the unslid vmaddr of the first segment
634 * of the mach-o (which is assumed to always point to the mach-o
635 * header).
636 */
637 if (!kernel_mach_header_is_in_fileset(k_mh)) {
638 DBG("[MH] kcgen-style KC\n");
639 return;
640 }
641
642 /*
643 * The kernel is part of a MH_FILESET kernel collection: determine slide
644 * based on first segment's mach-o vmaddr.
645 */
646 seg = (kernel_segment_command_t *)((uintptr_t)k_mh + sizeof(*k_mh));
647 assert(seg->cmd == LC_SEGMENT_KERNEL);
648 slide = (uintptr_t)k_mh - seg->vmaddr;
649 DBG("[MH] Sliding new-style KC: %llu\n", (unsigned long long)slide);
650
651 /*
652 * The kernel collection mach-o header should be the start address
653 * passed to us by EFI.
654 */
655 kc_mh = (kernel_mach_header_t *)(kstart_addr);
656 assert(kc_mh->filetype == MH_FILESET);
657
658 PE_set_kc_header(KCKindPrimary, kc_mh, slide);
659
660 /*
661 * rebase/slide all the kexts in the collection
662 * (EFI should have already rebased the kernel)
663 */
664 kernel_collection_slide(kc_mh, (const void **) (void *)collection_base_pointers);
665
666
667 /*
668 * Now adjust the vmaddr fields of all mach-o headers
669 * and symbols in this MH_FILESET
670 */
671 kernel_collection_adjust_mh_addrs(kc_mh, slide, false,
672 NULL, NULL, NULL, NULL, NULL, NULL, &kc_highest_nonlinkedit_vmaddr);
673 }
674
675 /*
676 * vstart() is called in the natural mode (64bit for K64, 32 for K32)
677 * on a set of bootstrap pagetables which use large, 2MB pages to map
678 * all of physical memory in both. See idle_pt.c for details.
679 *
680 * In K64 this identity mapping is mirrored the top and bottom 512GB
681 * slots of PML4.
682 *
683 * The bootstrap processor called with argument boot_args_start pointing to
684 * the boot-args block. The kernel's (4K page) page tables are allocated and
685 * initialized before switching to these.
686 *
687 * Non-bootstrap processors are called with argument boot_args_start NULL.
688 * These processors switch immediately to the existing kernel page tables.
689 */
690 __attribute__((noreturn))
691 void
vstart(vm_offset_t boot_args_start)692 vstart(vm_offset_t boot_args_start)
693 {
694 boolean_t is_boot_cpu = !(boot_args_start == 0);
695 int cpu = 0;
696 uint32_t lphysfree;
697 #if DEBUG
698 uint64_t gsbase;
699 #endif
700
701
702 postcode(VSTART_ENTRY);
703
704 /*
705 * Set-up temporary trap handlers during page-table set-up.
706 */
707
708 if (is_boot_cpu) {
709 vstart_idt_init(TRUE);
710 postcode(VSTART_IDT_INIT);
711
712 /*
713 * Ensure that any %gs-relative access results in an immediate fault
714 * until gsbase is properly initialized below
715 */
716 wrmsr64(MSR_IA32_GS_BASE, EARLY_GSBASE_MAGIC);
717
718 /*
719 * Get startup parameters.
720 */
721 kernelBootArgs = (boot_args *)boot_args_start;
722 lphysfree = kernelBootArgs->kaddr + kernelBootArgs->ksize;
723 physfree = (void *)(uintptr_t)((lphysfree + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1));
724
725 pal_serial_init();
726
727 DBG("revision 0x%x\n", kernelBootArgs->Revision);
728 DBG("version 0x%x\n", kernelBootArgs->Version);
729 DBG("command line %s\n", kernelBootArgs->CommandLine);
730 DBG("memory map 0x%x\n", kernelBootArgs->MemoryMap);
731 DBG("memory map sz 0x%x\n", kernelBootArgs->MemoryMapSize);
732 DBG("kaddr 0x%x\n", kernelBootArgs->kaddr);
733 DBG("ksize 0x%x\n", kernelBootArgs->ksize);
734 DBG("physfree %p\n", physfree);
735 DBG("bootargs: %p, &ksize: %p &kaddr: %p\n",
736 kernelBootArgs,
737 &kernelBootArgs->ksize,
738 &kernelBootArgs->kaddr);
739 DBG("SMBIOS mem sz 0x%llx\n", kernelBootArgs->PhysicalMemorySize);
740 DBG("KC_hdrs_vaddr %p\n", (void *)kernelBootArgs->KC_hdrs_vaddr);
741
742 if (kernelBootArgs->Version >= 2 && kernelBootArgs->Revision >= 1 &&
743 kernelBootArgs->KC_hdrs_vaddr != 0) {
744 /*
745 * slide the header addresses in all mach-o segments and sections, and
746 * perform any new-style chained-fixup sliding for kexts, as necessary.
747 * Note that efiboot has already loaded the kernel and all LC_SEGMENT_64s
748 * that correspond to the kexts present in the primary KC, into slid addresses.
749 */
750 i386_slide_and_rebase_image((uintptr_t)ml_static_ptovirt(kernelBootArgs->KC_hdrs_vaddr));
751 }
752
753 /*
754 * Setup boot args given the physical start address.
755 * Note: PE_init_platform needs to be called before Idle_PTs_init
756 * because access to the DeviceTree is required to read the
757 * random seed before generating a random physical map slide.
758 */
759 kernelBootArgs = (boot_args *)
760 ml_static_ptovirt(boot_args_start);
761 DBG("i386_init(0x%lx) kernelBootArgs=%p\n",
762 (unsigned long)boot_args_start, kernelBootArgs);
763
764 #if KASAN
765 kasan_reserve_memory(kernelBootArgs);
766 #endif
767
768 PE_init_platform(FALSE, kernelBootArgs);
769 postcode(PE_INIT_PLATFORM_D);
770
771 Idle_PTs_init();
772 postcode(VSTART_IDLE_PTS_INIT);
773
774 #if KASAN
775 /* Init kasan and map whatever was stolen from physfree */
776 kasan_init();
777 kasan_notify_stolen((uintptr_t)ml_static_ptovirt((vm_offset_t)physfree));
778 #endif
779
780 #if CONFIG_CPU_COUNTERS
781 mt_early_init();
782 #endif /* CONFIG_CPU_COUNTERS */
783
784 first_avail = (vm_offset_t)ID_MAP_VTOP(physfree);
785
786 cpu_data_alloc(TRUE);
787
788 cpu_desc_init(cpu_datap(0));
789 postcode(VSTART_CPU_DESC_INIT);
790 cpu_desc_load(cpu_datap(0));
791
792 postcode(VSTART_CPU_MODE_INIT);
793 cpu_syscall_init(cpu_datap(0)); /* cpu_syscall_init() will be
794 * invoked on the APs
795 * via i386_init_slave()
796 */
797 } else {
798 /* Slave CPUs should use the basic IDT until i386_init_slave() */
799 vstart_idt_init(FALSE);
800
801 /* Switch to kernel's page tables (from the Boot PTs) */
802 set_cr3_raw((uintptr_t)ID_MAP_VTOP(IdlePML4));
803
804 /* Find our logical cpu number */
805 cpu = lapic_to_cpu[lapic_safe_apicid()];
806 #if DEBUG
807 gsbase = rdmsr64(MSR_IA32_GS_BASE);
808 #endif
809 cpu_desc_load(cpu_datap(cpu));
810 #if DEBUG
811 DBG("CPU: %d, GSBASE initial value: 0x%llx\n", cpu, (unsigned long long)gsbase);
812 #endif
813
814 /*
815 * Before we can discover our local APIC ID, we need to potentially
816 * initialize X2APIC, if it's enabled and firmware started us with
817 * the APIC in legacy mode.
818 */
819 lapic_init_slave();
820 }
821
822 early_boot = 0;
823 postcode(VSTART_EXIT);
824 x86_init_wrapper(is_boot_cpu ? (uintptr_t) i386_init
825 : (uintptr_t) i386_init_slave,
826 cpu_datap(cpu)->cpu_int_stack_top);
827 }
828
829 void
pstate_trace(void)830 pstate_trace(void)
831 {
832 }
833
834 /*
835 * Cpu initialization. Running virtual, but without MACH VM
836 * set up.
837 */
838 void
i386_init(void)839 i386_init(void)
840 {
841 unsigned int maxmem;
842 uint64_t maxmemtouse;
843 unsigned int cpus = 0;
844 boolean_t fidn;
845 boolean_t IA32e = TRUE;
846
847 postcode(I386_INIT_ENTRY);
848
849 pal_i386_init();
850 tsc_init();
851 rtclock_early_init(); /* mach_absolute_time() now functional */
852
853 kernel_debug_string_early("i386_init");
854 pstate_trace();
855
856 #if CONFIG_MCA
857 /* Initialize machine-check handling */
858 mca_cpu_init();
859 #endif
860
861 master_cpu = 0;
862
863 kernel_debug_string_early("kernel_startup_bootstrap");
864 kernel_startup_bootstrap();
865
866 /*
867 * Initialize the timer callout world
868 */
869 timer_call_init();
870
871 cpu_init();
872
873 postcode(CPU_INIT_D);
874
875 /* setup debugging output if one has been chosen */
876 kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
877 kprintf("kprintf initialized\n");
878
879 if (!PE_parse_boot_argn("diag", &dgWork.dgFlags, sizeof(dgWork.dgFlags))) {
880 dgWork.dgFlags = 0;
881 }
882
883 if (PE_parse_boot_argn("insn_capcnt", &insn_copyin_count, sizeof(insn_copyin_count))) {
884 /*
885 * Enforce max and min values (allowing 0 to disable copying completely)
886 * for the instruction copyin count
887 */
888 if (insn_copyin_count > x86_INSTRUCTION_STATE_MAX_INSN_BYTES ||
889 (insn_copyin_count != 0 && insn_copyin_count < 64)) {
890 insn_copyin_count = DEFAULT_INSN_COPYIN_COUNT;
891 }
892 } else {
893 insn_copyin_count = DEFAULT_INSN_COPYIN_COUNT;
894 }
895
896 #if DEVELOPMENT || DEBUG
897 if (!PE_parse_boot_argn("panic_clmismatch", &panic_on_cacheline_mismatch,
898 sizeof(panic_on_cacheline_mismatch))) {
899 panic_on_cacheline_mismatch = 0;
900 }
901
902 if (!PE_parse_boot_argn("panic_on_trap_procname", &panic_on_trap_procname[0],
903 sizeof(panic_on_trap_procname))) {
904 panic_on_trap_procname[0] = 0;
905 }
906
907 if (!PE_parse_boot_argn("panic_on_trap_mask", &panic_on_trap_mask,
908 sizeof(panic_on_trap_mask))) {
909 if (panic_on_trap_procname[0] != 0) {
910 panic_on_trap_mask = DEFAULT_PANIC_ON_TRAP_MASK;
911 } else {
912 panic_on_trap_mask = 0;
913 }
914 }
915 #endif
916 /* But allow that to be overridden via boot-arg: */
917 if (!PE_parse_boot_argn("lbr_support", &last_branch_enabled_modes,
918 sizeof(last_branch_enabled_modes))) {
919 /* Disable LBR support by default due to its high context switch overhead */
920 last_branch_enabled_modes = LBR_ENABLED_NONE;
921 }
922
923 serialmode = 0;
924 if (PE_parse_boot_argn("serial", &serialmode, sizeof(serialmode))) {
925 /* We want a serial keyboard and/or console */
926 kprintf("Serial mode specified: %08X\n", serialmode);
927 int force_sync = serialmode & SERIALMODE_SYNCDRAIN;
928 disable_iolog_serial_output = (serialmode & SERIALMODE_NO_IOLOG) != 0;
929 enable_dklog_serial_output = restore_boot || (serialmode & SERIALMODE_DKLOG) != 0;
930 if (force_sync || PE_parse_boot_argn("drain_uart_sync", &force_sync, sizeof(force_sync))) {
931 if (force_sync) {
932 serialmode |= SERIALMODE_SYNCDRAIN;
933 kprintf(
934 "WARNING: Forcing uart driver to output synchronously."
935 "printf()s/IOLogs will impact kernel performance.\n"
936 "You are advised to avoid using 'drain_uart_sync' boot-arg.\n");
937 }
938 }
939 }
940 if (serialmode & SERIALMODE_OUTPUT) {
941 serial_console_enabled = true;
942 (void)switch_to_serial_console();
943 disableConsoleOutput = FALSE; /* Allow printfs to happen */
944 }
945
946 /* setup console output */
947 kernel_debug_string_early("PE_init_printf");
948 PE_init_printf(FALSE);
949
950 kprintf("version_variant = %s\n", version_variant);
951 kprintf("version = %s\n", version);
952
953 if (!PE_parse_boot_argn("maxmem", &maxmem, sizeof(maxmem))) {
954 maxmemtouse = 0;
955 } else {
956 maxmemtouse = ((uint64_t)maxmem) * MB;
957 }
958
959 max_cpus_from_firmware = acpi_count_enabled_logical_processors();
960
961 if (PE_parse_boot_argn("cpus", &cpus, sizeof(cpus))) {
962 if ((0 < cpus) && (cpus < max_ncpus)) {
963 max_ncpus = cpus;
964 }
965 }
966
967 /*
968 * debug support for > 4G systems
969 */
970 PE_parse_boot_argn("himemory_mode", &vm_himemory_mode, sizeof(vm_himemory_mode));
971 if (!vm_himemory_mode) {
972 kprintf("himemory_mode disabled\n");
973 }
974
975 if (!PE_parse_boot_argn("immediate_NMI", &fidn, sizeof(fidn))) {
976 force_immediate_debugger_NMI = FALSE;
977 } else {
978 force_immediate_debugger_NMI = fidn;
979 }
980
981 #if DEBUG
982 nanoseconds_to_absolutetime(URGENCY_NOTIFICATION_ASSERT_NS, &urgency_notification_assert_abstime_threshold);
983 #endif
984 PE_parse_boot_argn("urgency_notification_abstime",
985 &urgency_notification_assert_abstime_threshold,
986 sizeof(urgency_notification_assert_abstime_threshold));
987
988 if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD)) {
989 nx_enabled = 0;
990 }
991
992 /*
993 * VM initialization, after this we're using page tables...
994 * Thn maximum number of cpus must be set beforehand.
995 */
996 kernel_debug_string_early("i386_vm_init");
997 i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
998
999 /* create the console for verbose or pretty mode */
1000 /* Note: doing this prior to tsc_init() allows for graceful panic! */
1001 PE_init_platform(TRUE, kernelBootArgs);
1002 PE_create_console();
1003
1004 /* set %gs early so that power management can use locks */
1005 thread_t thread = thread_bootstrap();
1006 machine_set_current_thread(thread);
1007
1008 kernel_debug_string_early("power_management_init");
1009 power_management_init();
1010 xcpm_bootstrap();
1011
1012 #if CONFIG_CPU_COUNTERS
1013 mt_cpu_up(cpu_datap(0));
1014 #endif /* CONFIG_CPU_COUNTERS */
1015
1016 processor_bootstrap();
1017
1018 pstate_trace();
1019 kernel_debug_string_early("machine_startup");
1020 machine_startup();
1021 pstate_trace();
1022 }
1023
1024 static void __dead2
do_init_slave(boolean_t fast_restart)1025 do_init_slave(boolean_t fast_restart)
1026 {
1027 void *init_param = FULL_SLAVE_INIT;
1028
1029 postcode(I386_INIT_SLAVE);
1030
1031 if (!fast_restart) {
1032 /* Ensure that caching and write-through are enabled */
1033 set_cr0(get_cr0() & ~(CR0_NW | CR0_CD));
1034
1035 DBG("i386_init_slave() CPU%d: phys (%d) active.\n",
1036 get_cpu_number(), get_cpu_phys_number());
1037
1038 assert(!ml_get_interrupts_enabled());
1039
1040 cpu_syscall_init(current_cpu_datap());
1041 pmap_cpu_init();
1042
1043 #if CONFIG_MCA
1044 mca_cpu_init();
1045 #endif
1046
1047 LAPIC_INIT();
1048 /*
1049 * Note that the true argument here does not necessarily mean we're
1050 * here from a resume (this code path is also executed on boot).
1051 * The implementation of lapic_configure checks to see if the
1052 * state variable has been initialized, as it would be before
1053 * sleep. If it has not been, it's construed as an indicator of
1054 * first boot.
1055 */
1056 lapic_configure(true);
1057 LAPIC_DUMP();
1058 LAPIC_CPU_MAP_DUMP();
1059
1060 init_fpu();
1061
1062 #if CONFIG_MTRR
1063 mtrr_update_cpu();
1064 #endif
1065 /* update CPU microcode and apply CPU workarounds */
1066 ucode_update_wake_and_apply_cpu_was();
1067
1068 /* Enable LBRs on non-boot CPUs */
1069 i386_lbr_init(cpuid_info(), false);
1070 } else {
1071 init_param = FAST_SLAVE_INIT;
1072 }
1073
1074 #if CONFIG_VMX
1075 /* resume VT operation */
1076 vmx_resume(FALSE);
1077 #endif
1078
1079 #if CONFIG_MTRR
1080 if (!fast_restart) {
1081 pat_init();
1082 }
1083 #endif
1084
1085 cpu_thread_init(); /* not strictly necessary */
1086
1087 cpu_init(); /* Sets cpu_running which starter cpu waits for */
1088
1089
1090 #if CONFIG_CPU_COUNTERS
1091 mt_cpu_up(current_cpu_datap());
1092 #endif /* CONFIG_CPU_COUNTERS */
1093
1094 #if KPERF
1095 /*
1096 * We can only directly invoke kptimer_curcpu_up() when there is already an
1097 * active thread (that is, that this CPU has already been started at some point),
1098 * otherwise the ktrace calls within the kptimer operations will try to deref
1099 * the current thread and will instead cause a system reset.
1100 * If this is the first time the CPU is being started, we don't need to call
1101 * kptimer_curcpu_up().
1102 */
1103 if (current_processor()->active_thread != THREAD_NULL) {
1104 kptimer_curcpu_up();
1105 }
1106 #endif /* KPERF */
1107
1108 secondary_cpu_main(init_param);
1109
1110 panic("do_init_slave() returned from secondary_cpu_main()");
1111 }
1112
1113 /*
1114 * i386_init_slave() is called from pstart.
1115 * We're in the cpu's interrupt stack with interrupts disabled.
1116 * At this point we are in legacy mode. We need to switch on IA32e
1117 * if the mode is set to 64-bits.
1118 */
1119 void
i386_init_slave(void)1120 i386_init_slave(void)
1121 {
1122 do_init_slave(FALSE);
1123 }
1124
1125 /*
1126 * i386_init_slave_fast() is called from pmCPUHalt.
1127 * We're running on the idle thread and need to fix up
1128 * some accounting and get it so that the scheduler sees this
1129 * CPU again.
1130 */
1131 void
i386_init_slave_fast(void)1132 i386_init_slave_fast(void)
1133 {
1134 do_init_slave(TRUE);
1135 }
1136
1137
1138 /* TODO: Evaluate global PTEs for the double-mapped translations */
1139
1140 uint64_t dblmap_base, dblmap_max;
1141 kernel_segment_command_t *hdescseg;
1142
1143 pt_entry_t *dblmapL3;
1144 unsigned int dblallocs;
1145 uint64_t dblmap_dist;
1146 extern uint64_t idt64_hndl_table0[];
1147
1148
1149 void
doublemap_init(uint8_t randL3)1150 doublemap_init(uint8_t randL3)
1151 {
1152 dblmapL3 = ALLOCPAGES(1); // for 512 1GiB entries
1153 dblallocs++;
1154
1155 struct {
1156 pt_entry_t entries[PTE_PER_PAGE];
1157 } * dblmapL2 = ALLOCPAGES(1); // for 512 2MiB entries
1158 dblallocs++;
1159
1160 dblmapL3[randL3] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0]))
1161 | INTEL_PTE_VALID
1162 | INTEL_PTE_WRITE;
1163
1164 hdescseg = getsegbynamefromheader(&_mh_execute_header, "__HIB");
1165
1166 vm_offset_t hdescb = hdescseg->vmaddr;
1167 unsigned long hdescsz = hdescseg->vmsize;
1168 unsigned long hdescszr = round_page_64(hdescsz);
1169 vm_offset_t hdescc = hdescb, hdesce = hdescb + hdescszr;
1170
1171 kernel_section_t *thdescsect = getsectbynamefromheader(&_mh_execute_header, "__HIB", "__text");
1172 vm_offset_t thdescb = thdescsect->addr;
1173 unsigned long thdescsz = thdescsect->size;
1174 unsigned long thdescszr = round_page_64(thdescsz);
1175 vm_offset_t thdesce = thdescb + thdescszr;
1176
1177 assert((hdescb & 0xFFF) == 0);
1178 /* Mirror HIB translations into the double-mapped pagetable subtree*/
1179 for (int i = 0; hdescc < hdesce; i++) {
1180 struct {
1181 pt_entry_t entries[PTE_PER_PAGE];
1182 } * dblmapL1 = ALLOCPAGES(1);
1183 dblallocs++;
1184 dblmapL2[0].entries[i] = ((uintptr_t)ID_MAP_VTOP(&dblmapL1[0])) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
1185 int hdescn = (int) ((hdesce - hdescc) / PAGE_SIZE);
1186 for (int j = 0; j < MIN(PTE_PER_PAGE, hdescn); j++) {
1187 uint64_t template = INTEL_PTE_VALID;
1188 if ((hdescc >= thdescb) && (hdescc < thdesce)) {
1189 /* executable */
1190 } else {
1191 template |= INTEL_PTE_WRITE | INTEL_PTE_NX; /* Writeable, NX */
1192 }
1193 dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template;
1194 hdescc += PAGE_SIZE;
1195 }
1196 }
1197
1198 IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
1199
1200 dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, randL3, 0, 0);
1201 dblmap_max = dblmap_base + hdescszr;
1202 /* Calculate the double-map distance, which accounts for the current
1203 * KASLR slide
1204 */
1205
1206 dblmap_dist = dblmap_base - hdescb;
1207 idt64_hndl_table0[1] = DBLMAP(idt64_hndl_table0[1]); /* 64-bit exit trampoline */
1208 idt64_hndl_table0[3] = DBLMAP(idt64_hndl_table0[3]); /* 32-bit exit trampoline */
1209 idt64_hndl_table0[6] = (uint64_t)(uintptr_t)&kernel_stack_mask;
1210
1211 extern cpu_data_t cpshadows[], scdatas[];
1212 uintptr_t cd1 = (uintptr_t) &cpshadows[0];
1213 uintptr_t cd2 = (uintptr_t) &scdatas[0];
1214 /* Record the displacement from the kernel's per-CPU data pointer, eventually
1215 * programmed into GSBASE, to the "shadows" in the doublemapped
1216 * region. These are not aliases, but separate physical allocations
1217 * containing data required in the doublemapped trampolines.
1218 */
1219 idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2;
1220
1221 DBG("Double map base: 0x%qx\n", dblmap_base);
1222 DBG("double map idlepml4[%d]: 0x%llx\n", KERNEL_DBLMAP_PML4_INDEX, IdlePML4[KERNEL_DBLMAP_PML4_INDEX]);
1223 assert(LDTSZ > LDTSZ_MIN);
1224 }
1225
1226 vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t);
1227
1228 #include <i386/pmap_internal.h>
1229
1230 /* Use of this routine is expected to be synchronized by callers
1231 * Creates non-executable aliases.
1232 */
1233 vm_offset_t
dyn_dblmap(vm_offset_t cva,vm_offset_t sz)1234 dyn_dblmap(vm_offset_t cva, vm_offset_t sz)
1235 {
1236 vm_offset_t ava = dblmap_max;
1237
1238 assert((sz & PAGE_MASK) == 0);
1239 assert(cva != 0);
1240
1241 pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP);
1242 dblmap_max += sz;
1243 return ava - cva;
1244 }
1245 /* Adjust offsets interior to the bootstrap interrupt descriptor table to redirect
1246 * control to the double-mapped interrupt vectors. The IDTR proper will be
1247 * programmed via cpu_desc_load()
1248 */
1249 void
idt64_remap(void)1250 idt64_remap(void)
1251 {
1252 for (int i = 0; i < IDTSZ; i++) {
1253 master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64);
1254 }
1255 }
1256