1 /*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 /*
58 */
59
60 #include <kern/cpu_number.h>
61 #include <kern/cpu_data.h>
62 #include <kern/percpu.h>
63 #include <kern/monotonic.h>
64 #include <kern/misc_protos.h>
65 #include <mach/mach_types.h>
66 #include <mach/machine.h>
67 #include <mach/vm_map.h>
68 #include <mach/machine/vm_param.h>
69 #include <vm/vm_kern.h>
70 #include <vm/vm_map.h>
71 #include <san/kasan.h>
72
73 #include <i386/bit_routines.h>
74 #include <i386/mp_desc.h>
75 #include <i386/misc_protos.h>
76 #include <i386/mp.h>
77 #include <i386/pmap.h>
78 #include <i386/postcode.h>
79 #include <i386/pmap_internal.h>
80 #if CONFIG_MCA
81 #include <i386/machine_check.h>
82 #endif
83
84 #define K_INTR_GATE (ACC_P|ACC_PL_K|ACC_INTR_GATE)
85 #define U_INTR_GATE (ACC_P|ACC_PL_U|ACC_INTR_GATE)
86
87 // Declare macros that will declare the externs
88 #define TRAP(n, name) extern void *name ;
89 #define TRAP_ERR(n, name) extern void *name ;
90 #define TRAP_SPC(n, name) extern void *name ;
91 #define TRAP_IST1(n, name) extern void *name ;
92 #define TRAP_IST2(n, name) extern void *name ;
93 #define INTERRUPT(n) extern void *_intr_ ## n ;
94 #define USER_TRAP(n, name) extern void *name ;
95 #define USER_TRAP_SPC(n, name) extern void *name ;
96
97 // Include the table to declare the externs
98 #include "../x86_64/idt_table.h"
99
100 // Undef the macros, then redefine them so we can declare the table
101 #undef TRAP
102 #undef TRAP_ERR
103 #undef TRAP_SPC
104 #undef TRAP_IST1
105 #undef TRAP_IST2
106 #undef INTERRUPT
107 #undef USER_TRAP
108 #undef USER_TRAP_SPC
109
110 #define TRAP(n, name) \
111 [n] = { \
112 (uintptr_t)&name, \
113 KERNEL64_CS, \
114 0, \
115 K_INTR_GATE, \
116 0 \
117 },
118
119 #define TRAP_ERR TRAP
120 #define TRAP_SPC TRAP
121
122 #define TRAP_IST1(n, name) \
123 [n] = { \
124 (uintptr_t)&name, \
125 KERNEL64_CS, \
126 1, \
127 K_INTR_GATE, \
128 0 \
129 },
130
131 #define TRAP_IST2(n, name) \
132 [n] = { \
133 (uintptr_t)&name, \
134 KERNEL64_CS, \
135 2, \
136 K_INTR_GATE, \
137 0 \
138 },
139
140 #define INTERRUPT(n) \
141 [n] = { \
142 (uintptr_t)&_intr_ ## n,\
143 KERNEL64_CS, \
144 0, \
145 K_INTR_GATE, \
146 0 \
147 },
148
149 #define USER_TRAP(n, name) \
150 [n] = { \
151 (uintptr_t)&name, \
152 KERNEL64_CS, \
153 0, \
154 U_INTR_GATE, \
155 0 \
156 },
157
158 #define USER_TRAP_SPC USER_TRAP
159
160 // Declare the table using the macros we just set up
161 struct fake_descriptor64 master_idt64[IDTSZ]
162 __attribute__ ((section("__HIB,__desc")))
163 __attribute__ ((aligned(PAGE_SIZE))) = {
164 #include "../x86_64/idt_table.h"
165 };
166
167 /*
168 * First cpu`s interrupt stack.
169 */
170 extern uint32_t low_intstack[]; /* bottom */
171 extern uint32_t low_eintstack[]; /* top */
172
173 /*
174 * Per-cpu data area pointers.
175 */
176 cpu_data_t cpshadows[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc")));
177 cpu_data_t scdatas[MAX_CPUS] __attribute__((aligned(64))) = {
178 [0].cpu_this = &scdatas[0],
179 [0].cpu_nanotime = &pal_rtc_nanotime_info,
180 [0].cpu_int_stack_top = (vm_offset_t) low_eintstack,
181 [0].cd_shadow = &cpshadows[0]
182 };
183 cpu_data_t *cpu_data_master = &scdatas[0];
184
185 cpu_data_t *cpu_data_ptr[MAX_CPUS] = {[0] = &scdatas[0] };
186
187 SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base;
188
189 decl_simple_lock_data(, ncpus_lock); /* protects real_ncpus */
190 unsigned int real_ncpus = 1;
191 unsigned int max_ncpus = MAX_CPUS;
192 unsigned int max_cpus_from_firmware = 0;
193
194 extern void hi64_sysenter(void);
195 extern void hi64_syscall(void);
196
197 typedef struct {
198 struct real_descriptor pcldts[LDTSZ];
199 } cldt_t;
200
201 cpu_desc_table64_t scdtables[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc")));
202 cpu_fault_stack_t scfstks[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc")));
203
204 cldt_t *dyn_ldts;
205
206 /*
207 * Multiprocessor i386/i486 systems use a separate copy of the
208 * GDT, IDT, LDT, and kernel TSS per processor. The first three
209 * are separate to avoid lock contention: the i386 uses locked
210 * memory cycles to access the descriptor tables. The TSS is
211 * separate since each processor needs its own kernel stack,
212 * and since using a TSS marks it busy.
213 */
214
215 /*
216 * Allocate and initialize the per-processor descriptor tables.
217 */
218
219 /*
220 * This is the expanded, 64-bit variant of the kernel LDT descriptor.
221 * When switching to 64-bit mode this replaces KERNEL_LDT entry
222 * and the following empty slot. This enables the LDT to be referenced
223 * in the uber-space remapping window on the kernel.
224 */
225 struct fake_descriptor64 kernel_ldt_desc64 = {
226 .offset64 = 0,
227 .lim_or_seg = LDTSZ_MIN * sizeof(struct fake_descriptor) - 1,
228 .size_or_IST = 0,
229 .access = ACC_P | ACC_PL_K | ACC_LDT,
230 .reserved = 0
231 };
232
233 /*
234 * This is the expanded, 64-bit variant of the kernel TSS descriptor.
235 * It is follows pattern of the KERNEL_LDT.
236 */
237 struct fake_descriptor64 kernel_tss_desc64 = {
238 .offset64 = 0,
239 .lim_or_seg = sizeof(struct x86_64_tss) - 1,
240 .size_or_IST = 0,
241 .access = ACC_P | ACC_PL_K | ACC_TSS,
242 .reserved = 0
243 };
244
245 /*
246 * Convert a descriptor from fake to real format.
247 *
248 * Fake descriptor format:
249 * bytes 0..3 base 31..0
250 * bytes 4..5 limit 15..0
251 * byte 6 access byte 2 | limit 19..16
252 * byte 7 access byte 1
253 *
254 * Real descriptor format:
255 * bytes 0..1 limit 15..0
256 * bytes 2..3 base 15..0
257 * byte 4 base 23..16
258 * byte 5 access byte 1
259 * byte 6 access byte 2 | limit 19..16
260 * byte 7 base 31..24
261 *
262 * Fake gate format:
263 * bytes 0..3 offset
264 * bytes 4..5 selector
265 * byte 6 word count << 4 (to match fake descriptor)
266 * byte 7 access byte 1
267 *
268 * Real gate format:
269 * bytes 0..1 offset 15..0
270 * bytes 2..3 selector
271 * byte 4 word count
272 * byte 5 access byte 1
273 * bytes 6..7 offset 31..16
274 */
275 void
fix_desc(void * d,int num_desc)276 fix_desc(void *d, int num_desc)
277 {
278 uint8_t *desc = (uint8_t*) d;
279
280 do {
281 if ((desc[7] & 0x14) == 0x04) { /* gate */
282 uint32_t offset;
283 uint16_t selector;
284 uint8_t wordcount;
285 uint8_t acc;
286
287 offset = *((uint32_t*)(desc));
288 selector = *((uint32_t*)(desc + 4));
289 wordcount = desc[6] >> 4;
290 acc = desc[7];
291
292 *((uint16_t*)desc) = offset & 0xFFFF;
293 *((uint16_t*)(desc + 2)) = selector;
294 desc[4] = wordcount;
295 desc[5] = acc;
296 *((uint16_t*)(desc + 6)) = offset >> 16;
297 } else { /* descriptor */
298 uint32_t base;
299 uint16_t limit;
300 uint8_t acc1, acc2;
301
302 base = *((uint32_t*)(desc));
303 limit = *((uint16_t*)(desc + 4));
304 acc2 = desc[6];
305 acc1 = desc[7];
306
307 *((uint16_t*)(desc)) = limit;
308 *((uint16_t*)(desc + 2)) = base & 0xFFFF;
309 desc[4] = (base >> 16) & 0xFF;
310 desc[5] = acc1;
311 desc[6] = acc2;
312 desc[7] = base >> 24;
313 }
314 desc += 8;
315 } while (--num_desc);
316 }
317
318 void
fix_desc64(void * descp,int count)319 fix_desc64(void *descp, int count)
320 {
321 struct fake_descriptor64 *fakep;
322 union {
323 struct real_gate64 gate;
324 struct real_descriptor64 desc;
325 } real;
326 int i;
327
328 fakep = (struct fake_descriptor64 *) descp;
329
330 for (i = 0; i < count; i++, fakep++) {
331 /*
332 * Construct the real decriptor locally.
333 */
334
335 bzero((void *) &real, sizeof(real));
336
337 switch (fakep->access & ACC_TYPE) {
338 case 0:
339 break;
340 case ACC_CALL_GATE:
341 case ACC_INTR_GATE:
342 case ACC_TRAP_GATE:
343 real.gate.offset_low16 = (uint16_t)(fakep->offset64 & 0xFFFF);
344 real.gate.selector16 = fakep->lim_or_seg & 0xFFFF;
345 real.gate.IST = fakep->size_or_IST & 0x7;
346 real.gate.access8 = fakep->access;
347 real.gate.offset_high16 = (uint16_t)((fakep->offset64 >> 16) & 0xFFFF);
348 real.gate.offset_top32 = (uint32_t)(fakep->offset64 >> 32);
349 break;
350 default: /* Otherwise */
351 real.desc.limit_low16 = fakep->lim_or_seg & 0xFFFF;
352 real.desc.base_low16 = (uint16_t)(fakep->offset64 & 0xFFFF);
353 real.desc.base_med8 = (uint8_t)((fakep->offset64 >> 16) & 0xFF);
354 real.desc.access8 = fakep->access;
355 real.desc.limit_high4 = (fakep->lim_or_seg >> 16) & 0xFF;
356 real.desc.granularity4 = fakep->size_or_IST;
357 real.desc.base_high8 = (uint8_t)((fakep->offset64 >> 24) & 0xFF);
358 real.desc.base_top32 = (uint32_t)(fakep->offset64 >> 32);
359 }
360
361 /*
362 * Now copy back over the fake structure.
363 */
364 bcopy((void *) &real, (void *) fakep, sizeof(real));
365 }
366 }
367
368 extern unsigned mldtsz;
369 void
cpu_desc_init(cpu_data_t * cdp)370 cpu_desc_init(cpu_data_t *cdp)
371 {
372 cpu_desc_index_t *cdi = &cdp->cpu_desc_index;
373
374 if (cdp == cpu_data_master) {
375 /*
376 * Populate the double-mapped 'u' and base 'b' fields in the
377 * KTSS with I/G/LDT and sysenter stack data.
378 */
379 cdi->cdi_ktssu = (void *)DBLMAP(&master_ktss64);
380 cdi->cdi_ktssb = (void *)&master_ktss64;
381 cdi->cdi_sstku = (vm_offset_t) DBLMAP(&master_sstk.top);
382 cdi->cdi_sstkb = (vm_offset_t) &master_sstk.top;
383
384 cdi->cdi_gdtu.ptr = (void *)DBLMAP((uintptr_t) &master_gdt);
385 cdi->cdi_gdtb.ptr = (void *)&master_gdt;
386 cdi->cdi_idtu.ptr = (void *)DBLMAP((uintptr_t) &master_idt64);
387 cdi->cdi_idtb.ptr = (void *)((uintptr_t) &master_idt64);
388 cdi->cdi_ldtu = (struct real_descriptor *)DBLMAP((uintptr_t)&master_ldt[0]);
389 cdi->cdi_ldtb = &master_ldt[0];
390
391 /* Replace the expanded LDTs and TSS slots in the GDT */
392 kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu;
393 *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
394 kernel_ldt_desc64;
395 *(struct fake_descriptor64 *) &master_gdt[sel_idx(USER_LDT)] =
396 kernel_ldt_desc64;
397 kernel_tss_desc64.offset64 = (uintptr_t) DBLMAP(&master_ktss64);
398 *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
399 kernel_tss_desc64;
400
401 /* Fix up the expanded descriptors for 64-bit. */
402 fix_desc64((void *) &master_idt64, IDTSZ);
403 fix_desc64((void *) &master_gdt[sel_idx(KERNEL_LDT)], 1);
404 fix_desc64((void *) &master_gdt[sel_idx(USER_LDT)], 1);
405 fix_desc64((void *) &master_gdt[sel_idx(KERNEL_TSS)], 1);
406
407 /*
408 * Set the NMI/fault stacks as IST2/IST1 in the 64-bit TSS
409 */
410 master_ktss64.ist2 = (uintptr_t) low_eintstack;
411 master_ktss64.ist1 = (uintptr_t) low_eintstack - sizeof(x86_64_intr_stack_frame_t);
412 } else if (cdi->cdi_ktssu == NULL) { /* Skipping re-init on wake */
413 cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;
414
415 cdi->cdi_idtu.ptr = (void *)DBLMAP((uintptr_t) &master_idt64);
416
417 cdi->cdi_ktssu = (void *)DBLMAP(&cdt->ktss);
418 cdi->cdi_ktssb = (void *)(&cdt->ktss);
419 cdi->cdi_sstku = (vm_offset_t)DBLMAP(&cdt->sstk.top);
420 cdi->cdi_sstkb = (vm_offset_t)(&cdt->sstk.top);
421 cdi->cdi_ldtu = (void *)LDTALIAS(cdp->cpu_ldtp);
422 cdi->cdi_ldtb = (void *)(cdp->cpu_ldtp);
423
424 /*
425 * Copy the tables
426 */
427 bcopy((char *)master_gdt, (char *)cdt->gdt, sizeof(master_gdt));
428 bcopy((char *)master_ldt, (char *)cdp->cpu_ldtp, mldtsz);
429 bcopy((char *)&master_ktss64, (char *)&cdt->ktss, sizeof(struct x86_64_tss));
430 cdi->cdi_gdtu.ptr = (void *)DBLMAP(cdt->gdt);
431 cdi->cdi_gdtb.ptr = (void *)(cdt->gdt);
432 /*
433 * Fix up the entries in the GDT to point to
434 * this LDT and this TSS.
435 * Note reuse of global 'kernel_ldt_desc64, which is not
436 * concurrency-safe. Higher level synchronization is expected
437 */
438 kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu;
439 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
440 kernel_ldt_desc64;
441 fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
442
443 kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu;
444 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
445 kernel_ldt_desc64;
446 fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
447
448 kernel_tss_desc64.offset64 = (uintptr_t) cdi->cdi_ktssu;
449 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
450 kernel_tss_desc64;
451 fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
452
453 /* Set (zeroed) fault stack as IST1, NMI intr stack IST2 */
454 uint8_t *cfstk = &scfstks[cdp->cpu_number].fstk[0];
455 cdt->fstkp = cfstk;
456 bzero((void *) cfstk, FSTK_SZ);
457 cdt->ktss.ist2 = DBLMAP((uint64_t)cdt->fstkp + FSTK_SZ);
458 cdt->ktss.ist1 = cdt->ktss.ist2 - sizeof(x86_64_intr_stack_frame_t);
459 }
460
461 /* Require that the top of the sysenter stack is 16-byte aligned */
462 if ((cdi->cdi_sstku % 16) != 0) {
463 panic("cpu_desc_init() sysenter stack not 16-byte aligned");
464 }
465 }
466 void
cpu_desc_load(cpu_data_t * cdp)467 cpu_desc_load(cpu_data_t *cdp)
468 {
469 cpu_desc_index_t *cdi = &cdp->cpu_desc_index;
470
471 postcode(CPU_DESC_LOAD_ENTRY);
472
473 /* Stuff the kernel per-cpu data area address into the MSRs */
474 postcode(CPU_DESC_LOAD_GS_BASE);
475 wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
476 postcode(CPU_DESC_LOAD_KERNEL_GS_BASE);
477 wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
478
479 /*
480 * Ensure the TSS segment's busy bit is clear. This is required
481 * for the case of reloading descriptors at wake to avoid
482 * their complete re-initialization.
483 */
484 gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY;
485
486 /* Load the GDT, LDT, IDT and TSS */
487 cdi->cdi_gdtb.size = sizeof(struct real_descriptor) * GDTSZ - 1;
488 cdi->cdi_gdtu.size = cdi->cdi_gdtb.size;
489 cdi->cdi_idtb.size = 0x1000 + cdp->cpu_number;
490 cdi->cdi_idtu.size = cdi->cdi_idtb.size;
491
492 postcode(CPU_DESC_LOAD_GDT);
493 lgdt((uintptr_t *) &cdi->cdi_gdtu);
494 postcode(CPU_DESC_LOAD_IDT);
495 lidt((uintptr_t *) &cdi->cdi_idtu);
496 postcode(CPU_DESC_LOAD_LDT);
497 lldt(KERNEL_LDT);
498 postcode(CPU_DESC_LOAD_TSS);
499 set_tr(KERNEL_TSS);
500
501 postcode(CPU_DESC_LOAD_EXIT);
502 }
503
504 /*
505 * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit.
506 */
507 void
cpu_syscall_init(cpu_data_t * cdp)508 cpu_syscall_init(cpu_data_t *cdp)
509 {
510 #pragma unused(cdp)
511
512 wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS);
513 wrmsr64(MSR_IA32_SYSENTER_EIP, DBLMAP((uintptr_t) hi64_sysenter));
514 wrmsr64(MSR_IA32_SYSENTER_ESP, current_cpu_datap()->cpu_desc_index.cdi_sstku);
515 /* Enable syscall/sysret */
516 wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE);
517
518 /*
519 * MSRs for 64-bit syscall/sysret
520 * Note USER_CS because sysret uses this + 16 when returning to
521 * 64-bit code.
522 */
523 wrmsr64(MSR_IA32_LSTAR, DBLMAP((uintptr_t) hi64_syscall));
524 wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) | (((uint64_t)KERNEL64_CS) << 32));
525 /*
526 * Emulate eflags cleared by sysenter but note that
527 * we also clear the trace trap to avoid the complications
528 * of single-stepping into a syscall. The nested task bit
529 * is also cleared to avoid a spurious "task switch"
530 * should we choose to return via an IRET.
531 */
532 wrmsr64(MSR_IA32_FMASK, EFL_DF | EFL_IF | EFL_TF | EFL_NT);
533 }
534 extern vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t);
535 uint64_t ldt_alias_offset;
536
537 __startup_func
538 static void
cpu_data_startup_init(void)539 cpu_data_startup_init(void)
540 {
541 int flags = KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_PERMANENT |
542 KMA_ZERO | KMA_KOBJECT | KMA_NOFAIL;
543 uint32_t cpus = max_cpus_from_firmware;
544 vm_size_t size = percpu_section_size() * (cpus - 1);
545
546 percpu_base.size = percpu_section_size();
547 if (cpus == 0) {
548 panic("percpu: max_cpus_from_firmware not yet initialized");
549 }
550 if (cpus == 1) {
551 percpu_base.start = VM_MAX_KERNEL_ADDRESS;
552 return;
553 }
554
555 kmem_alloc(kernel_map, &percpu_base.start,
556 round_page(size) + ptoa(2), flags, VM_KERN_MEMORY_CPU);
557
558 percpu_base.start += PAGE_SIZE - percpu_section_start();
559 percpu_base.end = percpu_base.start + size - 1;
560 }
561 STARTUP(PERCPU, STARTUP_RANK_FIRST, cpu_data_startup_init);
562
563 cpu_data_t *
cpu_data_alloc(boolean_t is_boot_cpu)564 cpu_data_alloc(boolean_t is_boot_cpu)
565 {
566 cpu_data_t *cdp;
567
568 if (is_boot_cpu) {
569 assert(real_ncpus == 1);
570 cdp = cpu_datap(0);
571 if (cdp->cpu_processor == NULL) {
572 simple_lock_init(&ncpus_lock, 0);
573 cdp->cpu_processor = PERCPU_GET_MASTER(processor);
574 }
575 return cdp;
576 }
577
578 boolean_t do_ldt_alloc = FALSE;
579 simple_lock(&ncpus_lock, LCK_GRP_NULL);
580 int cnum = real_ncpus;
581 real_ncpus++;
582 if (dyn_ldts == NULL) {
583 do_ldt_alloc = TRUE;
584 }
585 simple_unlock(&ncpus_lock);
586
587 /*
588 * Allocate per-cpu data:
589 */
590
591 cdp = &scdatas[cnum];
592 bzero((void*) cdp, sizeof(cpu_data_t));
593 cdp->cpu_this = cdp;
594 cdp->cpu_number = cnum;
595 cdp->cd_shadow = &cpshadows[cnum];
596 cdp->cpu_pcpu_base = percpu_base.start + (cnum - 1) * percpu_section_size();
597 cdp->cpu_processor = PERCPU_GET_WITH_BASE(cdp->cpu_pcpu_base, processor);
598
599 /*
600 * Allocate interrupt stack:
601 */
602 kmem_alloc(kernel_map, (vm_offset_t *)&cdp->cpu_int_stack_top,
603 INTSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
604 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KOBJECT, VM_KERN_MEMORY_CPU);
605 cdp->cpu_int_stack_top += INTSTACK_SIZE + PAGE_SIZE;
606
607 /*
608 * Allocate descriptor table:
609 */
610
611 cdp->cpu_desc_tablep = (struct cpu_desc_table *) &scdtables[cnum];
612 /*
613 * Allocate LDT
614 */
615 if (do_ldt_alloc) {
616 boolean_t do_ldt_free = FALSE;
617 vm_offset_t sldtoffset = 0;
618 /*
619 * Allocate LDT
620 */
621 vm_offset_t ldtalloc = 0, ldtallocsz = round_page_64(MAX_CPUS * sizeof(struct real_descriptor) * LDTSZ);
622
623 kmem_alloc(kernel_map, (vm_offset_t *)&ldtalloc,
624 ldtallocsz, KMA_NOFAIL | KMA_KOBJECT, VM_KERN_MEMORY_CPU);
625
626 simple_lock(&ncpus_lock, LCK_GRP_NULL);
627 if (dyn_ldts == NULL) {
628 dyn_ldts = (cldt_t *)ldtalloc;
629 } else {
630 do_ldt_free = TRUE;
631 }
632 simple_unlock(&ncpus_lock);
633
634 if (do_ldt_free) {
635 kmem_free(kernel_map, ldtalloc, ldtallocsz);
636 } else {
637 /* CPU registration and startup are expected to execute
638 * serially, as invoked by the platform driver.
639 * Create trampoline alias of LDT region.
640 */
641 sldtoffset = dyn_dblmap(ldtalloc, ldtallocsz);
642 ldt_alias_offset = sldtoffset;
643 }
644 }
645 cdp->cpu_ldtp = &dyn_ldts[cnum].pcldts[0];
646
647 #if CONFIG_MCA
648 /* Machine-check shadow register allocation. */
649 mca_cpu_alloc(cdp);
650 #endif
651
652 /*
653 * Before this cpu has been assigned a real thread context,
654 * we give it a fake, unique, non-zero thread id which the locking
655 * primitives use as their lock value.
656 * Note that this does not apply to the boot processor, cpu 0, which
657 * transitions to a thread context well before other processors are
658 * started.
659 */
660 cdp->cpu_active_thread = (thread_t) (uintptr_t) cdp->cpu_number;
661 cdp->cpu_NMI_acknowledged = TRUE;
662 cdp->cpu_nanotime = &pal_rtc_nanotime_info;
663
664 kprintf("cpu_data_alloc(%d) %p desc_table: %p "
665 "ldt: %p "
666 "int_stack: 0x%lx-0x%lx\n",
667 cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp,
668 (long)(cdp->cpu_int_stack_top - INTSTACK_SIZE), (long)(cdp->cpu_int_stack_top));
669 cpu_data_ptr[cnum] = cdp;
670
671 return cdp;
672 }
673
674 boolean_t
valid_user_data_selector(uint16_t selector)675 valid_user_data_selector(uint16_t selector)
676 {
677 sel_t sel = selector_to_sel(selector);
678
679 if (selector == 0) {
680 return TRUE;
681 }
682
683 if (sel.ti == SEL_LDT) {
684 return TRUE;
685 } else if (sel.index < GDTSZ) {
686 if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U) {
687 return TRUE;
688 }
689 }
690 return FALSE;
691 }
692
693 boolean_t
valid_user_code_selector(uint16_t selector)694 valid_user_code_selector(uint16_t selector)
695 {
696 sel_t sel = selector_to_sel(selector);
697
698 if (selector == 0) {
699 return FALSE;
700 }
701
702 if (sel.ti == SEL_LDT) {
703 if (sel.rpl == USER_PRIV) {
704 return TRUE;
705 }
706 } else if (sel.index < GDTSZ && sel.rpl == USER_PRIV) {
707 if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U) {
708 return TRUE;
709 }
710 /* Explicitly validate the system code selectors
711 * even if not instantaneously privileged,
712 * since they are dynamically re-privileged
713 * at context switch
714 */
715 if ((selector == USER_CS) || (selector == USER64_CS)) {
716 return TRUE;
717 }
718 }
719
720 return FALSE;
721 }
722
723 boolean_t
valid_user_stack_selector(uint16_t selector)724 valid_user_stack_selector(uint16_t selector)
725 {
726 sel_t sel = selector_to_sel(selector);
727
728 if (selector == 0) {
729 return FALSE;
730 }
731
732 if (sel.ti == SEL_LDT) {
733 if (sel.rpl == USER_PRIV) {
734 return TRUE;
735 }
736 } else if (sel.index < GDTSZ && sel.rpl == USER_PRIV) {
737 if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U) {
738 return TRUE;
739 }
740 }
741
742 return FALSE;
743 }
744
745 boolean_t
valid_user_segment_selectors(uint16_t cs,uint16_t ss,uint16_t ds,uint16_t es,uint16_t fs,uint16_t gs)746 valid_user_segment_selectors(uint16_t cs,
747 uint16_t ss,
748 uint16_t ds,
749 uint16_t es,
750 uint16_t fs,
751 uint16_t gs)
752 {
753 return valid_user_code_selector(cs) &&
754 valid_user_stack_selector(ss) &&
755 valid_user_data_selector(ds) &&
756 valid_user_data_selector(es) &&
757 valid_user_data_selector(fs) &&
758 valid_user_data_selector(gs);
759 }
760
761 /*
762 * Allocate a new interrupt stack for the boot processor from the
763 * heap rather than continue to use the statically allocated space.
764 * Also switch to a dynamically allocated cpu data area.
765 */
766 void
cpu_data_realloc(void)767 cpu_data_realloc(void)
768 {
769 vm_offset_t istk;
770 cpu_data_t *cdp;
771 boolean_t istate;
772
773 kmem_alloc(kernel_map, &istk,
774 INTSTACK_SIZE + ptoa(2), KMA_NOFAIL | KMA_PERMANENT | KMA_ZERO |
775 KMA_GUARD_FIRST | KMA_GUARD_LAST | KMA_KOBJECT, VM_KERN_MEMORY_CPU);
776
777 istk += INTSTACK_SIZE + PAGE_SIZE;
778
779 cdp = &scdatas[0];
780
781 /* Copy old contents into new area and make fix-ups */
782 assert(cpu_number() == 0);
783 bcopy((void *) cpu_data_ptr[0], (void*) cdp, sizeof(cpu_data_t));
784 cdp->cpu_this = cdp;
785 cdp->cpu_int_stack_top = istk;
786 timer_call_queue_init(&cdp->rtclock_timer.queue);
787 cdp->cpu_desc_tablep = (struct cpu_desc_table *) &scdtables[0];
788 cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;
789
790 uint8_t *cfstk = &scfstks[cdp->cpu_number].fstk[0];
791 cdt->fstkp = cfstk;
792 cfstk += FSTK_SZ;
793
794 /*
795 * With interrupts disabled commmit the new areas.
796 */
797 istate = ml_set_interrupts_enabled(FALSE);
798 cpu_data_ptr[0] = cdp;
799 master_ktss64.ist2 = DBLMAP((uintptr_t) cfstk);
800 master_ktss64.ist1 = DBLMAP((uintptr_t) cfstk - sizeof(x86_64_intr_stack_frame_t));
801 wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
802 wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
803 (void) ml_set_interrupts_enabled(istate);
804
805 kprintf("Reallocated master cpu data: %p,"
806 " interrupt stack: %p, fault stack: %p\n",
807 (void *) cdp, (void *) istk, (void *) cfstk);
808 }
809