1/* 2 * Copyright (c) 2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or [email protected] 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58 59#include <debug.h> 60 61#include <i386/asm.h> 62#include <i386/proc_reg.h> 63#include <i386/postcode.h> 64#include <assym.s> 65 66#include <i386/cpuid.h> 67#include <i386/acpi.h> 68 69.code32 70 71 72/* 73 * Interrupt and bootup stack for initial processor. 74 * Note: we switch to a dynamically allocated interrupt stack once VM is up. 75 */ 76 77/* in the __HIB section since the hibernate restore code uses this stack. */ 78 .section __HIB, __data 79 .align 12 80 81 .globl EXT(low_intstack) 82EXT(low_intstack): 83 .globl EXT(gIOHibernateRestoreStack) 84EXT(gIOHibernateRestoreStack): 85 86 .space INTSTACK_SIZE 87 88 .globl EXT(low_eintstack) 89EXT(low_eintstack:) 90 .globl EXT(gIOHibernateRestoreStackEnd) 91EXT(gIOHibernateRestoreStackEnd): 92 93 /* back to the regular __DATA section. */ 94 95 .section __DATA, __data 96 97/* 98 * Stack for machine-check handler. 99 */ 100 .align 12 101 .globl EXT(mc_task_stack) 102EXT(mc_task_stack): 103 .space INTSTACK_SIZE 104 .globl EXT(mc_task_stack_end) 105EXT(mc_task_stack_end): 106 107 /* Must not clobber EDI */ 108#define SWITCH_TO_64BIT_MODE \ 109 movl $(CR4_PAE),%eax /* enable PAE */ ;\ 110 movl %eax,%cr4 ;\ 111 movl $MSR_IA32_EFER,%ecx ;\ 112 rdmsr ;\ 113 /* enable long mode, NX */ ;\ 114 orl $(MSR_IA32_EFER_LME | MSR_IA32_EFER_NXE),%eax ;\ 115 wrmsr ;\ 116 movl $EXT(BootPML4),%eax ;\ 117 movl %eax,%cr3 ;\ 118 movl %cr0,%eax ;\ 119 orl $(CR0_PG|CR0_WP),%eax /* enable paging */ ;\ 120 movl %eax,%cr0 ;\ 121 ljmpl $KERNEL64_CS,$64f ;\ 12264: ;\ 123 .code64 124 125/* 126 * BSP CPU start here. 127 * eax points to kernbootstruct 128 * 129 * Environment: 130 * protected mode, no paging, flat 32-bit address space. 131 * (Code/data/stack segments have base == 0, limit == 4G) 132 */ 133 134.code32 135 .text 136 .section __HIB, __text 137 .align ALIGN 138 .globl EXT(_start) 139 .globl EXT(pstart) 140LEXT(_start) 141LEXT(pstart) 142 143/* 144 * Here we do the minimal setup to switch from 32 bit mode to 64 bit long mode. 145 * 146 * Initial memory layout: 147 * 148 * ------------------------- 149 * | | 150 * | Kernel text/data | 151 * | | 152 * |-----------------------| Kernel text base addr - 2MB-aligned 153 * | padding | 154 * |-----------------------| 155 * | __HIB section | 156 * |-----------------------| Page-aligned 157 * | | 158 * | padding | 159 * | | 160 * ------------------------- 0 161 * 162 */ 163 mov %eax, %edi /* save kernbootstruct */ 164 165 /* Use low 32-bits of address as 32-bit stack */ 166 movl $EXT(low_eintstack), %esp 167 168 POSTCODE(PSTART_ENTRY) 169 170 /* 171 * Set up segmentation 172 */ 173 movl $EXT(protected_mode_gdtr), %eax 174 lgdtl (%eax) 175 176 /* 177 * Rebase Boot page tables to kernel base address. 178 */ 179 movl $EXT(BootPML4), %eax // Level 4: 180 add %eax, 0*8+0(%eax) // - 1:1 181 add %eax, KERNEL_PML4_INDEX*8+0(%eax) // - kernel space 182 183 movl $EXT(BootPDPT), %edx // Level 3: 184 add %eax, 0*8+0(%edx) 185 add %eax, 1*8+0(%edx) 186 add %eax, 2*8+0(%edx) 187 add %eax, 3*8+0(%edx) 188 189 POSTCODE(PSTART_REBASE) 190 191/* the following code is shared by the master CPU and all slave CPUs */ 192L_pstart_common: 193 /* 194 * switch to 64 bit mode 195 */ 196 SWITCH_TO_64BIT_MODE 197 198 /* Flush data segment selectors */ 199 xor %eax, %eax 200 mov %ax, %ss 201 mov %ax, %ds 202 mov %ax, %es 203 mov %ax, %fs 204 mov %ax, %gs 205 206 test %edi, %edi /* Populate stack canary on BSP */ 207 jz Lvstartshim 208 209 mov $1, %eax 210 cpuid 211 test $(1 << 30), %ecx 212 jz Lnon_rdrand 213 rdrand %rax /* RAX := 64 bits of DRBG entropy */ 214 jnc Lnon_rdrand /* TODO: complain if DRBG fails at this stage */ 215 216Lstore_random_guard: 217 xor %ah, %ah /* Security: zero second byte of stack canary */ 218 movq %rax, ___stack_chk_guard(%rip) 219 /* %edi = boot_args_start if BSP */ 220Lvstartshim: 221 222 POSTCODE(PSTART_VSTART) 223 224 /* %edi = boot_args_start */ 225 226 leaq _vstart(%rip), %rcx 227 movq $(KERNEL_BASE), %rax /* adjust pointer up high */ 228 or %rax, %rsp /* and stack pointer up there */ 229 or %rcx, %rax 230 andq $0xfffffffffffffff0, %rsp /* align stack */ 231 xorq %rbp, %rbp /* zero frame pointer */ 232 callq *%rax 233 234Lnon_rdrand: 235 rdtsc /* EDX:EAX := TSC */ 236 /* Distribute low order bits */ 237 mov %eax, %ecx 238 xor %al, %ah 239 shl $16, %rcx 240 xor %rcx, %rax 241 xor %eax, %edx 242 243 /* Incorporate ASLR entropy, if any */ 244 lea (%rip), %rcx 245 shr $21, %rcx 246 movzbl %cl, %ecx 247 shl $16, %ecx 248 xor %ecx, %edx 249 250 mov %ah, %cl 251 ror %cl, %edx /* Right rotate EDX (TSC&0xFF ^ (TSC>>8 & 0xFF))&1F */ 252 shl $32, %rdx 253 xor %rdx, %rax 254 mov %cl, %al 255 jmp Lstore_random_guard 256/* 257 * AP (slave) CPUs enter here. 258 * 259 * Environment: 260 * protected mode, no paging, flat 32-bit address space. 261 * (Code/data/stack segments have base == 0, limit == 4G) 262 */ 263 .align ALIGN 264 .globl EXT(slave_pstart) 265LEXT(slave_pstart) 266 .code32 267 cli /* disable interrupts, so we don`t */ 268 /* need IDT for a while */ 269 POSTCODE(SLAVE_PSTART) 270 271 movl $EXT(mp_slave_stack) + PAGE_SIZE, %esp 272 273 xor %edi, %edi /* AP, no "kernbootstruct" */ 274 275 jmp L_pstart_common /* hop a ride to vstart() */ 276 277 278/* BEGIN HIBERNATE CODE */ 279 280.section __HIB, __text 281/* 282 * This code is linked into the kernel but part of the "__HIB" section, 283 * which means it's used by code running in the special context of restoring 284 * the kernel text and data from the hibernation image read by the booter. 285 * hibernate_kernel_entrypoint() and everything it calls or references 286 * (ie. hibernate_restore_phys_page()) needs to be careful to only touch 287 * memory also in the "__HIB" section. 288 */ 289 290 .align ALIGN 291 .globl EXT(hibernate_machine_entrypoint) 292.code32 293LEXT(hibernate_machine_entrypoint) 294 movl %eax, %edi /* regparm(1) calling convention */ 295 296 /* Use low 32-bits of address as 32-bit stack */ 297 movl $EXT(low_eintstack), %esp 298 299 /* 300 * Set up GDT 301 */ 302 movl $EXT(master_gdtr), %eax 303 lgdtl (%eax) 304 305 /* Switch to 64-bit on the Boot PTs */ 306 SWITCH_TO_64BIT_MODE 307 308 leaq EXT(hibernate_kernel_entrypoint)(%rip),%rcx 309 310 /* adjust the pointers to be up high */ 311 movq $(KERNEL_BASE), %rax 312 orq %rax, %rsp 313 orq %rcx, %rax 314 315 /* %edi is already filled with header pointer */ 316 xorl %esi, %esi /* zero 2nd arg */ 317 xorl %edx, %edx /* zero 3rd arg */ 318 xorl %ecx, %ecx /* zero 4th arg */ 319 andq $0xfffffffffffffff0, %rsp /* align stack */ 320 321 /* call instead of jmp to keep the required stack alignment */ 322 xorq %rbp, %rbp /* zero frame pointer */ 323 call *%rax 324 325 /* NOTREACHED */ 326 hlt 327 328/* END HIBERNATE CODE */ 329 330#if CONFIG_SLEEP 331/* BEGIN ACPI WAKEUP CODE */ 332 333#include <i386/acpi.h> 334 335 336/* 337 * acpi_wake_start 338 */ 339 340.section __TEXT,__text 341.code64 342 343/* 344 * acpi_sleep_cpu(acpi_sleep_callback func, void * refcon) 345 * 346 * Save CPU state before platform sleep. Restore CPU state 347 * following wake up. 348 */ 349 350ENTRY(acpi_sleep_cpu) 351 push %rbp 352 mov %rsp, %rbp 353 354 /* save flags */ 355 pushf 356 357 /* save general purpose registers */ 358 push %rax 359 push %rbx 360 push %rcx 361 push %rdx 362 push %rbp 363 push %rsi 364 push %rdi 365 push %r8 366 push %r9 367 push %r10 368 push %r11 369 push %r12 370 push %r13 371 push %r14 372 push %r15 373 374 mov %rsp, saved_rsp(%rip) 375 376 /* make sure tlb is flushed */ 377 mov %cr3,%rax 378 mov %rax,%cr3 379 380 /* save control registers */ 381 mov %cr0, %rax 382 mov %rax, saved_cr0(%rip) 383 mov %cr2, %rax 384 mov %rax, saved_cr2(%rip) 385 mov %cr3, %rax 386 mov %rax, saved_cr3(%rip) 387 mov %cr4, %rax 388 mov %rax, saved_cr4(%rip) 389 390 /* save segment registers */ 391 movw %es, saved_es(%rip) 392 movw %fs, saved_fs(%rip) 393 movw %gs, saved_gs(%rip) 394 movw %ss, saved_ss(%rip) 395 396 /* save the 64bit user and kernel gs base */ 397 /* note: user's curently swapped into kernel base MSR */ 398 mov $MSR_IA32_KERNEL_GS_BASE, %rcx 399 rdmsr 400 movl %eax, saved_ugs_base(%rip) 401 movl %edx, saved_ugs_base+4(%rip) 402 swapgs 403 rdmsr 404 movl %eax, saved_kgs_base(%rip) 405 movl %edx, saved_kgs_base+4(%rip) 406 swapgs 407 408 /* save descriptor table registers */ 409 sgdt saved_gdt(%rip) 410 sldt saved_ldt(%rip) 411 sidt saved_idt(%rip) 412 str saved_tr(%rip) 413 414 /* 415 * Call ACPI function provided by the caller to sleep the platform. 416 * This call will not return on success. 417 */ 418 419 xchgq %rdi, %rsi 420 call *%rsi 421 422 /* sleep failed, no cpu context lost */ 423 jmp wake_restore 424 425.section __HIB, __text 426.code32 427.globl EXT(acpi_wake_prot) 428EXT(acpi_wake_prot): 429 /* protected mode, paging disabled */ 430 movl $EXT(low_eintstack), %esp 431 432 SWITCH_TO_64BIT_MODE 433 434 jmp Lwake_64 435 436.section __TEXT,__text 437.code64 438 439.globl EXT(acpi_wake_prot_entry) 440EXT(acpi_wake_prot_entry): 441 POSTCODE(ACPI_WAKE_PROT_ENTRY) 442 /* Return from hibernate code in iokit/Kernel/IOHibernateRestoreKernel.c 443 */ 444Lwake_64: 445 /* 446 * restore cr4, PAE and NXE states in an orderly fashion 447 */ 448 mov saved_cr4(%rip), %rcx 449 mov %rcx, %cr4 450 451 mov $(MSR_IA32_EFER), %ecx /* MSR number in ecx */ 452 rdmsr /* MSR value in edx:eax */ 453 or $(MSR_IA32_EFER_NXE), %eax /* Set NXE bit in low 32-bits */ 454 wrmsr /* Update */ 455 456 movq saved_cr2(%rip), %rax 457 mov %rax, %cr2 458 459 /* restore CR0, paging enabled */ 460 mov saved_cr0(%rip), %rax 461 mov %rax, %cr0 462 463 /* restore the page tables */ 464 mov saved_cr3(%rip), %rax 465 mov %rax, %cr3 466 467 /* protected mode, paging enabled */ 468 POSTCODE(ACPI_WAKE_PAGED_ENTRY) 469 470 /* load null segment selectors */ 471 xor %eax, %eax 472 movw %ax, %ss 473 movw %ax, %ds 474 475 /* restore descriptor tables */ 476 lgdt saved_gdt(%rip) 477 lldt saved_ldt(%rip) 478 lidt saved_idt(%rip) 479 480 /* restore segment registers */ 481 movw saved_es(%rip), %es 482 movw saved_fs(%rip), %fs 483 movw saved_gs(%rip), %gs 484 movw saved_ss(%rip), %ss 485 486 /* restore the 64bit kernel and user gs base */ 487 mov $MSR_IA32_KERNEL_GS_BASE, %rcx 488 movl saved_kgs_base(%rip), %eax 489 movl saved_kgs_base+4(%rip), %edx 490 wrmsr 491 swapgs 492 movl saved_ugs_base(%rip), %eax 493 movl saved_ugs_base+4(%rip), %edx 494 wrmsr 495 496 /* 497 * Restore task register. Before doing this, clear the busy flag 498 * in the TSS descriptor set by the CPU. 499 */ 500 lea saved_gdt(%rip), %rax 501 movq 2(%rax), %rdx /* GDT base, skip limit word */ 502 movl $(KERNEL_TSS), %eax /* TSS segment selector */ 503 movb $(K_TSS), 5(%rdx, %rax) /* clear busy flag */ 504 505 ltr saved_tr(%rip) /* restore TR */ 506 507wake_restore: 508 mov saved_rsp(%rip), %rsp 509 510 /* restore general purpose registers */ 511 pop %r15 512 pop %r14 513 pop %r13 514 pop %r12 515 pop %r11 516 pop %r10 517 pop %r9 518 pop %r8 519 pop %rdi 520 pop %rsi 521 pop %rbp 522 pop %rdx 523 pop %rcx 524 pop %rbx 525 pop %rax 526 527 /* restore flags */ 528 popf 529 530 leave 531 ret 532 533/* END ACPI WAKEUP CODE */ 534#endif /* CONFIG_SLEEP */ 535 536/* Code to get from real mode to protected mode */ 537 538#define operand_size_prefix .byte 0x66 539#define address_size_prefix .byte 0x67 540#define cs_base_prefix .byte 0x2e 541 542#define LJMP(segment,address) \ 543 operand_size_prefix ;\ 544 .byte 0xea ;\ 545 .long address-EXT(real_mode_bootstrap_base) ;\ 546 .word segment 547 548#define LGDT(address) \ 549 cs_base_prefix ;\ 550 address_size_prefix ;\ 551 operand_size_prefix ;\ 552 .word 0x010f ;\ 553 .byte 0x15 ;\ 554 .long address-EXT(real_mode_bootstrap_base) 555 556.section __HIB, __text 557.align 12 /* Page align for single bcopy_phys() */ 558.code32 559Entry(real_mode_bootstrap_base) 560 cli 561 562 LGDT(EXT(protected_mode_gdtr)) 563 564 /* set the PE bit of CR0 */ 565 mov %cr0, %eax 566 inc %eax 567 mov %eax, %cr0 568 569 /* reload CS register */ 570 LJMP(KERNEL32_CS, 1f + REAL_MODE_BOOTSTRAP_OFFSET) 5711: 572 573 /* we are in protected mode now */ 574 /* set up the segment registers */ 575 mov $KERNEL_DS, %eax 576 movw %ax, %ds 577 movw %ax, %es 578 movw %ax, %ss 579 xor %eax,%eax 580 movw %ax, %fs 581 movw %ax, %gs 582 583 POSTCODE(SLAVE_STARTPROG_ENTRY); 584 585 mov PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET, %ecx 586 jmp *%ecx 587 588Entry(protected_mode_gdtr) 589 .short 160 /* limit (8*20 segs) */ 590 .quad EXT(master_gdt) 591 592Entry(real_mode_bootstrap_end) 593 594/* Save area used across sleep/wake */ 595.section __HIB, __data 596.align 2 597 598/* gdtr for real address of master_gdt in HIB (not the aliased address) */ 599Entry(master_gdtr) 600 .word 160 /* limit (8*20 segs) */ 601 .quad EXT(master_gdt) 602 603saved_gdt: .word 0 604 .quad 0 605saved_rsp: .quad 0 606saved_es: .word 0 607saved_fs: .word 0 608saved_gs: .word 0 609saved_ss: .word 0 610saved_cr0: .quad 0 611saved_cr2: .quad 0 612saved_cr3: .quad 0 613saved_cr4: .quad 0 614saved_idt: .word 0 615 .quad 0 616saved_ldt: .word 0 617saved_tr: .word 0 618saved_kgs_base: .quad 0 619saved_ugs_base: .quad 0 620 621