xref: /xnu-12377.81.4/osfmk/arm64/cswitch.s (revision 043036a2b3718f7f0be807e2870f8f47d3fa0796) 
1/*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#include <machine/asm.h>
29#include <arm64/machine_machdep.h>
30#include <arm64/machine_routines_asm.h>
31#include <arm64/pac_asm.h>
32#include <arm64/proc_reg.h>
33#include "assym.s"
34
35/*
36 * save_general_registers
37 *
38 * Saves variable registers to kernel PCB.
39 *   arg0 - thread_kernel_state pointer
40 *   arg1 - Scratch register
41 */
42
43.macro	save_general_registers
44/* AAPCS-64 Page 14
45 *
46 * A subroutine invocation must preserve the contents of the registers r19-r29
47 * and SP.
48 */
49#if __has_feature(ptrauth_calls)
50	paciasp
51#endif
52	stp		x19, x20, [$0, SS64_KERNEL_X19]
53	stp		x21, x22, [$0, SS64_KERNEL_X21]
54	stp		x23, x24, [$0, SS64_KERNEL_X23]
55	stp		x25, x26, [$0, SS64_KERNEL_X25]
56	stp		x27, x28, [$0, SS64_KERNEL_X27]
57	stp		fp, lr, [$0, SS64_KERNEL_FP]
58	strb	wzr, [$0, SS64_KERNEL_PC_WAS_IN_USER]
59	mov		x$1, sp
60	str		x$1, [$0, SS64_KERNEL_SP]
61#if HAS_ARM_FEAT_SSBS2
62#if APPLEVIRTUALPLATFORM
63	adrp	x$1, EXT(gARM_FEAT_SSBS)@page
64	ldrh	w$1, [x$1, EXT(gARM_FEAT_SSBS)@pageoff]
65	cbz		x$1, 1f
66#endif
67	mrs		x$1, SSBS
68	lsr     x$1, x$1, #0 + PSR64_SSBS_SHIFT_64
69	strb	w$1, [$0, SS64_KERNEL_SSBS]
701:
71#endif // HAS_ARM_FEAT_SSBS2
72#if HAS_MTE
73	mrs		x$1, TCO
74	lsr     x$1, x$1, #0 + PSR64_TCO_SHIFT
75	strb	w$1, [$0, SS64_KERNEL_TCO]
76#endif //HAS_MTE
77#if __ARM_ARCH_8_4__
78	mrs		x$1, DIT
79	lsr     x$1, x$1, #0 + PSR64_DIT_SHIFT
80	strb	w$1, [$0, SS64_KERNEL_DIT]
81#endif //__ARM_ARCH_8_4__
82#if __ARM_ARCH_8_2__
83	mrs		x$1, UAO
84	lsr     x$1, x$1, #0 + PSR64_UAO_SHIFT
85	strb	w$1, [$0, SS64_KERNEL_UAO]
86#endif //__ARM_ARCH_8_2__
87
88/* AAPCS-64 Page 14
89 *
90 * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
91 * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
92 * (or should be preserved by the caller).
93 */
94	str		d8,	[$0, NS64_KERNEL_D8]
95	str		d9,	[$0, NS64_KERNEL_D9]
96	str		d10,[$0, NS64_KERNEL_D10]
97	str		d11,[$0, NS64_KERNEL_D11]
98	str		d12,[$0, NS64_KERNEL_D12]
99	str		d13,[$0, NS64_KERNEL_D13]
100	str		d14,[$0, NS64_KERNEL_D14]
101	str		d15,[$0, NS64_KERNEL_D15]
102
103	mrs		x$1, FPCR
104	str		w$1, [$0, NS64_KERNEL_FPCR]
105.endmacro
106
107/*
108 * load_general_registers
109 *
110 * Loads variable registers from kernel PCB.
111 *   arg0 - thread_kernel_state pointer
112 *   arg1 - Scratch register
113 */
114.macro	load_general_registers
115	ldr		w$1, [$0, NS64_KERNEL_FPCR]
116	mrs		x19, FPCR
117	CMSR FPCR, x19, x$1, 1
1181:
119
120	ldp		x19, x20, [$0, SS64_KERNEL_X19]
121	ldp		x21, x22, [$0, SS64_KERNEL_X21]
122	ldp		x23, x24, [$0, SS64_KERNEL_X23]
123	ldp		x25, x26, [$0, SS64_KERNEL_X25]
124	ldp		x27, x28, [$0, SS64_KERNEL_X27]
125	ldp		fp, lr, [$0, SS64_KERNEL_FP]
126	ldr		x$1, [$0, SS64_KERNEL_SP]
127	mov		sp, x$1
128#if HAS_ARM_FEAT_SSBS2
129#if APPLEVIRTUALPLATFORM
130	adrp	x$1, EXT(gARM_FEAT_SSBS)@page
131	ldrh	w$1, [x$1, EXT(gARM_FEAT_SSBS)@pageoff]
132	cbz		x$1, 1f
133#endif // APPLEVIRTUALPLATFORM
134	ldrb	w$1, [$0, SS64_KERNEL_SSBS]
135	lsl     x$1, x$1, #0 + PSR64_SSBS_SHIFT_64
136	msr		SSBS, x$1
1371:
138#endif // HAS_ARM_FEAT_SSBS2
139#if HAS_MTE
140	ldrb	w$1, [$0, SS64_KERNEL_TCO]
141	lsl     x$1, x$1, #0 + PSR64_TCO_SHIFT
142	msr		TCO, x$1
143#endif //HAS_MTE
144#if __ARM_ARCH_8_2__
145	ldrb	w$1, [$0, SS64_KERNEL_UAO]
146	lsl     x$1, x$1, #0 + PSR64_UAO_SHIFT
147	msr		UAO, x$1
148#endif //__ARM_ARCH_8_2__
149#if __ARM_ARCH_8_4__
150	ldrb	w$1, [$0, SS64_KERNEL_DIT]
151	lsl     x$1, x$1, #0 + PSR64_DIT_SHIFT
152	msr		DIT, x$1
153#endif //__ARM_ARCH_8_4__
154
155	ldr		d8,	[$0, NS64_KERNEL_D8]
156	ldr		d9,	[$0, NS64_KERNEL_D9]
157	ldr		d10,[$0, NS64_KERNEL_D10]
158	ldr		d11,[$0, NS64_KERNEL_D11]
159	ldr		d12,[$0, NS64_KERNEL_D12]
160	ldr		d13,[$0, NS64_KERNEL_D13]
161	ldr		d14,[$0, NS64_KERNEL_D14]
162	ldr		d15,[$0, NS64_KERNEL_D15]
163.endmacro
164
165/*
166 * cswitch_epilogue
167 *
168 * Returns to the address reloaded into LR, authenticating if needed.
169 */
170.macro	cswitch_epilogue
171#if __has_feature(ptrauth_calls)
172	retaa
173#else
174	ret
175#endif
176.endm
177
178
179/*
180 * set_thread_registers
181 *
182 * Updates thread registers during context switch
183 *  arg0 - New thread pointer
184 *  arg1 - Scratch register
185 *  arg2 - Scratch register
186 */
187.macro	set_thread_registers
188	msr		TPIDR_EL1, $0						// Write new thread pointer to TPIDR_EL1
189	ldr		$1, [$0, ACT_CPUDATAP]
190	str		$0, [$1, CPU_ACTIVE_THREAD]
191
192	ldr	    $2, [$1, CPU_TPIDR_EL0]             // Write encoded CPU info to TPIDR_EL0
193	msr		TPIDR_EL0, $2
194
195	ldr		$1, [$0, TH_CTH_SELF]				// Get cthread pointer
196	msr		TPIDRRO_EL0, $1
197
198	ldr		$1, [$0, TH_THREAD_ID]				// Save the bottom 32-bits of the thread ID into
199	msr		CONTEXTIDR_EL1, $1					// CONTEXTIDR_EL1 (top 32-bits are RES0).
200.endmacro
201
202#define CSWITCH_ROP_KEYS	(HAS_APPLE_PAC && HAS_PARAVIRTUALIZED_PAC)
203#define CSWITCH_JOP_KEYS	(HAS_APPLE_PAC && HAS_PARAVIRTUALIZED_PAC)
204
205/*
206 * set_process_dependent_keys_and_sync_context
207 *
208 * Updates process dependent keys and issues explicit context sync during context switch if necessary
209 *  Per CPU Data rop_key is initialized in arm_init() for bootstrap processor
210 *  and in cpu_data_init for slave processors
211 *
212 *  thread - New thread pointer
213 *  new_key - Scratch register: New Thread Key
214 *  tmp_key - Scratch register: Current CPU Key
215 *  cpudatap - Scratch register: Current CPU Data pointer
216 *  wsync - Half-width scratch register: CPU sync required flag
217 *
218 *  to save on ISBs, for ARMv8.5 we use the CPU_SYNC_ON_CSWITCH field, cached in wsync, for pre-ARMv8.5,
219 *  we just use wsync to keep track of needing an ISB
220 */
221.macro set_process_dependent_keys_and_sync_context	thread, new_key, tmp_key, cpudatap, wsync
222
223
224#if defined(HAS_APPLE_PAC)
225	ldr		\cpudatap, [\thread, ACT_CPUDATAP]
226#endif /* defined(HAS_APPLE_PAC) */
227
228	mov		\wsync, #0
229
230#if CSWITCH_ROP_KEYS
231	ldr		\new_key, [\thread, TH_ROP_PID]
232	REPROGRAM_ROP_KEYS	Lskip_rop_keys_\@, \new_key, \cpudatap, \tmp_key
233#if HAS_PARAVIRTUALIZED_PAC
234	/* xnu hypervisor guarantees context synchronization during guest re-entry */
235	mov		\wsync, #0
236#else
237	mov		\wsync, #1
238#endif
239Lskip_rop_keys_\@:
240#endif /* CSWITCH_ROP_KEYS */
241
242#if CSWITCH_JOP_KEYS
243	ldr		\new_key, [\thread, TH_JOP_PID]
244	REPROGRAM_JOP_KEYS	Lskip_jop_keys_\@, \new_key, \cpudatap, \tmp_key
245#if HAS_PARAVIRTUALIZED_PAC
246	mov		\wsync, #0
247#else
248	mov		\wsync, #1
249#endif
250Lskip_jop_keys_\@:
251#endif /* CSWITCH_JOP_KEYS */
252
253	cbnz	\wsync, Lsync_now_\@
254#if !HAS_MTE
255	b		1f
256#else
257	/*
258	 * The HAS_MTE case:
259	 * If the new thread is inside an unprivileged access region and there is
260	 * a sync pending, we can't wait until an eret so synchronize it now.
261	 */
262	ldrb	\wsync, [\thread, IN_UNPRIVILEGED_ACCESS]
263	cbz		\wsync, 1f
264	ldrb	\wsync, [\cpudatap, CPU_SYNC_ON_CSWITCH]
265	cbz		\wsync, 1f
266#endif /* !HAS_MTE */
267
268Lsync_now_\@:
269	isb		sy
270
271#if HAS_PARAVIRTUALIZED_PAC
2721:	/* guests need to clear the sync flag even after skipping the isb, in case they synced via hvc instead */
273#endif
274	strb	wzr, [\cpudatap, CPU_SYNC_ON_CSWITCH]
2751:
276.endmacro
277
278/*
279 * void     machine_load_context(thread_t        thread)
280 *
281 * Load the context for the first thread to run on a
282 * cpu, and go.
283 */
284	.text
285	.align 2
286	.globl	EXT(machine_load_context)
287
288LEXT(machine_load_context)
289	ARM64_PROLOG
290	set_thread_registers 	x0, x1, x2
291	LOAD_KERN_STACK_TOP	dst=x1, src=x0, tmp=x2	// Get top of kernel stack
292	load_general_registers 	x1, 2
293	set_process_dependent_keys_and_sync_context	x0, x1, x2, x3, w4
294	mov		x0, #0								// Clear argument to thread_continue
295	cswitch_epilogue
296
297/*
298 *  typedef void (*thread_continue_t)(void *param, wait_result_t)
299 *
300 *	void Call_continuation( thread_continue_t continuation,
301 *	            			void *param,
302 *				            wait_result_t wresult,
303 *                          bool enable interrupts)
304 */
305	.text
306	.align	5
307	.globl	EXT(Call_continuation)
308
309LEXT(Call_continuation)
310	ARM64_PROLOG
311	mrs		x4, TPIDR_EL1						// Get the current thread pointer
312
313	/* ARM64_TODO arm loads the kstack top instead of arg4. What should we use? */
314	LOAD_KERN_STACK_TOP	dst=x5, src=x4, tmp=x6
315	mov		sp, x5								// Set stack pointer
316	mov		fp, #0								// Clear the frame pointer
317
318	set_process_dependent_keys_and_sync_context	x4, x5, x6, x7, w20
319
320	mov x20, x0  //continuation
321	mov x21, x1  //continuation parameter
322	mov x22, x2  //wait result
323
324	cbz x3, 1f
325	mov x0, #1
326	bl EXT(ml_set_interrupts_enabled)
3271:
328
329	mov		x0, x21								// Set the first parameter
330	mov		x1, x22								// Set the wait result arg
331#ifdef HAS_APPLE_PAC
332	mov		x21, THREAD_CONTINUE_T_DISC
333	blraa	x20, x21							// Branch to the continuation
334#else
335	blr		x20									// Branch to the continuation
336#endif
337	mrs		x0, TPIDR_EL1						// Get the current thread pointer
338	b		EXT(thread_terminate)				// Kill the thread
339
340
341/*
342 *	thread_t Switch_context(thread_t	old,
343 * 				void		(*cont)(void),
344 *				thread_t	new)
345 */
346	.text
347	.align 5
348	.globl	EXT(Switch_context)
349
350LEXT(Switch_context)
351	ARM64_PROLOG
352	cbnz	x1, Lswitch_threads					// Skip saving old state if blocking on continuation
353	LOAD_KERN_STACK_TOP	dst=x3, src=x0, tmp=x4	// Get the old kernel stack top
354	save_general_registers	x3, 4
355Lswitch_threads:
356	set_thread_registers	x2, x3, x4
357	LOAD_KERN_STACK_TOP	dst=x3, src=x2, tmp=x4
358	load_general_registers	x3, 4
359	set_process_dependent_keys_and_sync_context	x2, x3, x4, x5, w6
360	cswitch_epilogue
361
362/*
363 *	thread_t Shutdown_context(void (*doshutdown)(processor_t), processor_t processor)
364 *
365 */
366	.text
367	.align 5
368	.globl	EXT(Shutdown_context)
369
370LEXT(Shutdown_context)
371	ARM64_PROLOG
372	mrs		x10, TPIDR_EL1							// Get thread pointer
373	LOAD_KERN_STACK_TOP	dst=x11, src=x10, tmp=x12	// Get the top of the kernel stack
374	save_general_registers	x11, 12
375	msr		DAIFSet, #(DAIFSC_STANDARD_DISABLE)	// Disable interrupts
376	LOAD_INT_STACK_THREAD dst=x12, src=x10, tmp=x11
377	mov		sp, x12
378	b		EXT(cpu_doshutdown)
379
380/*
381 *	thread_t Idle_context(void)
382 *
383 */
384	.text
385	.align 5
386	.globl	EXT(Idle_context)
387
388LEXT(Idle_context)
389	ARM64_PROLOG
390	mrs		x0, TPIDR_EL1						// Get thread pointer
391	LOAD_KERN_STACK_TOP	dst=x1, src=x0, tmp=x2	// Get the top of the kernel stack
392	save_general_registers	x1, 2
393	LOAD_INT_STACK_THREAD	dst=x2, src=x0, tmp=x1
394	mov		sp, x2
395	b		EXT(cpu_idle)
396
397/*
398 *	thread_t Idle_context(void)
399 *
400 */
401	.text
402	.align 5
403	.globl	EXT(Idle_load_context)
404
405LEXT(Idle_load_context)
406	ARM64_PROLOG
407	mrs		x0, TPIDR_EL1						// Get thread pointer
408	LOAD_KERN_STACK_TOP	dst=x1, src=x0, tmp=x2	// Get the top of the kernel stack
409	load_general_registers	x1, 2
410	set_process_dependent_keys_and_sync_context	x0, x1, x2, x3, w4
411	cswitch_epilogue
412
413	.align	2
414	.globl	EXT(machine_set_current_thread)
415LEXT(machine_set_current_thread)
416	ARM64_PROLOG
417	set_thread_registers x0, x1, x2
418	ret
419
420
421/* vim: set ts=4: */
422