xref: /xnu-11215.61.5/osfmk/arm64/caches_asm.s (revision 4f1223e81cd707a65cc109d0b8ad6653699da3c4) 
1/*
2 * Copyright (c) 2010-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <machine/asm.h>
30#include <arm64/proc_reg.h>
31#include <arm/pmap.h>
32#include <sys/errno.h>
33#include "assym.s"
34
35/*
36 *	void InvalidatePoU_Icache(void)
37 *
38 *		Invalidate i-cache
39 */
40	.text
41	.align 2
42	.globl EXT(InvalidatePoU_Icache)
43	.globl EXT(invalidate_mmu_icache)
44LEXT(InvalidatePoU_Icache)
45LEXT(invalidate_mmu_icache)
46	ARM64_PROLOG
47	dsb		sy
48	ic		ialluis								// Invalidate icache
49	dsb		sy
50	isb		sy
51	ret
52
53/*
54 *	void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length)
55 *
56 *		Invalidate icache region
57 */
58	.text
59	.align 2
60	.globl EXT(InvalidatePoU_IcacheRegion)
61LEXT(InvalidatePoU_IcacheRegion)
62	ARM64_STACK_PROLOG
63	PUSH_FRAME
64	bl		EXT(CleanPoU_DcacheRegion)
65#if __ARM_IC_NOALIAS_ICACHE__
66	mov		x9, #((1<<MMU_I_CLINE)-1)
67	and		x2, x0, x9
68	bic		x0, x0, x9							// Cached aligned
69	add		x1, x1, x2
70	sub		x1, x1, #1
71	lsr		x1, x1, #MMU_I_CLINE					// Set cache line counter
72L_ipui_loop:
73	ic		ivau, x0							// Invalidate icache line
74	add		x0, x0, #1<<MMU_I_CLINE				// Get next cache aligned addr
75	subs	x1, x1, #1							// Decrementer cache line counter
76	b.pl	L_ipui_loop							// Loop in counter not null
77	dsb		sy
78	isb		sy
79#else
80	bl		EXT(InvalidatePoU_Icache)
81#endif
82	POP_FRAME
83	ARM64_STACK_EPILOG
84
85/*
86 *	Obtains cache physical layout information required for way/set
87 *	data cache maintenance operations.
88 *
89 *	$0: Data cache level, starting from 0
90 *	$1: Output register for set increment
91 *	$2: Output register for last valid set
92 *	$3: Output register for way increment
93 */
94.macro GET_CACHE_CONFIG
95	lsl		$0, $0, #1
96	msr		CSSELR_EL1, $0						// Select appropriate cache
97	isb											// Synchronize context
98
99	mrs		$0, CCSIDR_EL1
100	ubfx	$1, $0, #3, #10						// extract number of ways - 1
101	mov		$2, $1
102	add		$1, $1, #1							// calculate number of ways
103
104	mov		$0, #63
105	and		$2, $2, $1
106	cmp		$2, #0
107	cinc	$0, $0, ne
108	clz		$1, $1
109	sub		$0, $0, $1
110
111	mov 	$1, #32								// calculate way increment
112	sub		$3, $1, $0
113	mov		$1, #1
114	lsl		$3, $1, $3
115
116	mrs		$0, CCSIDR_EL1
117	ubfx	$1, $0, #0, #3						// extract log2(line size) - 4
118	add		$1, $1, #4							// calculate log2(line size)
119	mov		$2, #1
120	lsl		$1, $2, $1							// calculate set increment
121
122	ubfx	$2, $0, #13, #15					// extract number of sets - 1
123	add		$2, $2, #1							// calculate number of sets
124	mul		$2, $1, $2							// calculate last valid set
125.endmacro
126
127/*
128 * Returns the cache configuration for the specified level
129 *	$0: Output register
130 *	$1: Cache level register
131 *	$2: Scratch register
132 */
133.macro CACHE_AT_LEVEL
134	mrs		$0, CLIDR_EL1
135	add		$2, $1, $1, lsl #1
136	lsr		$0, $0, $2
137	and		$0, $0, #7					// extract cache type
138.endmacro
139
140/*
141 * Perform set/way maintenance to the desired cache level
142 *	$0: 'dc' set/way variant, e.g. csw or cisw
143 *	x0: maximum cache level, 0-based, inclusive
144 */
145.macro DCACHE_SET_WAY
146	dmb		sy
147	mov		x1, #0
1481:
149	CACHE_AT_LEVEL x2, x1, x3
150	cbz		x2, 5f			// No cache at this level, all higher levels may be skipped
151	cmp		x2, #2
152	b.lt		4f			// No data cache at this level, skip to next level
153	mov		x2, x1
154	GET_CACHE_CONFIG x2, x9, x10, x11
155	lsl		x2, x1, #1		// level field for cisw/csw, bits 1:3
1562:
1573:
158	dc		$0, x2			// clean dcache line by way/set
159	add		x2, x2, x9		// increment set index
160	tst		x2, x10			// look for overflow
161	b.eq		3b
162	bic		x2, x2, x10		// clear set overflow
163	adds		w2, w2, w11		// increment way
164	b.cc		2b			// loop
165	dsb		sy			// ensure completion of prior level maintenance
1664:
167	add		x1, x1, #1
168	cmp		x1, x0
169	b.ls		1b			// next level
1705:
171	ret
172.endmacro
173
174#if defined(APPLE_ARM64_ARCH_FAMILY) && !APPLEVIRTUALPLATFORM
175/*
176 * Enables cache maintenance by VA instructions on Apple SoCs.
177 *
178 *	$0: Scratch register
179 *	$1: Scratch register
180 */
181.macro ENABLE_DC_MVA_OPS
182	isb		sy
183	ARM64_IS_ECORE          $0
184	ARM64_READ_CORE_SYSREG   $0, $1, EHID4, HID4
185	and                     $1, $1, (~ARM64_REG_HID4_DisDcMVAOps)
186	ARM64_WRITE_CORE_SYSREG  $0, $1, EHID4, HID4
187	isb		sy
188.endmacro
189
190/*
191 * Disables cache maintenance by VA instructions on Apple SoCs.
192 *
193 *	$0: Scratch register
194 *	$1: Scratch register
195 */
196.macro DISABLE_DC_MVA_OPS
197	isb		sy
198	ARM64_IS_ECORE          $0
199	ARM64_READ_CORE_SYSREG   $0, $1, EHID4, HID4
200	orr                     $1, $1, ARM64_REG_HID4_DisDcMVAOps
201	ARM64_WRITE_CORE_SYSREG  $0, $1, EHID4, HID4
202	isb		sy
203.endmacro
204#endif
205
206/*
207 * void CleanPoC_Dcache(void)
208 *
209 *		Clean all d-caches
210 */
211	.text
212	.align 2
213	.globl EXT(CleanPoC_Dcache)
214	.globl EXT(clean_mmu_dcache)
215LEXT(CleanPoC_Dcache)
216	ARM64_PROLOG
217#if  defined(APPLE_ARM64_ARCH_FAMILY)
218	dsb		sy
219	ret
220	/* "Fully Coherent." */
221#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
222	mrs		x0, CLIDR_EL1
223	ubfx		x0, x0, #24, #3	// extract CLIDR_EL1.LoC
224	DCACHE_SET_WAY csw
225#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
226
227/*
228 * void CleanPoU_Dcache(void)
229 *
230 *		Clean D-cache to Point of Unification
231 */
232	.text
233	.align 2
234	.globl EXT(CleanPoU_Dcache)
235LEXT(CleanPoU_Dcache)
236	ARM64_PROLOG
237#if defined(APPLE_ARM64_ARCH_FAMILY)
238	dsb sy
239	ret
240	/* "Fully Coherent." */
241#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
242	mrs		x0, CLIDR_EL1
243	ubfx		x0, x0, #21, 3	// extract CLIDR_EL1.LoUIS
244	DCACHE_SET_WAY csw
245#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
246
247/*
248 *	void CleanPoU_DcacheRegion(vm_offset_t va, unsigned length)
249 *
250 *		Clean d-cache region to Point of Unification
251 */
252	.text
253	.align 2
254	.globl EXT(CleanPoU_DcacheRegion)
255LEXT(CleanPoU_DcacheRegion)
256	ARM64_PROLOG
257#if defined(APPLE_ARM64_ARCH_FAMILY)
258	/* "Fully Coherent." */
259#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
260	mov		x9, #((1<<MMU_CLINE)-1)
261	and		x2, x0, x9
262	bic		x3, x0, x9							// Cached aligned
263	add		x4, x1, x2
264	sub		x4, x4, #1
265	lsr		x4, x4, #MMU_CLINE					// Set cache line counter
266	dmb		sy
267L_cpudr_loop:
268	dc		cvau, x3							// Clean dcache line to PoU
269	add		x3, x3, #(1<<MMU_CLINE)				// Get next cache aligned addr
270	subs	x4, x4, #1							// Decrementer cache line counter
271	b.pl	L_cpudr_loop						// Loop in counter not null
272#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
273	dsb		sy
274	ret
275
276.macro CLEANPOC_DCACHEREGION
277	#define CLINE_FLUSH_STRIDE MMU_CLINE
278
279	ARM64_PROLOG
280	mov		x9, #((1<<CLINE_FLUSH_STRIDE)-1)
281	and		x2, x0, x9
282	bic		x0, x0, x9							// Cached aligned
283	add		x1, x1, x2
284	sub		x1, x1, #1
285	lsr		x1, x1, #(CLINE_FLUSH_STRIDE)		// Set cache line counter
2861:
287#if defined(APPLE_ARM64_ARCH_FAMILY)
288	// It may be tempting to clean the cache (dc cvac), but it's always a NOP on
289	// Apple hardware.
290	dc		civac, x0							// Clean & Invalidate dcache line to PoC
291#else /* defined(APPLE_ARM64_ARCH_FAMILY) */
292	dc		cvac, x0 							// Clean dcache line to PoC
293#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
294	add		x0, x0, #(1<<CLINE_FLUSH_STRIDE)	// Get next cache aligned addr
295	subs	x1, x1, #1							// Decrementer cache line counter
296	b.pl	1b						// Loop in counter not null
297.endmacro
298
299
300/*
301 *	void CleanPoC_DcacheRegion_internal(vm_offset_t va, size_t length)
302 *
303 *		Clean d-cache region to Point of Coherency
304 */
305	.text
306	.align 2
307LEXT(CleanPoC_DcacheRegion_internal)
308	dsb		sy
309	CLEANPOC_DCACHEREGION
310	dsb		sy
311	ret
312
313/*
314 *	void CleanPoC_DcacheRegion(vm_offset_t va, size_t length)
315 *
316 *		Clean d-cache region to Point of Coherency
317 */
318	.text
319	.align 2
320	.globl EXT(CleanPoC_DcacheRegion)
321LEXT(CleanPoC_DcacheRegion)
322	ARM64_PROLOG
323#if defined(APPLE_ARM64_ARCH_FAMILY)
324	/* "Fully Coherent." */
325	dsb		sy
326	ret
327#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
328	b EXT(CleanPoC_DcacheRegion_internal)
329#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
330
331
332/*
333 *	void CleanPoC_DcacheRegion_Force(vm_offset_t va, size_t length)
334 *
335 *		Clean d-cache region to Point of Coherency -  when you really
336 *		need to flush even on coherent platforms, e.g. panic log
337 */
338	.text
339	.align 2
340	.globl EXT(CleanPoC_DcacheRegion_Force)
341LEXT(CleanPoC_DcacheRegion_Force)
342#if defined(APPLE_ARM64_ARCH_FAMILY)
343	ARM64_STACK_PROLOG
344	PUSH_FRAME
345	stp		x0, x1, [sp, #-16]!
346	bl		EXT(_disable_preemption)
347	ldp		x0, x1, [sp], #16
348	bl		EXT(CleanPoC_DcacheRegion_Force_nopreempt)
349	bl		EXT(_enable_preemption)
350	POP_FRAME
351	ARM64_STACK_EPILOG
352#else
353	ARM64_PROLOG
354	b		EXT(CleanPoC_DcacheRegion_internal)
355#endif /* APPLE_ARM64_ARCH_FAMILY */
356
357/*
358 *	void FlushPoC_Dcache(void)
359 *
360 *		Clean and Invalidate dcaches to Point of Coherency
361 */
362	.text
363	.align 2
364	.globl EXT(FlushPoC_Dcache)
365LEXT(FlushPoC_Dcache)
366	ARM64_PROLOG
367#if defined(APPLE_ARM64_ARCH_FAMILY)
368	dsb sy
369	ret
370	/* "Fully Coherent." */
371#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
372	mrs		x0, CLIDR_EL1
373	ubfx		x0, x0, #24, #3	// extract CLIDR_EL1.LoC
374	DCACHE_SET_WAY cisw
375#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
376
377/*
378 * 	void Flush_Dcache(void)
379 *
380 * 		Clean and invalidate D-cache, all levels
381 */
382	.text
383	.align 2
384	.globl EXT(Flush_Dcache)
385LEXT(Flush_Dcache)
386	ARM64_PROLOG
387	mov x0, #6 // Maximum allowable caching level (0-based)
388	DCACHE_SET_WAY cisw
389
390/*
391 * void FlushPoU_Dcache(void)
392 *
393 *		Flush D-cache to Point of Unification
394 */
395	.text
396	.align 2
397	.globl EXT(FlushPoU_Dcache)
398LEXT(FlushPoU_Dcache)
399	ARM64_PROLOG
400#if defined(APPLE_ARM64_ARCH_FAMILY)
401	dsb sy
402	ret
403	/* "Fully Coherent." */
404#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
405	mrs		x0, CLIDR_EL1
406	ubfx		x0, x0, #21, 3	// extract CLIDR_EL1.LoUIS
407	DCACHE_SET_WAY	cisw
408#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
409
410/*
411 *	void FlushPoC_DcacheRegion(vm_offset_t va, unsigned length)
412 *
413 *		Clean and Invalidate d-cache region to Point of Coherency
414 */
415	.text
416	.align 2
417	.globl EXT(FlushPoC_DcacheRegion)
418LEXT(FlushPoC_DcacheRegion)
419	ARM64_PROLOG
420#if defined(APPLE_ARM64_ARCH_FAMILY)
421	/* "Fully Coherent." */
422#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
423	mov		x9, #((1<<MMU_CLINE)-1)
424	and		x2, x0, x9
425	bic		x0, x0, x9							// Cached aligned
426	add		x1, x1, x2
427	sub		x1, x1, #1
428	lsr		x1, x1, #MMU_CLINE					// Set cache line counter
429	dmb		sy
430L_fpcdr_loop:
431	dc		civac, x0							// Clean invalidate dcache line to PoC
432	add		x0, x0, #(1<<MMU_CLINE)				// Get next cache aligned addr
433	subs	x1, x1, #1							// Decrementer cache line counter
434	b.pl	L_fpcdr_loop						// Loop in counter not null
435#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
436	dsb		sy
437	ret
438
439/*
440 *      void flush_dcache64(addr64_t addr, unsigned length, boolean_t phys)
441 */
442        .text
443        .align 2
444        .globl EXT(flush_dcache64)
445LEXT(flush_dcache64)
446	ARM64_PROLOG
447	BRANCH_EXTERN    flush_dcache
448
449/*
450 *      void clean_dcache64(addr64_t addr, unsigned length, boolean_t phys)
451 */
452        .text
453        .align 2
454        .globl EXT(clean_dcache64)
455LEXT(clean_dcache64)
456	ARM64_PROLOG
457	BRANCH_EXTERN    clean_dcache
458
459/*
460 *      void invalidate_icache(vm_offset_t va, unsigned length, boolean_t phys)
461 *      void invalidate_icache64(addr64_t va, unsigned length, boolean_t phys)
462 */
463        .text
464        .align 2
465        .globl EXT(invalidate_icache64)
466        .globl EXT(invalidate_icache)
467LEXT(invalidate_icache64)
468LEXT(invalidate_icache)
469	ARM64_PROLOG
470	cmp     w2, #0								// Is it physical?
471	b.eq	Lcall_invalidate_worker
472	adrp	x2, _gPhysBase@page
473	add		x2, x2, _gPhysBase@pageoff
474	ldr		x2, [x2]
475	sub		x0, x0, x2
476	adrp	x2, _gVirtBase@page
477	add		x2, x2, _gVirtBase@pageoff
478	ldr		x2, [x2]
479	add		x0, x0, x2
480Lcall_invalidate_worker:
481	b		EXT(InvalidatePoU_IcacheRegion)
482
483
484/* vim: set ts=4: */
485