xref: /xnu-10002.81.5/osfmk/arm64/caches_asm.s (revision 5e3eaea39dcf651e66cb99ba7d70e32cc4a99587) 
1/*
2 * Copyright (c) 2010-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <machine/asm.h>
30#include <arm64/proc_reg.h>
31#include <arm/pmap.h>
32#include <sys/errno.h>
33#include "assym.s"
34
35/*
36 *	void InvalidatePoU_Icache(void)
37 *
38 *		Invalidate i-cache
39 */
40	.text
41	.align 2
42	.globl EXT(InvalidatePoU_Icache)
43	.globl EXT(invalidate_mmu_icache)
44LEXT(InvalidatePoU_Icache)
45LEXT(invalidate_mmu_icache)
46	dsb		sy
47	ic		ialluis								// Invalidate icache
48	dsb		sy
49	isb		sy
50	ret
51
52/*
53 *	void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length)
54 *
55 *		Invalidate icache region
56 */
57	.text
58	.align 2
59	.globl EXT(InvalidatePoU_IcacheRegion)
60LEXT(InvalidatePoU_IcacheRegion)
61	ARM64_STACK_PROLOG
62	PUSH_FRAME
63	bl		EXT(CleanPoU_DcacheRegion)
64#if __ARM_IC_NOALIAS_ICACHE__
65	mov		x9, #((1<<MMU_I_CLINE)-1)
66	and		x2, x0, x9
67	bic		x0, x0, x9							// Cached aligned
68	add		x1, x1, x2
69	sub		x1, x1, #1
70	lsr		x1, x1, #MMU_I_CLINE					// Set cache line counter
71L_ipui_loop:
72	ic		ivau, x0							// Invalidate icache line
73	add		x0, x0, #1<<MMU_I_CLINE				// Get next cache aligned addr
74	subs	x1, x1, #1							// Decrementer cache line counter
75	b.pl	L_ipui_loop							// Loop in counter not null
76	dsb		sy
77	isb		sy
78#else
79	bl		EXT(InvalidatePoU_Icache)
80#endif
81	POP_FRAME
82	ARM64_STACK_EPILOG
83
84/*
85 *	Obtains cache physical layout information required for way/set
86 *	data cache maintenance operations.
87 *
88 *	$0: Data cache level, starting from 0
89 *	$1: Output register for set increment
90 *	$2: Output register for last valid set
91 *	$3: Output register for way increment
92 */
93.macro GET_CACHE_CONFIG
94	lsl		$0, $0, #1
95	msr		CSSELR_EL1, $0						// Select appropriate cache
96	isb											// Synchronize context
97
98	mrs		$0, CCSIDR_EL1
99	ubfx	$1, $0, #3, #10						// extract number of ways - 1
100	mov		$2, $1
101	add		$1, $1, #1							// calculate number of ways
102
103	mov		$0, #63
104	and		$2, $2, $1
105	cmp		$2, #0
106	cinc	$0, $0, ne
107	clz		$1, $1
108	sub		$0, $0, $1
109
110	mov 	$1, #32								// calculate way increment
111	sub		$3, $1, $0
112	mov		$1, #1
113	lsl		$3, $1, $3
114
115	mrs		$0, CCSIDR_EL1
116	ubfx	$1, $0, #0, #3						// extract log2(line size) - 4
117	add		$1, $1, #4							// calculate log2(line size)
118	mov		$2, #1
119	lsl		$1, $2, $1							// calculate set increment
120
121	ubfx	$2, $0, #13, #15					// extract number of sets - 1
122	add		$2, $2, #1							// calculate number of sets
123	mul		$2, $1, $2							// calculate last valid set
124.endmacro
125
126/*
127 * Returns the cache configuration for the specified level
128 *	$0: Output register
129 *	$1: Cache level register
130 *	$2: Scratch register
131 */
132.macro CACHE_AT_LEVEL
133	mrs		$0, CLIDR_EL1
134	add		$2, $1, $1, lsl #1
135	lsr		$0, $0, $2
136	and		$0, $0, #7					// extract cache type
137.endmacro
138
139/*
140 * Perform set/way maintenance to the desired cache level
141 *	$0: 'dc' set/way variant, e.g. csw or cisw
142 *	x0: maximum cache level, 0-based, inclusive
143 */
144.macro DCACHE_SET_WAY
145	dmb		sy
146	mov		x1, #0
1471:
148	CACHE_AT_LEVEL x2, x1, x3
149	cbz		x2, 5f			// No cache at this level, all higher levels may be skipped
150	cmp		x2, #2
151	b.lt		4f			// No data cache at this level, skip to next level
152	mov		x2, x1
153	GET_CACHE_CONFIG x2, x9, x10, x11
154	lsl		x2, x1, #1		// level field for cisw/csw, bits 1:3
1552:
1563:
157	dc		$0, x2			// clean dcache line by way/set
158	add		x2, x2, x9		// increment set index
159	tst		x2, x10			// look for overflow
160	b.eq		3b
161	bic		x2, x2, x10		// clear set overflow
162	adds		w2, w2, w11		// increment way
163	b.cc		2b			// loop
164	dsb		sy			// ensure completion of prior level maintenance
1654:
166	add		x1, x1, #1
167	cmp		x1, x0
168	b.ls		1b			// next level
1695:
170	ret
171.endmacro
172
173#if defined(APPLE_ARM64_ARCH_FAMILY) && !APPLEVIRTUALPLATFORM
174/*
175 * Enables cache maintenance by VA instructions on Apple SoCs.
176 *
177 *	$0: Scratch register
178 *	$1: Scratch register
179 */
180.macro ENABLE_DC_MVA_OPS
181	isb		sy
182	ARM64_IS_PCORE $0
183	ARM64_READ_EP_SPR $0, $1, EHID4, HID4
184	and		$1, $1, (~ARM64_REG_HID4_DisDcMVAOps)
185	ARM64_WRITE_EP_SPR $0, $1, EHID4, HID4
186	isb		sy
187.endmacro
188
189/*
190 * Disables cache maintenance by VA instructions on Apple SoCs.
191 *
192 *	$0: Scratch register
193 *	$1: Scratch register
194 */
195.macro DISABLE_DC_MVA_OPS
196	isb		sy
197	ARM64_IS_PCORE $0
198	ARM64_READ_EP_SPR $0, $1, EHID4, HID4
199	orr		$1, $1, ARM64_REG_HID4_DisDcMVAOps
200	ARM64_WRITE_EP_SPR $0, $1, EHID4, HID4
201	isb		sy
202.endmacro
203#endif
204
205/*
206 * void CleanPoC_Dcache(void)
207 *
208 *		Clean all d-caches
209 */
210	.text
211	.align 2
212	.globl EXT(CleanPoC_Dcache)
213	.globl EXT(clean_mmu_dcache)
214LEXT(CleanPoC_Dcache)
215#if  defined(APPLE_ARM64_ARCH_FAMILY)
216	dsb		sy
217	ret
218	/* "Fully Coherent." */
219#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
220	mrs		x0, CLIDR_EL1
221	ubfx		x0, x0, #24, #3	// extract CLIDR_EL1.LoC
222	DCACHE_SET_WAY csw
223#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
224
225/*
226 * void CleanPoU_Dcache(void)
227 *
228 *		Clean D-cache to Point of Unification
229 */
230	.text
231	.align 2
232	.globl EXT(CleanPoU_Dcache)
233LEXT(CleanPoU_Dcache)
234#if defined(APPLE_ARM64_ARCH_FAMILY)
235	dsb sy
236	ret
237	/* "Fully Coherent." */
238#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
239	mrs		x0, CLIDR_EL1
240	ubfx		x0, x0, #21, 3	// extract CLIDR_EL1.LoUIS
241	DCACHE_SET_WAY csw
242#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
243
244/*
245 *	void CleanPoU_DcacheRegion(vm_offset_t va, unsigned length)
246 *
247 *		Clean d-cache region to Point of Unification
248 */
249	.text
250	.align 2
251	.globl EXT(CleanPoU_DcacheRegion)
252LEXT(CleanPoU_DcacheRegion)
253#if defined(APPLE_ARM64_ARCH_FAMILY)
254	/* "Fully Coherent." */
255#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
256	mov		x9, #((1<<MMU_CLINE)-1)
257	and		x2, x0, x9
258	bic		x3, x0, x9							// Cached aligned
259	add		x4, x1, x2
260	sub		x4, x4, #1
261	lsr		x4, x4, #MMU_CLINE					// Set cache line counter
262	dmb		sy
263L_cpudr_loop:
264	dc		cvau, x3							// Clean dcache line to PoU
265	add		x3, x3, #(1<<MMU_CLINE)				// Get next cache aligned addr
266	subs	x4, x4, #1							// Decrementer cache line counter
267	b.pl	L_cpudr_loop						// Loop in counter not null
268#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
269	dsb		sy
270	ret
271
272/*
273 *	void CleanPoC_DcacheRegion_internal(vm_offset_t va, size_t length)
274 *
275 *		Clean d-cache region to Point of Coherency
276 */
277	.text
278	.align 2
279LEXT(CleanPoC_DcacheRegion_internal)
280	#define CLINE_FLUSH_STRIDE MMU_CLINE
281
282	mov		x9, #((1<<CLINE_FLUSH_STRIDE)-1)
283	and		x2, x0, x9
284	bic		x0, x0, x9							// Cached aligned
285	add		x1, x1, x2
286	sub		x1, x1, #1
287	lsr		x1, x1, #(CLINE_FLUSH_STRIDE)		// Set cache line counter
288	dsb		sy
289L_cpcdr_loop:
290#if defined(APPLE_ARM64_ARCH_FAMILY)
291	// It may be tempting to clean the cache (dc cvac), but it's always a NOP on
292	// Apple hardware.
293	dc		civac, x0							// Clean & Invalidate dcache line to PoC
294#else /* defined(APPLE_ARM64_ARCH_FAMILY) */
295	dc		cvac, x0 							// Clean dcache line to PoC
296#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
297	add		x0, x0, #(1<<CLINE_FLUSH_STRIDE)	// Get next cache aligned addr
298	subs	x1, x1, #1							// Decrementer cache line counter
299	b.pl	L_cpcdr_loop						// Loop in counter not null
300	dsb		sy
301	ret
302
303/*
304 *	void CleanPoC_DcacheRegion(vm_offset_t va, size_t length)
305 *
306 *		Clean d-cache region to Point of Coherency
307 */
308	.text
309	.align 2
310	.globl EXT(CleanPoC_DcacheRegion)
311LEXT(CleanPoC_DcacheRegion)
312#if defined(APPLE_ARM64_ARCH_FAMILY)
313	/* "Fully Coherent." */
314	dsb		sy
315	ret
316#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
317	b EXT(CleanPoC_DcacheRegion_internal)
318#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
319
320
321/*
322 *	void CleanPoC_DcacheRegion_Force(vm_offset_t va, size_t length)
323 *
324 *		Clean d-cache region to Point of Coherency -  when you really
325 *		need to flush even on coherent platforms, e.g. panic log
326 */
327	.text
328	.align 2
329	.globl EXT(CleanPoC_DcacheRegion_Force)
330LEXT(CleanPoC_DcacheRegion_Force)
331#if defined(APPLE_ARM64_ARCH_FAMILY)
332	ARM64_STACK_PROLOG
333	PUSH_FRAME
334	stp		x0, x1, [sp, #-16]!
335	bl		EXT(_disable_preemption)
336	ldp		x0, x1, [sp], #16
337	bl		EXT(CleanPoC_DcacheRegion_Force_nopreempt)
338	bl		EXT(_enable_preemption)
339	POP_FRAME
340	ARM64_STACK_EPILOG
341#else
342	b		EXT(CleanPoC_DcacheRegion_internal)
343#endif /* APPLE_ARM64_ARCH_FAMILY */
344
345/*
346 *	void FlushPoC_Dcache(void)
347 *
348 *		Clean and Invalidate dcaches to Point of Coherency
349 */
350	.text
351	.align 2
352	.globl EXT(FlushPoC_Dcache)
353LEXT(FlushPoC_Dcache)
354#if defined(APPLE_ARM64_ARCH_FAMILY)
355	dsb sy
356	ret
357	/* "Fully Coherent." */
358#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
359	mrs		x0, CLIDR_EL1
360	ubfx		x0, x0, #24, #3	// extract CLIDR_EL1.LoC
361	DCACHE_SET_WAY cisw
362#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
363
364/*
365 * 	void Flush_Dcache(void)
366 *
367 * 		Clean and invalidate D-cache, all levels
368 */
369	.text
370	.align 2
371	.globl EXT(Flush_Dcache)
372LEXT(Flush_Dcache)
373	mov x0, #6 // Maximum allowable caching level (0-based)
374	DCACHE_SET_WAY cisw
375
376/*
377 * void FlushPoU_Dcache(void)
378 *
379 *		Flush D-cache to Point of Unification
380 */
381	.text
382	.align 2
383	.globl EXT(FlushPoU_Dcache)
384LEXT(FlushPoU_Dcache)
385#if defined(APPLE_ARM64_ARCH_FAMILY)
386	dsb sy
387	ret
388	/* "Fully Coherent." */
389#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
390	mrs		x0, CLIDR_EL1
391	ubfx		x0, x0, #21, 3	// extract CLIDR_EL1.LoUIS
392	DCACHE_SET_WAY	cisw
393#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
394
395/*
396 *	void FlushPoC_DcacheRegion(vm_offset_t va, unsigned length)
397 *
398 *		Clean and Invalidate d-cache region to Point of Coherency
399 */
400	.text
401	.align 2
402	.globl EXT(FlushPoC_DcacheRegion)
403LEXT(FlushPoC_DcacheRegion)
404#if defined(APPLE_ARM64_ARCH_FAMILY)
405	/* "Fully Coherent." */
406#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
407	mov		x9, #((1<<MMU_CLINE)-1)
408	and		x2, x0, x9
409	bic		x0, x0, x9							// Cached aligned
410	add		x1, x1, x2
411	sub		x1, x1, #1
412	lsr		x1, x1, #MMU_CLINE					// Set cache line counter
413	dmb		sy
414L_fpcdr_loop:
415	dc		civac, x0							// Clean invalidate dcache line to PoC
416	add		x0, x0, #(1<<MMU_CLINE)				// Get next cache aligned addr
417	subs	x1, x1, #1							// Decrementer cache line counter
418	b.pl	L_fpcdr_loop						// Loop in counter not null
419#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
420	dsb		sy
421	ret
422
423/*
424 *      void flush_dcache64(addr64_t addr, unsigned length, boolean_t phys)
425 */
426        .text
427        .align 2
428        .globl EXT(flush_dcache64)
429LEXT(flush_dcache64)
430	BRANCH_EXTERN    flush_dcache
431
432/*
433 *      void clean_dcache64(addr64_t addr, unsigned length, boolean_t phys)
434 */
435        .text
436        .align 2
437        .globl EXT(clean_dcache64)
438LEXT(clean_dcache64)
439	BRANCH_EXTERN    clean_dcache
440
441/*
442 *      void invalidate_icache(vm_offset_t va, unsigned length, boolean_t phys)
443 *      void invalidate_icache64(addr64_t va, unsigned length, boolean_t phys)
444 */
445        .text
446        .align 2
447        .globl EXT(invalidate_icache64)
448        .globl EXT(invalidate_icache)
449LEXT(invalidate_icache64)
450LEXT(invalidate_icache)
451	cmp     w2, #0								// Is it physical?
452	b.eq	Lcall_invalidate_worker
453	adrp	x2, _gPhysBase@page
454	add		x2, x2, _gPhysBase@pageoff
455	ldr		x2, [x2]
456	sub		x0, x0, x2
457	adrp	x2, _gVirtBase@page
458	add		x2, x2, _gVirtBase@pageoff
459	ldr		x2, [x2]
460	add		x0, x0, x2
461Lcall_invalidate_worker:
462	b		EXT(InvalidatePoU_IcacheRegion)
463
464
465/* vim: set ts=4: */
466