xref: /xnu-8019.80.24/osfmk/arm64/caches_asm.s (revision a325d9c4a84054e40bbe985afedcb50ab80993ea) 
1/*
2 * Copyright (c) 2010-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <machine/asm.h>
30#include <arm64/proc_reg.h>
31#include <arm/pmap.h>
32#include <sys/errno.h>
33#include "assym.s"
34
35/*
36 *	void InvalidatePoU_Icache(void)
37 *
38 *		Invalidate i-cache
39 */
40	.text
41	.align 2
42	.globl EXT(InvalidatePoU_Icache)
43	.globl EXT(invalidate_mmu_icache)
44LEXT(InvalidatePoU_Icache)
45LEXT(invalidate_mmu_icache)
46	dsb		sy
47	ic		ialluis								// Invalidate icache
48	dsb		sy
49	isb		sy
50	ret
51
52/*
53 *	void InvalidatePoU_IcacheRegion(vm_offset_t va, unsigned length)
54 *
55 *		Invalidate icache region
56 */
57	.text
58	.align 2
59	.globl EXT(InvalidatePoU_IcacheRegion)
60LEXT(InvalidatePoU_IcacheRegion)
61	ARM64_STACK_PROLOG
62	PUSH_FRAME
63	bl		EXT(CleanPoU_DcacheRegion)
64#if __ARM_IC_NOALIAS_ICACHE__
65	mov		x9, #((1<<MMU_I_CLINE)-1)
66	and		x2, x0, x9
67	bic		x0, x0, x9							// Cached aligned
68	add		x1, x1, x2
69	sub		x1, x1, #1
70	lsr		x1, x1, #MMU_I_CLINE					// Set cache line counter
71L_ipui_loop:
72	ic		ivau, x0							// Invalidate icache line
73	add		x0, x0, #1<<MMU_I_CLINE				// Get next cache aligned addr
74	subs	x1, x1, #1							// Decrementer cache line counter
75	b.pl	L_ipui_loop							// Loop in counter not null
76	dsb		sy
77	isb		sy
78#else
79	bl		EXT(InvalidatePoU_Icache)
80#endif
81	POP_FRAME
82	ARM64_STACK_EPILOG
83
84/*
85 *	Obtains cache physical layout information required for way/set
86 *	data cache maintenance operations.
87 *
88 *	$0: Data cache level, starting from 0
89 *	$1: Output register for set increment
90 *	$2: Output register for last valid set
91 *	$3: Output register for way increment
92 */
93.macro GET_CACHE_CONFIG
94	lsl		$0, $0, #1
95	msr		CSSELR_EL1, $0						// Select appropriate cache
96	isb											// Synchronize context
97
98	mrs		$0, CCSIDR_EL1
99	ubfx	$1, $0, #3, #10						// extract number of ways - 1
100	mov		$2, $1
101	add		$1, $1, #1							// calculate number of ways
102
103	mov		$0, #63
104	and		$2, $2, $1
105	cmp		$2, #0
106	cinc	$0, $0, ne
107	clz		$1, $1
108	sub		$0, $0, $1
109
110	mov 	$1, #32								// calculate way increment
111	sub		$3, $1, $0
112	mov		$1, #1
113	lsl		$3, $1, $3
114
115	mrs		$0, CCSIDR_EL1
116	ubfx	$1, $0, #0, #3						// extract log2(line size) - 4
117	add		$1, $1, #4							// calculate log2(line size)
118	mov		$2, #1
119	lsl		$1, $2, $1							// calculate set increment
120
121	ubfx	$2, $0, #13, #15					// extract number of sets - 1
122	add		$2, $2, #1							// calculate number of sets
123	mul		$2, $1, $2							// calculate last valid set
124.endmacro
125
126/*
127 * Returns the cache configuration for the specified level
128 *	$0: Output register
129 *	$1: Cache level register
130 *	$2: Scratch register
131 */
132.macro CACHE_AT_LEVEL
133	mrs		$0, CLIDR_EL1
134	add		$2, $1, $1, lsl #1
135	lsr		$0, $0, $2
136	and		$0, $0, #7					// extract cache type
137.endmacro
138
139/*
140 * Perform set/way maintenance to the desired cache level
141 *	$0: 'dc' set/way variant, e.g. csw or cisw
142 *	x0: maximum cache level, 0-based, inclusive
143 */
144.macro DCACHE_SET_WAY
145	dmb		sy
146	mov		x1, #0
1471:
148	CACHE_AT_LEVEL x2, x1, x3
149	cbz		x2, 5f			// No cache at this level, all higher levels may be skipped
150	cmp		x2, #2
151	b.lt		4f			// No data cache at this level, skip to next level
152	mov		x2, x1
153	GET_CACHE_CONFIG x2, x9, x10, x11
154	lsl		x2, x1, #1		// level field for cisw/csw, bits 1:3
1552:
1563:
157	dc		$0, x2			// clean dcache line by way/set
158	add		x2, x2, x9		// increment set index
159	tst		x2, x10			// look for overflow
160	b.eq		3b
161	bic		x2, x2, x10		// clear set overflow
162	adds		w2, w2, w11		// increment way
163	b.cc		2b			// loop
164	dsb		sy			// ensure completion of prior level maintenance
1654:
166	add		x1, x1, #1
167	cmp		x1, x0
168	b.ls		1b			// next level
1695:
170	ret
171.endmacro
172
173/*
174 * void CleanPoC_Dcache(void)
175 *
176 *		Clean all d-caches
177 */
178	.text
179	.align 2
180	.globl EXT(CleanPoC_Dcache)
181	.globl EXT(clean_mmu_dcache)
182LEXT(CleanPoC_Dcache)
183#if  defined(APPLE_ARM64_ARCH_FAMILY)
184	dsb		sy
185	ret
186	/* "Fully Coherent." */
187#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
188	mrs		x0, CLIDR_EL1
189	ubfx		x0, x0, #24, #3	// extract CLIDR_EL1.LoC
190	DCACHE_SET_WAY csw
191#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
192
193/*
194 * void CleanPoU_Dcache(void)
195 *
196 *		Clean D-cache to Point of Unification
197 */
198	.text
199	.align 2
200	.globl EXT(CleanPoU_Dcache)
201LEXT(CleanPoU_Dcache)
202#if defined(APPLE_ARM64_ARCH_FAMILY)
203	dsb sy
204	ret
205	/* "Fully Coherent." */
206#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
207	mrs		x0, CLIDR_EL1
208	ubfx		x0, x0, #21, 3	// extract CLIDR_EL1.LoUIS
209	DCACHE_SET_WAY csw
210#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
211
212/*
213 *	void CleanPoU_DcacheRegion(vm_offset_t va, unsigned length)
214 *
215 *		Clean d-cache region to Point of Unification
216 */
217	.text
218	.align 2
219	.globl EXT(CleanPoU_DcacheRegion)
220LEXT(CleanPoU_DcacheRegion)
221#if defined(APPLE_ARM64_ARCH_FAMILY)
222	/* "Fully Coherent." */
223#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
224	mov		x9, #((1<<MMU_CLINE)-1)
225	and		x2, x0, x9
226	bic		x3, x0, x9							// Cached aligned
227	add		x4, x1, x2
228	sub		x4, x4, #1
229	lsr		x4, x4, #MMU_CLINE					// Set cache line counter
230	dmb		sy
231L_cpudr_loop:
232	dc		cvau, x3							// Clean dcache line to PoU
233	add		x3, x3, #(1<<MMU_CLINE)				// Get next cache aligned addr
234	subs	x4, x4, #1							// Decrementer cache line counter
235	b.pl	L_cpudr_loop						// Loop in counter not null
236#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
237	dsb		sy
238	ret
239
240/*
241 *	void CleanPoC_DcacheRegion_internal(vm_offset_t va, size_t length)
242 *
243 *		Clean d-cache region to Point of Coherency
244 */
245	.text
246	.align 2
247LEXT(CleanPoC_DcacheRegion_internal)
248	#define CLINE_FLUSH_STRIDE MMU_CLINE
249
250	mov		x9, #((1<<CLINE_FLUSH_STRIDE)-1)
251	and		x2, x0, x9
252	bic		x0, x0, x9							// Cached aligned
253	add		x1, x1, x2
254	sub		x1, x1, #1
255	lsr		x1, x1, #(CLINE_FLUSH_STRIDE)		// Set cache line counter
256	dsb		sy
257L_cpcdr_loop:
258#if defined(APPLE_ARM64_ARCH_FAMILY)
259	// It may be tempting to clean the cache (dc cvac), but it's always a NOP on
260	// Apple hardware.
261	dc		civac, x0							// Clean & Invalidate dcache line to PoC
262#else /* defined(APPLE_ARM64_ARCH_FAMILY) */
263	dc		cvac, x0 							// Clean dcache line to PoC
264#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
265	add		x0, x0, #(1<<CLINE_FLUSH_STRIDE)	// Get next cache aligned addr
266	subs	x1, x1, #1							// Decrementer cache line counter
267	b.pl	L_cpcdr_loop						// Loop in counter not null
268	dsb		sy
269	ret
270
271/*
272 *	void CleanPoC_DcacheRegion(vm_offset_t va, size_t length)
273 *
274 *		Clean d-cache region to Point of Coherency
275 */
276	.text
277	.align 2
278	.globl EXT(CleanPoC_DcacheRegion)
279LEXT(CleanPoC_DcacheRegion)
280#if defined(APPLE_ARM64_ARCH_FAMILY)
281	/* "Fully Coherent." */
282	dsb		sy
283	ret
284#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
285	b EXT(CleanPoC_DcacheRegion_internal)
286#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
287
288
289/*
290 *	void CleanPoC_DcacheRegion_Force(vm_offset_t va, size_t length)
291 *
292 *		Clean d-cache region to Point of Coherency -  when you really
293 *		need to flush even on coherent platforms, e.g. panic log
294 */
295	.text
296	.align 2
297	.globl EXT(CleanPoC_DcacheRegion_Force)
298LEXT(CleanPoC_DcacheRegion_Force)
299#if defined(APPLE_ARM64_ARCH_FAMILY)
300	ARM64_STACK_PROLOG
301	PUSH_FRAME
302	stp		x0, x1, [sp, #-16]!
303	bl		EXT(_disable_preemption)
304	ldp		x0, x1, [sp], #16
305	bl		EXT(CleanPoC_DcacheRegion_Force_nopreempt)
306	bl		EXT(_enable_preemption)
307	POP_FRAME
308	ARM64_STACK_EPILOG
309#else
310	b		EXT(CleanPoC_DcacheRegion_internal)
311#endif // APPLE_ARM64_ARCH_FAMILY
312
313/*
314 *	void FlushPoC_Dcache(void)
315 *
316 *		Clean and Invalidate dcaches to Point of Coherency
317 */
318	.text
319	.align 2
320	.globl EXT(FlushPoC_Dcache)
321LEXT(FlushPoC_Dcache)
322#if defined(APPLE_ARM64_ARCH_FAMILY)
323	dsb sy
324	ret
325	/* "Fully Coherent." */
326#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
327	mrs		x0, CLIDR_EL1
328	ubfx		x0, x0, #24, #3	// extract CLIDR_EL1.LoC
329	DCACHE_SET_WAY cisw
330#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
331
332/*
333 * 	void Flush_Dcache(void)
334 *
335 * 		Clean and invalidate D-cache, all levels
336 */
337	.text
338	.align 2
339	.globl EXT(Flush_Dcache)
340LEXT(Flush_Dcache)
341	mov x0, #6 // Maximum allowable caching level (0-based)
342	DCACHE_SET_WAY cisw
343
344/*
345 * void FlushPoU_Dcache(void)
346 *
347 *		Flush D-cache to Point of Unification
348 */
349	.text
350	.align 2
351	.globl EXT(FlushPoU_Dcache)
352LEXT(FlushPoU_Dcache)
353#if defined(APPLE_ARM64_ARCH_FAMILY)
354	dsb sy
355	ret
356	/* "Fully Coherent." */
357#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
358	mrs		x0, CLIDR_EL1
359	ubfx		x0, x0, #21, 3	// extract CLIDR_EL1.LoUIS
360	DCACHE_SET_WAY	cisw
361#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
362
363/*
364 *	void FlushPoC_DcacheRegion(vm_offset_t va, unsigned length)
365 *
366 *		Clean and Invalidate d-cache region to Point of Coherency
367 */
368	.text
369	.align 2
370	.globl EXT(FlushPoC_DcacheRegion)
371LEXT(FlushPoC_DcacheRegion)
372#if defined(APPLE_ARM64_ARCH_FAMILY)
373	/* "Fully Coherent." */
374#else /* !defined(APPLE_ARM64_ARCH_FAMILY) */
375	mov		x9, #((1<<MMU_CLINE)-1)
376	and		x2, x0, x9
377	bic		x0, x0, x9							// Cached aligned
378	add		x1, x1, x2
379	sub		x1, x1, #1
380	lsr		x1, x1, #MMU_CLINE					// Set cache line counter
381	dmb		sy
382L_fpcdr_loop:
383	dc		civac, x0							// Clean invalidate dcache line to PoC
384	add		x0, x0, #(1<<MMU_CLINE)				// Get next cache aligned addr
385	subs	x1, x1, #1							// Decrementer cache line counter
386	b.pl	L_fpcdr_loop						// Loop in counter not null
387#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
388	dsb		sy
389	ret
390
391/*
392 *      void flush_dcache64(addr64_t addr, unsigned length, boolean_t phys)
393 */
394        .text
395        .align 2
396        .globl EXT(flush_dcache64)
397LEXT(flush_dcache64)
398	BRANCH_EXTERN    flush_dcache
399
400/*
401 *      void clean_dcache64(addr64_t addr, unsigned length, boolean_t phys)
402 */
403        .text
404        .align 2
405        .globl EXT(clean_dcache64)
406LEXT(clean_dcache64)
407	BRANCH_EXTERN    clean_dcache
408
409/*
410 *      void invalidate_icache(vm_offset_t va, unsigned length, boolean_t phys)
411 *      void invalidate_icache64(addr64_t va, unsigned length, boolean_t phys)
412 */
413        .text
414        .align 2
415        .globl EXT(invalidate_icache64)
416        .globl EXT(invalidate_icache)
417LEXT(invalidate_icache64)
418LEXT(invalidate_icache)
419	cmp     w2, #0								// Is it physical?
420	b.eq	Lcall_invalidate_worker
421	adrp	x2, _gPhysBase@page
422	add		x2, x2, _gPhysBase@pageoff
423	ldr		x2, [x2]
424	sub		x0, x0, x2
425	adrp	x2, _gVirtBase@page
426	add		x2, x2, _gVirtBase@pageoff
427	ldr		x2, [x2]
428	add		x0, x0, x2
429Lcall_invalidate_worker:
430	b		EXT(InvalidatePoU_IcacheRegion)
431
432
433/* vim: set ts=4: */
434