1/* 2 * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 * 28 * This file implements the following functions for the arm64 architecture: 29 * 30 * void bzero(void *buffer, size_t length); 31 * void __bzero(void *buffer, size_t length); 32 * void *memset(void *buffer, int value, size_t length); 33 * 34 * The first two zero-fill a buffer. The third fills the buffer with the low 35 * byte of its second argument. 36 */ 37 38#include "asm.h" 39 40.globl _bzero 41.globl ___bzero 42.globl _memset 43.globl _secure_memset 44 45/***************************************************************************** 46 * bzero entrypoint * 47 *****************************************************************************/ 48 49.text 50.align 4 51_bzero: 52___bzero: 53 ARM64_STACK_PROLOG 54 PUSH_FRAME 55 mov x2, x1 56 eor x1, x1, x1 57 mov x3, x0 58 cmp x2, #128 59 b.cc L_bzeroSmall 60 61/***************************************************************************** 62 * Large buffer zero engine * 63 *****************************************************************************/ 64 65L_bzeroLarge: 66// Write the first 64 bytes of the buffer without regard to alignment, then 67// advance x3 to point to a cacheline-aligned location within the buffer, and 68// decrement the length accordingly. 69 stp x1, x1, [x0] 70 stp x1, x1, [x0, #16] 71 stp x1, x1, [x0, #32] 72 stp x1, x1, [x0, #48] 73 add x3, x0, #64 74 and x3, x3, #-64 75 add x2, x2, x0 // end of buffer 76 add x4, x3, #64 // end of first cacheline to zero 77 subs x2, x2, x4 // if the end of the buffer comes first, jump 78 b.ls 1f // directly to the cleanup pass. 790: dc zva, x3 // zero cacheline 80 add x3, x3, #64 // increment pointer 81 subs x2, x2, #64 // decrement length 82 b.hi 0b 831: add x3, x3, x2 // back up pointer to (end of buffer) - 64. 84 stp x1, x1, [x3] // and store 64 bytes to reach end of buffer. 85 stp x1, x1, [x3, #16] 86 stp x1, x1, [x3, #32] 87 stp x1, x1, [x3, #48] 88 POP_FRAME 89 ARM64_STACK_EPILOG _bzero 90 91/***************************************************************************** 92 * Small buffer store engine * 93 *****************************************************************************/ 94 950: str x1, [x3],#8 96L_bzeroSmall: 97 subs x2, x2, #8 98 b.cs 0b 99 adds x2, x2, #8 100 b.eq 2f 1011: strb w1, [x3],#1 102 subs x2, x2, #1 103 b.ne 1b 1042: POP_FRAME 105 ARM64_STACK_EPILOG _bzero 106 107/***************************************************************************** 108 * memset entrypoint * 109 *****************************************************************************/ 110 111.align 4 112/* 113 * It is important that secure_memset remains defined in assembly to avoid 114 * compiler optimizations. 115 */ 116_secure_memset: 117_memset: 118 ARM64_STACK_PROLOG 119 PUSH_FRAME 120 and x1, x1, #0xff 121 orr x3, xzr,#0x0101010101010101 122 mul x1, x1, x3 123 mov x3, x0 124 cmp x2, #64 125 b.cc L_memsetSmall 126 127/***************************************************************************** 128 * Large buffer store engine * 129 *****************************************************************************/ 130 131L_memsetLarge: 132// Write the first 64 bytes of the buffer without regard to alignment, then 133// advance x3 to point to an aligned location within the buffer, and 134// decrement the length accordingly. 135 stp x1, x1, [x0] 136 add x3, x0, #16 137 and x3, x3, #-16 138 add x2, x2, x0 // end of buffer 139 add x4, x3, #64 // end of first aligned 64-byte store 140 subs x2, x2, x4 // if the end of the buffer comes first, jump 141 b.ls 1f // directly to the cleanup store. 1420: stnp x1, x1, [x3] 143 stnp x1, x1, [x3, #16] 144 stnp x1, x1, [x3, #32] 145 stnp x1, x1, [x3, #48] 146 add x3, x3, #64 147 subs x2, x2, #64 148 b.hi 0b 1491: add x3, x3, x2 // back up pointer to (end of buffer) - 64. 150 stp x1, x1, [x3] 151 stp x1, x1, [x3, #16] 152 stp x1, x1, [x3, #32] 153 stp x1, x1, [x3, #48] 154 POP_FRAME 155 ARM64_STACK_EPILOG _memset 156 157/***************************************************************************** 158 * Small buffer store engine * 159 *****************************************************************************/ 160 1610: str x1, [x3],#8 162L_memsetSmall: 163 subs x2, x2, #8 164 b.cs 0b 165 adds x2, x2, #8 166 b.eq 2f 1671: strb w1, [x3],#1 168 subs x2, x2, #1 169 b.ne 1b 1702: POP_FRAME 171 ARM64_STACK_EPILOG _memset 172 173