1/* 2 * Copyright (c) 2012 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 * 28 * This file implements the following functions for the arm64 architecture. 29 * 30 * void bcopy(const void * source, 31 * void * destination, 32 * size_t length); 33 * 34 * void *memmove(void * destination, 35 * const void * source, 36 * size_t n); 37 * 38 * void *memcpy(void * restrict destination, 39 * const void * restrict source, 40 * size_t n); 41 * 42 * All copy n successive bytes from source to destination. Memmove and memcpy 43 * return destination, whereas bcopy has no return value. Copying takes place 44 * as if it were through a temporary buffer -- after return destination 45 * contains exactly the bytes from source, even if the buffers overlap (this is 46 * not required of memcpy by the C standard; its behavior is undefined if the 47 * buffers overlap, but we are holding ourselves to the historical behavior of 48 * this function on MacOS). 49 */ 50 51#include "asm.h" 52 53.globl _bcopy 54.globl _ovbcopy 55.globl _memcpy 56.globl _memmove 57 58/***************************************************************************** 59 * Macros * 60 *****************************************************************************/ 61 62#define kSmallCopy 64 63 64/***************************************************************************** 65 * Entrypoints * 66 *****************************************************************************/ 67 68.text 69.align 5 70_bcopy: 71_ovbcopy: 72// Translate bcopy into memcpy by swapping the first and second arguments. 73 mov x3, x0 74 mov x0, x1 75 mov x1, x3 76 77.align 4 78_memcpy: 79_memmove: 80// Our preference is to copy the data in ascending address order, but if the 81// buffers overlap such that the beginning of the destination buffer aliases 82// the end of the source buffer, we need to copy in descending address order 83// instead to preserve the memmove semantics. We detect this case with the 84// test: 85// 86// destination - source < length (unsigned compare) 87// 88// If the address of the source buffer is higher than the address of the 89// destination buffer, this arithmetic can overflow, but the overflowed value 90// can only be smaller than length if the buffers do not overlap, so we don't 91// need to worry about false positives due to the overflow (they happen, but 92// only in cases where copying in either order is correct). 93 ARM64_STACK_PROLOG 94 PUSH_FRAME 95 sub x3, x0, x1 96 cmp x3, x2 97 b.cc L_reverse 98 mov x3, x0 // copy destination pointer 99 cmp x2, #(kSmallCopy) 100 b.cc L_forwardSmallCopy 101 102/***************************************************************************** 103 * Forward large copy * 104 *****************************************************************************/ 105 106// Load the first 32 bytes from src, and compute the number of bytes to the 107// first 32-byte aligned location in dst. Even though we are going to copy 108// 32 bytes, only those preceeding that 32-byte location "count" towards 109// reducing the length of the buffer or advancing the pointers. We will need 110// to issue the first load from the advanced src pointer BEFORE the store to 111// the unmodified dst pointer. 112 add x3, x3, #32 113 and x3, x3, #-32 // aligned dst 114 ldp x12,x13,[x1] 115 ldp x14,x15,[x1, #16] 116 sub x5, x3, x0 // bytes between original dst and aligned dst 117 add x1, x1, x5 // update src pointer 118 119// At this point, data in the following registers is in flight: 120// 121// x0 original dst pointer 122// x1 corresponding location in src buffer. 123// x2 length from aligned location in dst to end of buffer. This is 124// guaranteed to be >= (64 - 32). 125// x3 aligned location in dst buffer. 126// x12:x15 first 32 bytes of src buffer. 127// 128// We now load 32 bytes from x1, and store 32 bytes from x12:x15 to x3. The 129// store *may* overlap the first 32 bytes of the load, so in order to get 130// correct memmove semantics, the first 32 byte load must occur before the 131// store. 132// 133// After loading these 32 bytes, we advance x1, and decrement the length by 134// 64. If the remaining length of the buffer was less than 64, then we jump 135// directly to the cleanup path. 136 ldp x8, x9, [x1] 137 ldp x10,x11,[x1, #16] 138 add x1, x1, #32 139 sub x2, x2, x5 // update length 140 stp x12,x13,[x0] // initial unaligned store 141 stp x14,x15,[x0, #16] // initial unaligned store 142 subs x2, x2, #64 143 b.ls L_forwardCleanup 144 145L_forwardCopyLoop: 146// Main copy loop: 147// 148// 1. store the 32 bytes loaded in the previous loop iteration 149// 2. advance the destination pointer 150// 3. load the next 32 bytes 151// 4. advance the source pointer 152// 5. subtract 32 from the length 153// 154// The loop is terminated when 32 or fewer bytes remain to be loaded. Those 155// trailing 1-32 bytes will be copied in the loop cleanup. 156 stnp x8, x9, [x3] 157 stnp x10,x11,[x3, #16] 158 add x3, x3, #32 159 ldnp x8, x9, [x1] 160 ldnp x10,x11,[x1, #16] 161 add x1, x1, #32 162 subs x2, x2, #32 163 b.hi L_forwardCopyLoop 164 165L_forwardCleanup: 166// There are 32 bytes in x8-x11 that were loaded in the previous loop 167// iteration, which need to be stored to [x3,x3+32). In addition, between 168// 0 and 32 more bytes need to be copied from x1 to x3 + 32. The exact 169// number of bytes to copy is x2 + 32. Instead of using smaller conditional 170// copies, we simply copy 32 unaligned bytes from x1+x2 to 64+x3+x2. 171// This copy may overlap with the first store, so the loads must come before 172// the store of the data from the previous loop iteration. 173 add x1, x1, x2 174 ldp x12,x13,[x1] 175 ldp x14,x15,[x1, #16] 176 stp x8, x9, [x3] 177 stp x10,x11,[x3, #16] 178 add x3, x3, x2 179 stp x12,x13,[x3, #32] 180 stp x14,x15,[x3, #48] 181 POP_FRAME 182 ARM64_STACK_EPILOG 183 184/***************************************************************************** 185 * forward small copy * 186 *****************************************************************************/ 187 188// Copy one quadword at a time until less than 8 bytes remain to be copied. 189// At the point of entry to L_forwardSmallCopy, the "calling convention" 190// is as follows: 191// 192// x0 pointer to first byte of destination 193// x1 pointer to first byte of source 194// x2 length of buffers 195// x3 pointer to first byte of destination 1960: ldr x6, [x1],#8 197 str x6, [x3],#8 198L_forwardSmallCopy: 199 subs x2, x2, #8 200 b.cs 0b 201 adds x2, x2, #8 202 b.eq 2f 2031: ldrb w6, [x1],#1 204 strb w6, [x3],#1 205 subs x2, x2, #1 206 b.ne 1b 2072: POP_FRAME 208 ARM64_STACK_EPILOG 209 210/***************************************************************************** 211 * Reverse copy engines * 212 *****************************************************************************/ 213 214// The reverse copy engines are identical in every way to the forward copy 215// engines, except in that they do everything backwards. For this reason, they 216// are somewhat more sparsely commented than the forward copy loops. I have 217// tried to only comment things that might be somewhat surprising in how they 218// differ from the forward implementation. 219// 220// The one important thing to note is that (almost without fail), x1 and x3 221// will point to ONE BYTE BEYOND the "right-hand edge" of the active buffer 222// throughout these copy loops. They are initially advanced to that position 223// in the L_reverse jump island. Because of this, whereas the forward copy 224// loops generally follow a "copy data, then advance pointers" scheme, in the 225// reverse copy loops, we advance the pointers, then copy the data. 226 227L_reverse: 228// As a minor optimization, we early out if dst == src. 229 cbz x3, L_return 230// advance both pointers to the ends of their respective buffers before 231// jumping into the appropriate reverse copy loop. 232 add x4, x0, x2 233 add x1, x1, x2 234 cmp x2, #(kSmallCopy) 235 b.cc L_reverseSmallCopy 236 237/***************************************************************************** 238 * Reverse large copy * 239 *****************************************************************************/ 240 241 ldp x12,x13,[x1, #-16] 242 ldp x14,x15,[x1, #-32] 243 sub x3, x4, #1 // In the forward copy, we used dst+32 & -32 244 and x3, x3, #-32 // to find an aligned location in the dest 245 sub x5, x4, x3 // buffer. Here we use dst-1 & -32 instead, 246 sub x1, x1, x5 // because we are going backwards. 247 sub x2, x2, x5 248 ldp x8, x9, [x1, #-16] 249 ldp x10,x11,[x1, #-32] 250 stp x12,x13,[x4, #-16] 251 stp x14,x15,[x4, #-32] 252 sub x1, x1, #32 253 subs x2, x2, #64 254 b.ls L_reverseCleanup 255 256L_reverseCopyLoop: 257 stnp x8, x9, [x3, #-16] 258 stnp x10,x11,[x3, #-32] 259 sub x3, x3, #32 260 ldnp x8, x9, [x1, #-16] 261 ldnp x10,x11,[x1, #-32] 262 sub x1, x1, #32 263 subs x2, x2, #32 264 b.hi L_reverseCopyLoop 265 266L_reverseCleanup: 267 sub x1, x1, x2 268 ldp x12,x13,[x1, #-16] 269 ldp x14,x15,[x1, #-32] 270 stp x8, x9, [x3, #-16] 271 stp x10,x11,[x3, #-32] 272 stp x12,x13,[x0, #16] // In the forward copy, we need to compute the 273 stp x14,x15,[x0] // address of these stores, but here we already 274 POP_FRAME // have a pointer to the start of the buffer. 275 ARM64_STACK_EPILOG 276 277/***************************************************************************** 278 * reverse small copy * 279 *****************************************************************************/ 280 2810: ldr x6, [x1,#-8]! 282 str x6, [x4,#-8]! 283L_reverseSmallCopy: 284 subs x2, x2, #8 285 b.cs 0b 286 adds x2, x2, #8 287 b.eq 2f 2881: ldrb w6, [x1,#-1]! 289 strb w6, [x4,#-1]! 290 subs x2, x2, #1 291 b.ne 1b 2922: POP_FRAME 293 ARM64_STACK_EPILOG 294 295 296L_return: 297 POP_FRAME 298 ARM64_STACK_EPILOG 299