1/* 2 * Copyright (c) 2020-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29/* 30 * extern int os_memcmp_mask_{16,32,48,64,80}B(const uint8_t *src1, 31 * const uint8_t *src2, const uint8_t *mask); 32 * 33 * This module implements fixed-length memory compare with mask routines, 34 * used mainly by the Skywalk networking subsystem. Each routine is called 35 * on every packet and therefore needs to be as efficient as possible. 36 * 37 * ARM64 kernel mode -- just like user mode -- no longer requires saving 38 * the vector registers, since it's done by the exception handler code. 39 */ 40 41#ifndef KERNEL 42#ifndef LIBSYSCALL_INTERFACE 43#error "LIBSYSCALL_INTERFACE not defined" 44#endif /* !LIBSYSCALL_INTERFACE */ 45#endif /* !KERNEL */ 46 47#define src1 x0 /* 1st arg */ 48#define src2 x1 /* 2nd arg */ 49#define mask x2 /* 3rd arg */ 50 51/* 52 * @abstract Compare 16-byte buffers src1 against src2, applying the byte 53 * masks to input data before comparison. 54 * 55 * @discussion 56 * Returns zero if the two buffers are identical after applying the byte 57 * masks, otherwise non-zero. 58 * 59 * @param src1 first 16-byte input buffer 60 * @param src2 second 16-byte input buffer 61 * @param byte_mask 16-byte byte mask applied before comparision 62 */ 63 .globl _os_memcmp_mask_16B 64 .text 65 .align 4 66_os_memcmp_mask_16B: 67 68 ld1.16b {v0}, [src1] 69 ld1.16b {v1}, [src2] 70 ld1.16b {v2}, [mask] 71 eor.16b v0, v0, v1 72 and.16b v0, v0, v2 73 umaxv b0, v0.16b 74 umov w0, v0.s[0] 75 76 ret lr 77 78/* 79 * @abstract Compare 32-byte buffers src1 against src2, applying the byte 80 * masks to input data before comparison. 81 * 82 * @discussion 83 * Returns zero if the two buffers are identical after applying the byte 84 * masks, otherwise non-zero. 85 * 86 * @param src1 first 32-byte input buffer 87 * @param src2 second 32-byte input buffer 88 * @param byte_mask 32-byte byte mask applied before comparision 89 */ 90 .globl _os_memcmp_mask_32B 91 .text 92 .align 4 93_os_memcmp_mask_32B: 94 95 ld1.16b {v0, v1}, [src1] 96 ld1.16b {v2, v3}, [src2] 97 ld1.16b {v4, v5}, [mask] 98 eor.16b v0, v0, v2 99 eor.16b v1, v1, v3 100 and.16b v0, v0, v4 101 and.16b v1, v1, v5 102 orr.16b v0, v0, v1 103 umaxv b0, v0.16b 104 umov w0, v0.s[0] 105 106 ret lr 107 108/* 109 * @abstract Compare 48-byte buffers src1 against src2, applying the byte 110 * masks to input data before comparison. 111 * 112 * @discussion 113 * Returns zero if the two buffers are identical after applying the byte 114 * masks, otherwise non-zero. 115 * 116 * @param src1 first 48-byte input buffer 117 * @param src2 second 48-byte input buffer 118 * @param byte_mask 48-byte byte mask applied before comparision 119 */ 120 .globl _os_memcmp_mask_48B 121 .text 122 .align 4 123_os_memcmp_mask_48B: 124 125 ld1.16b {v0, v1, v2}, [src1] 126 ld1.16b {v3, v4, v5}, [src2] 127 ld1.16b {v16, v17, v18}, [mask] 128 eor.16b v0, v0, v3 129 eor.16b v1, v1, v4 130 eor.16b v2, v2, v5 131 and.16b v0, v0, v16 132 and.16b v1, v1, v17 133 and.16b v2, v2, v18 134 orr.16b v0, v0, v1 135 orr.16b v0, v0, v2 136 umaxv b0, v0.16b 137 umov w0, v0.s[0] 138 139 ret lr 140 141/* 142 * @abstract Compare 64-byte buffers src1 against src2, applying the byte 143 * masks to input data before comparison. 144 * 145 * @discussion 146 * Returns zero if the two buffers are identical after applying the byte 147 * masks, otherwise non-zero. 148 * 149 * @param src1 first 64-byte input buffer 150 * @param src2 second 64-byte input buffer 151 * @param byte_mask 64-byte byte mask applied before comparision 152 */ 153 .globl _os_memcmp_mask_64B 154 .text 155 .align 4 156_os_memcmp_mask_64B: 157 158 ld1.16b {v0, v1, v2, v3}, [src1] 159 ld1.16b {v4, v5, v6, v7}, [src2] 160 ld1.16b {v16, v17, v18, v19}, [mask] 161 eor.16b v0, v0, v4 162 eor.16b v1, v1, v5 163 eor.16b v2, v2, v6 164 eor.16b v3, v3, v7 165 and.16b v0, v0, v16 166 and.16b v1, v1, v17 167 and.16b v2, v2, v18 168 and.16b v3, v3, v19 169 orr.16b v0, v0, v1 170 orr.16b v2, v2, v3 171 orr.16b v0, v0, v2 172 umaxv b0, v0.16b 173 umov w0, v0.s[0] 174 175 ret lr 176 177/* 178 * @abstract Compare 80-byte buffers src1 against src2, applying the byte 179 * masks to input data before comparison. 180 * 181 * @discussion 182 * Returns zero if the two buffers are identical after applying the byte 183 * masks, otherwise non-zero. 184 * 185 * @param src1 first 80-byte input buffer 186 * @param src2 second 80-byte input buffer 187 * @param byte_mask 80-byte byte mask applied before comparision 188 */ 189 .globl _os_memcmp_mask_80B 190 .text 191 .align 4 192_os_memcmp_mask_80B: 193 194 ld1.16b {v0, v1, v2, v3}, [src1], #64 195 ld1.16b {v4}, [src1] 196 ld1.16b {v16, v17, v18, v19}, [src2], #64 197 ld1.16b {v20}, [src2] 198 ld1.16b {v21, v22, v23, v24}, [mask], #64 199 ld1.16b {v25}, [mask] 200 eor.16b v0, v0, v16 201 eor.16b v1, v1, v17 202 eor.16b v2, v2, v18 203 eor.16b v3, v3, v19 204 eor.16b v4, v4, v20 205 and.16b v0, v0, v21 206 and.16b v1, v1, v22 207 and.16b v2, v2, v23 208 and.16b v3, v3, v24 209 and.16b v4, v4, v25 210 orr.16b v0, v0, v1 211 orr.16b v2, v2, v3 212 orr.16b v0, v0, v2 213 orr.16b v0, v0, v4 214 umaxv b0, v0.16b 215 umov w0, v0.s[0] 216 217 ret lr 218