1 /* 2 * Copyright (c) 2024 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _SPECULATION_H_ 30 #define _SPECULATION_H_ 31 #include <mach/vm_types.h> 32 33 34 /* 35 * SPECULATION_GUARD_ZEROING_???_CC 36 * 37 * These macros perform a speculation safe version of the following pseudo-code: 38 * cc = cmp(cmp_1, cmp_2) 39 * if (cc) 40 * out = value 41 * else 42 * out = 0 43 * 44 * where "cc" is an ARM64 condition code (EQ, HS, etc.). 45 * 46 * Additionally, we provide four variants (for the ??? in the name): 47 * 1. XXX (value is 64-bits, cmp_1 and cmp_2 are 64-bits) 48 * 2. XWW (value is 64-bits, cmp_1 and cmp_2 are 32-bits) 49 * 3. WXX (value is 32-bits, cmp_1 and cmp_2 are 64-bits) 50 * 4. WWW (value is 32-bits, cmp_1 and cmp_2 are 32-bits) 51 * 52 * The guard condition must always resolve to true on the non-speculative path. 53 * Otherwise, a panic is triggered. 54 */ 55 56 /** 57 * Generate the zeroing speculation guard expression 58 * 59 * out: The output location for the guarded value 60 * value: The input value to guard 61 * cmp_1, cmp_2: The two operands to compare 62 * cmp_prefix: The ASM prefix for the registers in the compare instruction. For 63 * 64-bit operands, pass an empty string. For 32-bit operands, pass "w" 64 * cs_prefix: The ASM prefix for the registers in the select instruction. 65 */ 66 #define SPECULATION_GUARD_ZEROING_GEN(out, value, cmp_1, cmp_2, cc, cmp_prefix, cs_prefix) \ 67 { \ 68 bool speculation_guard_zeroing_out_valid; \ 69 __asm__ ( \ 70 "cmp %" cmp_prefix "[_cmp_1], %" cmp_prefix "[_cmp_2]\n" \ 71 "csel %" cs_prefix "[_out], %" cs_prefix "[_value], %" cs_prefix "[_zero], " cc "\n" \ 72 "cset %w[_valid], " cc "\n" \ 73 "csdb\n" \ 74 : [_out] "=r" (out), [_valid] "=r" (speculation_guard_zeroing_out_valid) \ 75 : [_cmp_1] "r" (cmp_1), [_cmp_2] "r" (cmp_2), [_value] "r" (value), [_zero] "rz" (0ULL) \ 76 : "cc" \ 77 ); \ 78 if (!speculation_guard_zeroing_out_valid) { \ 79 panic("Speculation guard failed non-spec"); \ 80 } \ 81 } 82 83 #define SPECULATION_GUARD_ZEROING_XXX(out, value, cmp_1, cmp_2, cc) \ 84 SPECULATION_GUARD_ZEROING_GEN(out, value, cmp_1, cmp_2, cc, "", "") 85 86 #define SPECULATION_GUARD_ZEROING_XWW(out, value, cmp_1, cmp_2, cc) \ 87 SPECULATION_GUARD_ZEROING_GEN(out, value, cmp_1, cmp_2, cc, "w", "") 88 89 #define SPECULATION_GUARD_ZEROING_WXX(out, value, cmp_1, cmp_2, cc) \ 90 SPECULATION_GUARD_ZEROING_GEN(out, value, cmp_1, cmp_2, cc, "", "w") 91 92 #define SPECULATION_GUARD_ZEROING_WWW(out, value, cmp_1, cmp_2, cc) \ 93 SPECULATION_GUARD_ZEROING_GEN(out, value, cmp_1, cmp_2, cc, "w", "w") 94 95 /* 96 * SPECULATION_GUARD_SELECT_???_CC 97 * 98 * These macros perform a speculation safe version of the following pseudo-code: 99 * cc = cmp(cmp_1, cmp_2) 100 * if (cc) 101 * value = sel_1 102 * else 103 * value = sel_2 104 * 105 * where "cc" is an ARM64 condition code (EQ, HS, etc.). 106 * 107 * Due to the limitations of macros/ASM, callers must provide both CC and !CC 108 * (the compliment, e.g. EQ and NE, HS and LO, etc.). Passing an incorrect 109 * compliment may result in incorrect or otherwise surprising behavior. 110 * 111 * Additionally, we provide four variants (for the ??? in the name): 112 * 1. XXX (value is 64-bits, cmp_1 and cmp_2 are 64-bits) 113 * 2. XWW (value is 64-bits, cmp_1 and cmp_2 are 32-bits) 114 * 3. WXX (value is 32-bits, cmp_1 and cmp_2 are 64-bits) 115 * 4. WWW (value is 32-bits, cmp_1 and cmp_2 are 32-bits) 116 * 117 * This guard has no requirements on non-speculative resolution. 118 */ 119 120 /** 121 * Generate the selection speculation guard expression 122 * 123 * output: The output value 124 * cmp_1, cmp_2: The two operands to compare 125 * sel_1, sel_2: The values to pick if cc or n_cc (respectively) 126 * cc, n_cc: The ARM64 condition code CC and its compliment !CC 127 * cmp_prefix: The ASM prefix for the registers in the compare instruction. For 128 * 64-bit operands, pass an empty string. For 32-bit operands, pass "w" 129 * cs_prefix: The ASM prefix for the registers in the select instruction. 130 */ 131 #define SPECULATION_GUARD_SELECT_GEN(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2, cmp_prefix, cs_prefix) \ 132 __asm__ ( \ 133 "cmp %" cmp_prefix "[_cmp_1], %" cmp_prefix "[_cmp_2]\n" \ 134 "csel %" cs_prefix "[_out], %" cs_prefix "[_sel_1], %" cs_prefix "[_sel_2], " cc "\n" \ 135 "csdb\n" \ 136 : [_out] "=r" (out) \ 137 : [_cmp_1] "r" (cmp_1), [_cmp_2] "r" (cmp_2), [_sel_1] "r" (sel_1), [_sel_2] "r" (sel_2), [_zero] "rz" (0ULL) \ 138 : "cc" \ 139 ); 140 141 #define SPECULATION_GUARD_SELECT_XXX(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2) \ 142 SPECULATION_GUARD_SELECT_GEN(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2, "", "") 143 144 #define SPECULATION_GUARD_SELECT_XWW(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2) \ 145 SPECULATION_GUARD_SELECT_GEN(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2, "w", "") 146 147 #define SPECULATION_GUARD_SELECT_WXX(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2) \ 148 SPECULATION_GUARD_SELECT_GEN(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2, "", "w") 149 150 #define SPECULATION_GUARD_SELECT_WWW(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2) \ 151 SPECULATION_GUARD_SELECT_GEN(out, cmp_1, cmp_2, cc, sel_1, n_cc, sel_2, "w", "w") 152 153 #endif /* _SPECULATION_H_ */ 154