1 /*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
4 *
5 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. The rights granted to you under the License
11 * may not be used to create, or enable the creation or redistribution of,
12 * unlawful or unlicensed copies of an Apple operating system, or to
13 * circumvent, violate, or enable the circumvention or violation of, any
14 * terms of an Apple operating system software license agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 *
19 * The Original Code and all software distributed under the License are
20 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
21 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
22 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
24 * Please see the License for the specific language governing rights and
25 * limitations under the License.
26 *
27 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
28 */
29 /*
30 * @OSF_COPYRIGHT@
31 */
32 /*
33 * @APPLE_FREE_COPYRIGHT@
34 */
35 /*
36 * File: arm/commpage/commpage.c
37 * Purpose: Set up and export a RO/RW page
38 */
39 #include <libkern/section_keywords.h>
40 #include <mach/mach_types.h>
41 #include <mach/machine.h>
42 #include <mach/vm_map.h>
43 #include <machine/cpu_capabilities.h>
44 #include <machine/commpage.h>
45 #include <machine/config.h>
46 #include <machine/pmap.h>
47 #include <vm/vm_kern.h>
48 #include <vm/vm_map.h>
49 #include <vm/vm_protos.h>
50 #include <ipc/ipc_port.h>
51 #include <arm/cpuid.h> /* for cpuid_info() & cache_info() */
52 #include <arm/misc_protos.h>
53 #include <arm/rtclock.h>
54 #include <libkern/OSAtomic.h>
55 #include <stdatomic.h>
56 #include <kern/remote_time.h>
57 #include <kern/smr.h>
58 #include <machine/atomic.h>
59 #include <machine/machine_remote_time.h>
60 #include <machine/machine_routines.h>
61
62 #include <sys/kdebug.h>
63 #include <sys/random.h>
64
65 #if CONFIG_ATM
66 #include <atm/atm_internal.h>
67 #endif
68
69 static int commpage_cpus( void );
70
71
72 static void commpage_init_cpu_capabilities( void );
73
74 SECURITY_READ_ONLY_LATE(vm_address_t) commPagePtr = 0;
75 SECURITY_READ_ONLY_LATE(vm_address_t) commpage_rw_addr = 0;
76 SECURITY_READ_ONLY_LATE(vm_address_t) commpage_kernel_ro_addr = 0;
77 SECURITY_READ_ONLY_LATE(uint64_t) _cpu_capabilities = 0;
78 SECURITY_READ_ONLY_LATE(vm_address_t) commpage_rw_text_addr = 0;
79
80 extern user64_addr_t commpage_text64_location;
81 extern user32_addr_t commpage_text32_location;
82
83 /* For sysctl access from BSD side */
84 extern int gARMv8Crc32;
85 extern int gARMv8Gpi;
86 extern int gARM_FEAT_FlagM;
87 extern int gARM_FEAT_FlagM2;
88 extern int gARM_FEAT_FHM;
89 extern int gARM_FEAT_DotProd;
90 extern int gARM_FEAT_SHA3;
91 extern int gARM_FEAT_RDM;
92 extern int gARM_FEAT_LSE;
93 extern int gARM_FEAT_SHA256;
94 extern int gARM_FEAT_SHA512;
95 extern int gARM_FEAT_SHA1;
96 extern int gARM_FEAT_AES;
97 extern int gARM_FEAT_PMULL;
98 extern int gARM_FEAT_SPECRES;
99 extern int gARM_FEAT_SB;
100 extern int gARM_FEAT_FRINTTS;
101 extern int gARM_FEAT_LRCPC;
102 extern int gARM_FEAT_LRCPC2;
103 extern int gARM_FEAT_FCMA;
104 extern int gARM_FEAT_JSCVT;
105 extern int gARM_FEAT_PAuth;
106 extern int gARM_FEAT_PAuth2;
107 extern int gARM_FEAT_FPAC;
108 extern int gARM_FEAT_FPACCOMBINE;
109 extern int gARM_FEAT_DPB;
110 extern int gARM_FEAT_DPB2;
111 extern int gARM_FEAT_BF16;
112 extern int gARM_FEAT_I8MM;
113 extern int gARM_FEAT_RPRES;
114 extern int gARM_FEAT_ECV;
115 extern int gARM_FEAT_LSE2;
116 extern int gARM_FEAT_CSV2;
117 extern int gARM_FEAT_CSV3;
118 extern int gARM_FEAT_DIT;
119 extern int gARM_AdvSIMD;
120 extern int gARM_AdvSIMD_HPFPCvt;
121 extern int gARM_FEAT_FP16;
122 extern int gARM_FEAT_SSBS;
123 extern int gARM_FEAT_BTI;
124 extern int gARM_FP_SyncExceptions;
125 extern int gARM_FEAT_AFP;
126
127 extern int gUCNormalMem;
128
129 void
commpage_populate(void)130 commpage_populate(void)
131 {
132 uint16_t c2;
133 int cpufamily;
134
135 // Create the data and the text commpage
136 vm_map_address_t kernel_data_addr, kernel_text_addr, kernel_ro_data_addr, user_text_addr;
137 pmap_create_commpages(&kernel_data_addr, &kernel_text_addr, &kernel_ro_data_addr, &user_text_addr);
138
139 commpage_rw_addr = kernel_data_addr;
140 commpage_rw_text_addr = kernel_text_addr;
141 commpage_kernel_ro_addr = kernel_ro_data_addr;
142 commPagePtr = (vm_address_t) _COMM_PAGE_BASE_ADDRESS;
143
144 #if __arm64__
145 commpage_text64_location = user_text_addr;
146 bcopy(_COMM_PAGE64_SIGNATURE_STRING, (void *)(_COMM_PAGE_SIGNATURE + _COMM_PAGE_RW_OFFSET),
147 MIN(_COMM_PAGE_SIGNATURELEN, strlen(_COMM_PAGE64_SIGNATURE_STRING)));
148 #endif
149
150 *((uint16_t*)(_COMM_PAGE_VERSION + _COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION;
151
152 commpage_init_cpu_capabilities();
153 commpage_set_timestamp(0, 0, 0, 0, 0);
154
155 if (_cpu_capabilities & kCache32) {
156 c2 = 32;
157 } else if (_cpu_capabilities & kCache64) {
158 c2 = 64;
159 } else if (_cpu_capabilities & kCache128) {
160 c2 = 128;
161 } else {
162 c2 = 0;
163 }
164
165 *((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE + _COMM_PAGE_RW_OFFSET)) = c2;
166
167 commpage_update_active_cpus();
168 cpufamily = cpuid_get_cpufamily();
169 *((uint8_t*)(_COMM_PAGE_CPU_CLUSTERS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) ml_get_cluster_count();
170 *((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max;
171 *((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.logical_cpu_max;
172 *((uint64_t*)(_COMM_PAGE_MEMORY_SIZE + _COMM_PAGE_RW_OFFSET)) = machine_info.max_mem;
173 *((uint32_t*)(_COMM_PAGE_CPUFAMILY + _COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily;
174 *((uint32_t*)(_COMM_PAGE_DEV_FIRM_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
175 *((uint32_t*)(_COMM_PAGE_DEV_FIRM + _COMM_PAGE_RO_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
176 *((uint8_t*)(_COMM_PAGE_USER_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = user_timebase_type();
177
178 // Populate logical CPU -> logical cluster table
179 ml_map_cpus_to_clusters((uint8_t*)(_COMM_PAGE_CPU_TO_CLUSTER + _COMM_PAGE_RW_OFFSET));
180
181 *((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK + _COMM_PAGE_RW_OFFSET)) = (uint8_t)user_cont_hwclock_allowed();
182 *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift;
183 *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT + _COMM_PAGE_RO_OFFSET)) = (uint8_t) page_shift;
184
185 #if __arm64__
186 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32;
187 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) page_shift_user32;
188 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
189 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
190 #endif /* __arm64__ */
191
192 commpage_update_timebase();
193 commpage_update_mach_continuous_time(0);
194
195 clock_sec_t secs;
196 clock_usec_t microsecs;
197 clock_get_boottime_microtime(&secs, µsecs);
198 commpage_update_boottime(secs * USEC_PER_SEC + microsecs);
199
200 /*
201 * set commpage approximate time to zero for initialization.
202 * scheduler shall populate correct value before running user thread
203 */
204 *((uint64_t *)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET)) = 0;
205 #ifdef CONFIG_MACH_APPROXIMATE_TIME
206 *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 1;
207 #else
208 *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 0;
209 #endif
210
211 commpage_update_kdebug_state();
212
213 #if CONFIG_ATM
214 commpage_update_atm_diagnostic_config(atm_get_diagnostic_config());
215 #endif
216
217
218 *((uint64_t*)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET)) = BT_RESET_SENTINEL_TS;
219
220 #if CONFIG_QUIESCE_COUNTER
221 cpu_quiescent_set_storage((_Atomic uint64_t *)(_COMM_PAGE_CPU_QUIESCENT_COUNTER +
222 _COMM_PAGE_RW_OFFSET));
223 #endif /* CONFIG_QUIESCE_COUNTER */
224
225 /*
226 * Set random values for targets in Apple Security Bounty
227 * addr should be unmapped for userland processes
228 * kaddr should be unmapped for kernel
229 */
230 uint64_t asb_value, asb_addr, asb_kvalue, asb_kaddr;
231 uint64_t asb_rand_vals[] = {
232 0x93e78adcded4d3d5, 0xd16c5b76ad99bccf, 0x67dfbbd12c4a594e, 0x7365636e6f6f544f,
233 0x239a974c9811e04b, 0xbf60e7fa45741446, 0x8acf5210b466b05, 0x67dfbbd12c4a594e
234 };
235 const int nrandval = sizeof(asb_rand_vals) / sizeof(asb_rand_vals[0]);
236 uint8_t randidx;
237
238 read_random(&randidx, sizeof(uint8_t));
239 asb_value = asb_rand_vals[randidx++ % nrandval];
240 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_VALUE + _COMM_PAGE_RW_OFFSET)) = asb_value;
241
242 // userspace faulting address should be > MACH_VM_MAX_ADDRESS
243 asb_addr = asb_rand_vals[randidx++ % nrandval];
244 uint64_t user_min = MACH_VM_MAX_ADDRESS;
245 uint64_t user_max = UINT64_MAX;
246 asb_addr %= (user_max - user_min);
247 asb_addr += user_min;
248 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_ADDRESS + _COMM_PAGE_RW_OFFSET)) = asb_addr;
249
250 asb_kvalue = asb_rand_vals[randidx++ % nrandval];
251 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_KERN_VALUE + _COMM_PAGE_RW_OFFSET)) = asb_kvalue;
252
253 // kernel faulting address should be < VM_MIN_KERNEL_ADDRESS
254 asb_kaddr = asb_rand_vals[randidx++ % nrandval];
255 uint64_t kernel_min = 0x0LL;
256 uint64_t kernel_max = VM_MIN_KERNEL_ADDRESS;
257 asb_kaddr %= (kernel_max - kernel_min);
258 asb_kaddr += kernel_min;
259 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_KERN_ADDRESS + _COMM_PAGE_RW_OFFSET)) = asb_kaddr;
260 }
261
262 #define COMMPAGE_TEXT_SEGMENT "__TEXT_EXEC"
263 #define COMMPAGE_TEXT_SECTION "__commpage_text"
264
265 /* Get a pointer to the start of the ARM PFZ code section. This macro tell the
266 * linker that the storage for the variable here is at the start of the section */
267 extern char commpage_text_start[]
268 __SECTION_START_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
269
270 /* Get a pointer to the end of the ARM PFZ code section. This macro tell the
271 * linker that the storage for the variable here is at the end of the section */
272 extern char commpage_text_end[]
273 __SECTION_END_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
274
275 /* This is defined in the commpage text section as a symbol at the start of the preemptible
276 * functions */
277 extern char commpage_text_preemptible_functions;
278
279 #if CONFIG_ARM_PFZ
280 static size_t size_of_pfz = 0;
281 #endif
282
283 /* This is the opcode for brk #666 */
284 #define BRK_666_OPCODE 0xD4205340
285
286 void
commpage_text_populate(void)287 commpage_text_populate(void)
288 {
289 #if CONFIG_ARM_PFZ
290 size_t size_of_commpage_text = commpage_text_end - commpage_text_start;
291 if (size_of_commpage_text == 0) {
292 panic("ARM comm page text section %s,%s missing", COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
293 }
294 assert(size_of_commpage_text <= PAGE_SIZE);
295 assert(size_of_commpage_text > 0);
296
297 /* Get the size of the PFZ half of the comm page text section. */
298 size_of_pfz = &commpage_text_preemptible_functions - commpage_text_start;
299
300 // Copy the code segment of comm page text section into the PFZ
301 memcpy((void *) _COMM_PAGE64_TEXT_START_ADDRESS, (void *) commpage_text_start, size_of_commpage_text);
302
303 // Make sure to populate the rest of it with brk 666 so that undefined code
304 // doesn't get run
305 memset((char *) _COMM_PAGE64_TEXT_START_ADDRESS + size_of_commpage_text, BRK_666_OPCODE,
306 PAGE_SIZE - size_of_commpage_text);
307 #endif
308 }
309
310 uint32_t
commpage_is_in_pfz64(addr64_t addr64)311 commpage_is_in_pfz64(addr64_t addr64)
312 {
313 #if CONFIG_ARM_PFZ
314 if ((addr64 >= commpage_text64_location) &&
315 (addr64 < (commpage_text64_location + size_of_pfz))) {
316 return 1;
317 } else {
318 return 0;
319 }
320 #else
321 #pragma unused (addr64)
322 return 0;
323 #endif
324 }
325
326
327 void
commpage_set_timestamp(uint64_t tbr,uint64_t secs,uint64_t frac,uint64_t scale,uint64_t tick_per_sec)328 commpage_set_timestamp(
329 uint64_t tbr,
330 uint64_t secs,
331 uint64_t frac,
332 uint64_t scale,
333 uint64_t tick_per_sec)
334 {
335 new_commpage_timeofday_data_t *commpage_timeofday_datap;
336
337 if (commPagePtr == 0) {
338 return;
339 }
340
341 commpage_timeofday_datap = (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA + _COMM_PAGE_RW_OFFSET);
342
343 commpage_timeofday_datap->TimeStamp_tick = 0x0ULL;
344
345 __builtin_arm_dmb(DMB_ISH);
346
347 commpage_timeofday_datap->TimeStamp_sec = secs;
348 commpage_timeofday_datap->TimeStamp_frac = frac;
349 commpage_timeofday_datap->Ticks_scale = scale;
350 commpage_timeofday_datap->Ticks_per_sec = tick_per_sec;
351
352 __builtin_arm_dmb(DMB_ISH);
353
354 commpage_timeofday_datap->TimeStamp_tick = tbr;
355
356 }
357
358 /*
359 * Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure()
360 */
361
362 void
commpage_set_memory_pressure(unsigned int pressure)363 commpage_set_memory_pressure(
364 unsigned int pressure )
365 {
366 if (commPagePtr == 0) {
367 return;
368 }
369 *((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE + _COMM_PAGE_RW_OFFSET)) = pressure;
370 }
371
372 /*
373 * Determine number of CPUs on this system.
374 */
375 static int
commpage_cpus(void)376 commpage_cpus( void )
377 {
378 int cpus;
379
380 cpus = machine_info.max_cpus;
381
382 if (cpus == 0) {
383 panic("commpage cpus==0");
384 }
385 if (cpus > 0xFF) {
386 cpus = 0xFF;
387 }
388
389 return cpus;
390 }
391
392 uint64_t
_get_cpu_capabilities(void)393 _get_cpu_capabilities(void)
394 {
395 return _cpu_capabilities;
396 }
397
398 vm_address_t
_get_commpage_priv_address(void)399 _get_commpage_priv_address(void)
400 {
401 return commpage_rw_addr;
402 }
403
404 vm_address_t
_get_commpage_ro_address(void)405 _get_commpage_ro_address(void)
406 {
407 return commpage_kernel_ro_addr;
408 }
409
410 vm_address_t
_get_commpage_text_priv_address(void)411 _get_commpage_text_priv_address(void)
412 {
413 return commpage_rw_text_addr;
414 }
415
416 #if defined(__arm64__)
417 /**
418 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR0_EL1
419 */
420 static void
commpage_init_arm_optional_features_isar0(uint64_t * commpage_bits)421 commpage_init_arm_optional_features_isar0(uint64_t *commpage_bits)
422 {
423 uint64_t bits = 0;
424 uint64_t isar0 = __builtin_arm_rsr64("ID_AA64ISAR0_EL1");
425
426 if ((isar0 & ID_AA64ISAR0_EL1_TS_MASK) >= ID_AA64ISAR0_EL1_TS_FLAGM_EN) {
427 gARM_FEAT_FlagM = 1;
428 bits |= kHasFEATFlagM;
429 }
430 if ((isar0 & ID_AA64ISAR0_EL1_TS_MASK) >= ID_AA64ISAR0_EL1_TS_FLAGM2_EN) {
431 gARM_FEAT_FlagM2 = 1;
432 bits |= kHasFEATFlagM2;
433 }
434 if ((isar0 & ID_AA64ISAR0_EL1_FHM_MASK) >= ID_AA64ISAR0_EL1_FHM_8_2) {
435 gARM_FEAT_FHM = 1;
436 bits |= kHasFeatFHM;
437 }
438 if ((isar0 & ID_AA64ISAR0_EL1_DP_MASK) >= ID_AA64ISAR0_EL1_DP_EN) {
439 gARM_FEAT_DotProd = 1;
440 bits |= kHasFeatDotProd;
441 }
442 if ((isar0 & ID_AA64ISAR0_EL1_SHA3_MASK) >= ID_AA64ISAR0_EL1_SHA3_EN) {
443 gARM_FEAT_SHA3 = 1;
444 bits |= kHasFeatSHA3;
445 }
446 if ((isar0 & ID_AA64ISAR0_EL1_RDM_MASK) >= ID_AA64ISAR0_EL1_RDM_EN) {
447 gARM_FEAT_RDM = 1;
448 bits |= kHasFeatRDM;
449 }
450 if ((isar0 & ID_AA64ISAR0_EL1_ATOMIC_MASK) >= ID_AA64ISAR0_EL1_ATOMIC_8_1) {
451 gARM_FEAT_LSE = 1;
452 bits |= kHasFeatLSE;
453 }
454 if ((isar0 & ID_AA64ISAR0_EL1_SHA2_MASK) >= ID_AA64ISAR0_EL1_SHA2_512_EN) {
455 gARM_FEAT_SHA512 = 1;
456 bits |= kHasFeatSHA512;
457 }
458 if ((isar0 & ID_AA64ISAR0_EL1_CRC32_MASK) == ID_AA64ISAR0_EL1_CRC32_EN) {
459 gARMv8Crc32 = 1;
460 bits |= kHasARMv8Crc32;
461 }
462
463 #if __ARM_V8_CRYPTO_EXTENSIONS__
464 /**
465 * T7000 has a bug in the ISAR0 register that reports that PMULL is not
466 * supported when it actually is. To work around this, for all of the crypto
467 * extensions, just check if they're supported using the board_config.h
468 * values.
469 */
470 gARM_FEAT_PMULL = 1;
471 gARM_FEAT_SHA1 = 1;
472 gARM_FEAT_AES = 1;
473 gARM_FEAT_SHA256 = 1;
474 bits |= kHasARMv8Crypto;
475 #endif /* __ARM_V8_CRYPTO_EXTENSIONS__ */
476
477 *commpage_bits |= bits;
478 }
479
480 /**
481 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR1_EL1
482 */
483 static void
commpage_init_arm_optional_features_isar1(uint64_t * commpage_bits)484 commpage_init_arm_optional_features_isar1(uint64_t *commpage_bits)
485 {
486 uint64_t bits = 0;
487 uint64_t isar1 = __builtin_arm_rsr64("ID_AA64ISAR1_EL1");
488 uint64_t sctlr = __builtin_arm_rsr64("SCTLR_EL1");
489
490 if ((isar1 & ID_AA64ISAR1_EL1_SPECRES_MASK) >= ID_AA64ISAR1_EL1_SPECRES_EN &&
491 sctlr & SCTLR_EnRCTX) {
492 gARM_FEAT_SPECRES = 1;
493 bits |= kHasFeatSPECRES;
494 }
495 if ((isar1 & ID_AA64ISAR1_EL1_SB_MASK) >= ID_AA64ISAR1_EL1_SB_EN) {
496 gARM_FEAT_SB = 1;
497 bits |= kHasFeatSB;
498 }
499 if ((isar1 & ID_AA64ISAR1_EL1_FRINTTS_MASK) >= ID_AA64ISAR1_EL1_FRINTTS_EN) {
500 gARM_FEAT_FRINTTS = 1;
501 bits |= kHasFeatFRINTTS;
502 }
503 if ((isar1 & ID_AA64ISAR1_EL1_GPI_MASK) >= ID_AA64ISAR1_EL1_GPI_EN) {
504 gARMv8Gpi = 1;
505 bits |= kHasArmv8GPI;
506 }
507 if ((isar1 & ID_AA64ISAR1_EL1_LRCPC_MASK) >= ID_AA64ISAR1_EL1_LRCPC_EN) {
508 gARM_FEAT_LRCPC = 1;
509 bits |= kHasFeatLRCPC;
510 }
511 if ((isar1 & ID_AA64ISAR1_EL1_LRCPC_MASK) >= ID_AA64ISAR1_EL1_LRCP2C_EN) {
512 gARM_FEAT_LRCPC2 = 1;
513 bits |= kHasFeatLRCPC2;
514 }
515 if ((isar1 & ID_AA64ISAR1_EL1_FCMA_MASK) >= ID_AA64ISAR1_EL1_FCMA_EN) {
516 gARM_FEAT_FCMA = 1;
517 bits |= kHasFeatFCMA;
518 }
519 if ((isar1 & ID_AA64ISAR1_EL1_JSCVT_MASK) >= ID_AA64ISAR1_EL1_JSCVT_EN) {
520 gARM_FEAT_JSCVT = 1;
521 bits |= kHasFeatJSCVT;
522 }
523 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_PAuth_EN) {
524 gARM_FEAT_PAuth = 1;
525 bits |= kHasFeatPAuth;
526 }
527 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_PAuth2_EN) {
528 gARM_FEAT_PAuth2 = 1;
529 }
530 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_FPAC_EN) {
531 gARM_FEAT_FPAC = 1;
532 }
533 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_FPACCOMBINE) {
534 gARM_FEAT_FPACCOMBINE = 1;
535 }
536 if ((isar1 & ID_AA64ISAR1_EL1_DPB_MASK) >= ID_AA64ISAR1_EL1_DPB_EN) {
537 gARM_FEAT_DPB = 1;
538 bits |= kHasFeatDPB;
539 }
540 if ((isar1 & ID_AA64ISAR1_EL1_DPB_MASK) >= ID_AA64ISAR1_EL1_DPB2_EN) {
541 gARM_FEAT_DPB2 = 1;
542 bits |= kHasFeatDPB2;
543 }
544 if ((isar1 & ID_AA64ISAR1_EL1_BF16_MASK) >= ID_AA64ISAR1_EL1_BF16_EN) {
545 gARM_FEAT_BF16 = 1;
546 }
547 if ((isar1 & ID_AA64ISAR1_EL1_I8MM_MASK) >= ID_AA64ISAR1_EL1_I8MM_EN) {
548 gARM_FEAT_I8MM = 1;
549 }
550
551 *commpage_bits |= bits;
552 }
553
554 /**
555 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR2_EL1
556 */
557 static void
commpage_init_arm_optional_features_isar2(void)558 commpage_init_arm_optional_features_isar2(void)
559 {
560 uint64_t isar2 = __builtin_arm_rsr64("ID_AA64ISAR2_EL1");
561
562 if ((isar2 & ID_AA64ISAR2_EL1_RPRES_MASK) >= ID_AA64ISAR2_EL1_RPRES_EN) {
563 gARM_FEAT_RPRES = 1;
564 }
565 }
566
567 /**
568 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64MMFR0_EL1
569 */
570 static void
commpage_init_arm_optional_features_mmfr0(uint64_t * commpage_bits)571 commpage_init_arm_optional_features_mmfr0(uint64_t *commpage_bits)
572 {
573 uint64_t bits = 0;
574 uint64_t mmfr0 = __builtin_arm_rsr64("ID_AA64MMFR0_EL1");
575
576 if ((mmfr0 & ID_AA64MMFR0_EL1_ECV_MASK) >= ID_AA64MMFR0_EL1_ECV_EN) {
577 gARM_FEAT_ECV = 1;
578 }
579
580 *commpage_bits |= bits;
581 }
582
583 /**
584 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64MMFR2_EL1
585 */
586 static void
commpage_init_arm_optional_features_mmfr2(uint64_t * commpage_bits)587 commpage_init_arm_optional_features_mmfr2(uint64_t *commpage_bits)
588 {
589 uint64_t bits = 0;
590 uint64_t mmfr2 = __builtin_arm_rsr64("ID_AA64MMFR2_EL1");
591
592 if ((mmfr2 & ID_AA64MMFR2_EL1_AT_MASK) >= ID_AA64MMFR2_EL1_AT_LSE2_EN) {
593 gARM_FEAT_LSE2 = 1;
594 bits |= kHasFeatLSE2;
595 }
596
597 *commpage_bits |= bits;
598 }
599
600 /**
601 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64PFR0_EL1
602 */
603 static void
commpage_init_arm_optional_features_pfr0(uint64_t * commpage_bits)604 commpage_init_arm_optional_features_pfr0(uint64_t *commpage_bits)
605 {
606 uint64_t bits = 0;
607 uint64_t pfr0 = __builtin_arm_rsr64("ID_AA64PFR0_EL1");
608
609 if ((pfr0 & ID_AA64PFR0_EL1_CSV3_MASK) >= ID_AA64PFR0_EL1_CSV3_EN) {
610 gARM_FEAT_CSV3 = 1;
611 bits |= kHasFeatCSV3;
612 }
613 if ((pfr0 & ID_AA64PFR0_EL1_CSV2_MASK) >= ID_AA64PFR0_EL1_CSV2_EN) {
614 gARM_FEAT_CSV2 = 1;
615 bits |= kHasFeatCSV2;
616 }
617 if ((pfr0 & ID_AA64PFR0_EL1_DIT_MASK) >= ID_AA64PFR0_EL1_DIT_EN) {
618 gARM_FEAT_DIT = 1;
619 bits |= kHasFeatDIT;
620 }
621 if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) != ID_AA64PFR0_EL1_AdvSIMD_DIS) {
622 gARM_AdvSIMD = 1;
623 bits |= kHasAdvSIMD;
624 if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) >= ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT) {
625 gARM_AdvSIMD_HPFPCvt = 1;
626 bits |= kHasAdvSIMD_HPFPCvt;
627 }
628 if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) >= ID_AA64PFR0_EL1_AdvSIMD_FP16) {
629 gARM_FEAT_FP16 = 1;
630 bits |= kHasFeatFP16;
631 }
632 }
633
634 *commpage_bits |= bits;
635 }
636
637 /**
638 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64PFR1_EL1
639 */
640 static void
commpage_init_arm_optional_features_pfr1(uint64_t * commpage_bits)641 commpage_init_arm_optional_features_pfr1(uint64_t *commpage_bits)
642 {
643 uint64_t pfr1 = __builtin_arm_rsr64("ID_AA64PFR1_EL1");
644
645 if ((pfr1 & ID_AA64PFR1_EL1_SSBS_MASK) >= ID_AA64PFR1_EL1_SSBS_EN) {
646 gARM_FEAT_SSBS = 1;
647 }
648
649 if ((pfr1 & ID_AA64PFR1_EL1_BT_MASK) >= ID_AA64PFR1_EL1_BT_EN) {
650 gARM_FEAT_BTI = 1;
651 }
652
653 #pragma unused(commpage_bits)
654 }
655
656
657 static void
commpage_init_arm_optional_features_mmfr1(uint64_t * commpage_bits)658 commpage_init_arm_optional_features_mmfr1(uint64_t *commpage_bits)
659 {
660 uint64_t bits = 0;
661 const uint64_t mmfr1 = __builtin_arm_rsr64("ID_AA64MMFR1_EL1");
662
663 if ((mmfr1 & ID_AA64MMFR1_EL1_AFP_MASK) == ID_AA64MMFR1_EL1_AFP_EN) {
664 gARM_FEAT_AFP = 1;
665 bits |= kHasFeatAFP;
666 }
667
668 *commpage_bits |= bits;
669 }
670
671 /**
672 * Read the system register @name, attempt to set set bits of @mask if not
673 * already, test if bits were actually set, reset the register to its
674 * previous value if required, and 'return' @mask with only bits that
675 * were successfully set (or already set) in the system register. */
676 #define _test_sys_bits(name, mask) ({ \
677 const uint64_t src = __builtin_arm_rsr64(#name); \
678 uint64_t test = src | mask; \
679 if (test != src) { \
680 __builtin_arm_wsr64(#name, test); \
681 test = __builtin_arm_rsr64(#name); \
682 if (test != src) { \
683 __builtin_arm_wsr64(#name, src); \
684 }\
685 } \
686 mask & test; \
687 })
688
689 /**
690 * Reports whether FPU exceptions are supported.
691 * Possible FPU exceptions are :
692 * - input denormal;
693 * - inexact;
694 * - underflow;
695 * - overflow;
696 * - divide by 0;
697 * - invalid operation.
698 *
699 * Any of those can be supported or not but for now, we consider that
700 * it all or nothing : FPU exceptions support flag set <=> all 6 exceptions
701 * a supported.
702 */
703 static void
commpage_init_arm_optional_features_fpcr(uint64_t * commpage_bits)704 commpage_init_arm_optional_features_fpcr(uint64_t *commpage_bits)
705 {
706 uint64_t support_mask = FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE |
707 FPCR_DZE | FPCR_IOE;
708 uint64_t FPCR_bits = _test_sys_bits(FPCR, support_mask);
709 if (FPCR_bits == support_mask) {
710 gARM_FP_SyncExceptions = 1;
711 *commpage_bits |= kHasFP_SyncExceptions;
712 }
713 }
714
715 /**
716 * Initializes all commpage entries and sysctls for ARM64 optional features accessible from EL0.
717 */
718 static void
commpage_init_arm_optional_features(uint64_t * commpage_bits)719 commpage_init_arm_optional_features(uint64_t *commpage_bits)
720 {
721 commpage_init_arm_optional_features_isar0(commpage_bits);
722 commpage_init_arm_optional_features_isar1(commpage_bits);
723 commpage_init_arm_optional_features_isar2();
724 commpage_init_arm_optional_features_mmfr0(commpage_bits);
725 commpage_init_arm_optional_features_mmfr1(commpage_bits);
726 commpage_init_arm_optional_features_mmfr2(commpage_bits);
727 commpage_init_arm_optional_features_pfr0(commpage_bits);
728 commpage_init_arm_optional_features_pfr1(commpage_bits);
729 commpage_init_arm_optional_features_fpcr(commpage_bits);
730 }
731 #endif /* __arm64__ */
732
733 /*
734 * Initialize _cpu_capabilities vector
735 */
736 static void
commpage_init_cpu_capabilities(void)737 commpage_init_cpu_capabilities( void )
738 {
739 uint64_t bits;
740 int cpus;
741 ml_cpu_info_t cpu_info;
742
743 bits = 0;
744 ml_cpu_get_info(&cpu_info);
745
746 switch (cpu_info.cache_line_size) {
747 case 128:
748 bits |= kCache128;
749 break;
750 case 64:
751 bits |= kCache64;
752 break;
753 case 32:
754 bits |= kCache32;
755 break;
756 default:
757 break;
758 }
759 cpus = commpage_cpus();
760
761 if (cpus == 1) {
762 bits |= kUP;
763 }
764
765 bits |= (cpus << kNumCPUsShift);
766
767 bits |= kFastThreadLocalStorage; // TPIDRURO for TLS
768
769 bits |= kHasVfp;
770
771 #if defined(__arm64__)
772 bits |= kHasFMA;
773 #endif
774 bits |= kHasEvent;
775 #ifdef __arm64__
776 commpage_init_arm_optional_features(&bits);
777 #endif
778
779
780
781 #if HAS_UCNORMAL_MEM
782 gUCNormalMem = 1;
783 bits |= kHasUCNormalMemory;
784 #endif
785
786 _cpu_capabilities = bits;
787
788 *((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES + _COMM_PAGE_RW_OFFSET)) = (uint32_t)_cpu_capabilities;
789 *((uint64_t *)(_COMM_PAGE_CPU_CAPABILITIES64 + _COMM_PAGE_RW_OFFSET)) = _cpu_capabilities;
790
791 }
792
793 /*
794 * Updated every time a logical CPU goes offline/online
795 */
796 void
commpage_update_active_cpus(void)797 commpage_update_active_cpus(void)
798 {
799 if (!commPagePtr) {
800 return;
801 }
802 *((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t)processor_avail_count;
803
804 }
805
806 /*
807 * Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace)
808 */
809 void
commpage_update_timebase(void)810 commpage_update_timebase(void)
811 {
812 if (commPagePtr) {
813 *((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET + _COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime;
814 }
815 }
816
817 /*
818 * Update the commpage with current kdebug state: whether tracing is enabled, a
819 * typefilter is present, and continuous time should be used for timestamps.
820 *
821 * Disregards configuration and set to 0 if tracing is disabled.
822 */
823 void
commpage_update_kdebug_state(void)824 commpage_update_kdebug_state(void)
825 {
826 if (commPagePtr) {
827 uint32_t state = kdebug_commpage_state();
828 *((volatile uint32_t *)(_COMM_PAGE_KDEBUG_ENABLE + _COMM_PAGE_RW_OFFSET)) = state;
829 }
830 }
831
832 /* Ditto for atm_diagnostic_config */
833 void
commpage_update_atm_diagnostic_config(uint32_t diagnostic_config)834 commpage_update_atm_diagnostic_config(uint32_t diagnostic_config)
835 {
836 if (commPagePtr) {
837 *((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG + _COMM_PAGE_RW_OFFSET)) = diagnostic_config;
838 }
839 }
840
841 /*
842 * Update the commpage data with the state of multiuser mode for
843 * this device. Allowing various services in userspace to avoid
844 * IPC in the (more common) non-multiuser environment.
845 */
846 void
commpage_update_multiuser_config(uint32_t multiuser_config)847 commpage_update_multiuser_config(uint32_t multiuser_config)
848 {
849 if (commPagePtr) {
850 *((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG + _COMM_PAGE_RW_OFFSET)) = multiuser_config;
851 }
852 }
853
854 /*
855 * update the commpage data for
856 * last known value of mach_absolute_time()
857 */
858
859 void
commpage_update_mach_approximate_time(uint64_t abstime)860 commpage_update_mach_approximate_time(uint64_t abstime)
861 {
862 #ifdef CONFIG_MACH_APPROXIMATE_TIME
863 if (!commPagePtr) {
864 return;
865 }
866
867 uint64_t *approx_time_base = (uint64_t *)(uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET);
868
869 uint64_t saved_data = os_atomic_load_wide(approx_time_base, relaxed);
870 if (saved_data < abstime) {
871 /*
872 * ignore the success/fail return value assuming that
873 * if the value has been updated since we last read it,
874 * someone else has written a timestamp that is new enough.
875 */
876 __unused bool ret = os_atomic_cmpxchg(approx_time_base,
877 saved_data, abstime, relaxed);
878 }
879
880
881 #else /* CONFIG_MACH_APPROXIMATE_TIME */
882 #pragma unused (abstime)
883 #endif
884 }
885
886 /*
887 * update the commpage data's total system sleep time for
888 * userspace call to mach_continuous_time()
889 */
890 void
commpage_update_mach_continuous_time(uint64_t sleeptime)891 commpage_update_mach_continuous_time(uint64_t sleeptime)
892 {
893 if (!commPagePtr) {
894 return;
895 }
896
897 uint64_t *cont_time_base = (uint64_t *)(uintptr_t)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET);
898
899 os_atomic_store_wide(cont_time_base, sleeptime, relaxed);
900
901 }
902
903 void
commpage_update_mach_continuous_time_hw_offset(uint64_t offset)904 commpage_update_mach_continuous_time_hw_offset(uint64_t offset)
905 {
906 *((uint64_t *)(_COMM_PAGE_CONT_HW_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = offset;
907 }
908
909 /*
910 * update the commpage's value for the boot time
911 */
912 void
commpage_update_boottime(uint64_t value)913 commpage_update_boottime(uint64_t value)
914 {
915 if (!commPagePtr) {
916 return;
917 }
918
919 uint64_t *boottime_usec = (uint64_t *)(uintptr_t)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET);
920
921 os_atomic_store_wide(boottime_usec, value, relaxed);
922
923 }
924
925 /*
926 * set the commpage's remote time params for
927 * userspace call to mach_bridge_remote_time()
928 */
929 void
commpage_set_remotetime_params(double rate,uint64_t base_local_ts,uint64_t base_remote_ts)930 commpage_set_remotetime_params(double rate, uint64_t base_local_ts, uint64_t base_remote_ts)
931 {
932 if (commPagePtr) {
933 #ifdef __arm64__
934 struct bt_params *paramsp = (struct bt_params *)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET);
935 paramsp->base_local_ts = 0;
936 __builtin_arm_dmb(DMB_ISH);
937 paramsp->rate = rate;
938 paramsp->base_remote_ts = base_remote_ts;
939 __builtin_arm_dmb(DMB_ISH);
940 paramsp->base_local_ts = base_local_ts; //This will act as a generation count
941 #endif /* __arm64__ */
942 }
943 }
944
945
946 /*
947 * update the commpage with if dtrace user land probes are enabled
948 */
949 void
commpage_update_dof(boolean_t enabled)950 commpage_update_dof(boolean_t enabled)
951 {
952 #if CONFIG_DTRACE
953 *((uint8_t*)(_COMM_PAGE_DTRACE_DOF_ENABLED + _COMM_PAGE_RW_OFFSET)) = (enabled ? 1 : 0);
954 #else
955 (void)enabled;
956 #endif
957 }
958
959 /*
960 * update the dyld global config flags
961 */
962 void
commpage_update_dyld_flags(uint64_t value)963 commpage_update_dyld_flags(uint64_t value)
964 {
965 *((uint64_t*)(_COMM_PAGE_DYLD_FLAGS + _COMM_PAGE_RW_OFFSET)) = value;
966
967 }
968