1 /*
2 * Copyright (c) 2011 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* A pool of threads which attempt to verify multiprocessor TLB coherency.
30 * Creates -t threads, by default 4
31 * Creates -s separate mmap(MAP_ANON) R/W mappings, sized at 1 page each but
32 * alterable via -z <npages>
33 * Initially read-faults each mapping in, verifying first-word zerofill--
34 * The kernel typically uses the physical aperture to perform the zerofill
35 * Writes map_address (page_aligned) | low 12 bits of the PID at the first word
36 * This can help verify ASID related inconsistencies
37 * Records a timestamp in a Structure associated with each mapping
38 * With a custom kernel, it has the option of creating a remapping of the page in
39 * the kernel's address space to exercise shared kernel mapping coherency.
40 * Each thread subsequently loops around on the set of mappings. One thread is designated
41 * the observer thread. The thread acquires a lock on the arena element,
42 * verifies that the mapping has the expected pattern (Address | PID), if the
43 * element is in the MAPPED state. Can optionally tell the kernel to check its
44 * alias as well. If it notices a mismatch, it has the option to issue a syscall
45 * to stop kernel tracing. If the -f option is supplied, the test is terminated.
46 * If the page has lingered beyond -l microseconds, non-observer threads will
47 * unmap the page, optionally calling into the kernel to unmap its alias, and
48 * repopulate the element.
49 * After this sequence, the thread will optionally usleep for -p microseconds,
50 * to allow for idle power management to engage if possible (errata might exist
51 * in those areas), or context switches to occur.
52 * Created Derek Kumar, 2011.
53 */
54
55 #include <stdio.h>
56 #include <stdlib.h>
57 #include <unistd.h>
58 #include <sys/mman.h>
59 #include <pthread.h>
60 #include <string.h>
61 #include <mach/mach_time.h>
62 #include <libkern/OSAtomic.h>
63 #include <sys/syscall.h>
64 #include <sys/types.h>
65 #include <sys/sysctl.h>
66
67 typedef struct {
68 OSSpinLock tlock;
69 uintptr_t taddr;
70 unsigned tstate;
71 uint64_t tctime;
72 } cpage;
73
74 cpage *parray;
75
76 #define ARENASIZE (1024)
77 #define NTHREADS (4)
78 #define PAGE_LINGER_TIME (2000000)
79 #define MAX_THREADS (512)
80 #define MYSYS (215)
81 #define CONSISTENCY(...) fprintf(stderr, __VA_ARGS__ );
82
83 unsigned arenasize = ARENASIZE, mapping_size;
84 uint64_t page_linger_time = PAGE_LINGER_TIME;
85 enum arenastates {MTOUCHED = 1, UNMAPPED = 2, MAPPED = 4, WP =8};
86 enum syscaction {MDOMAP = 1, MDOUNMAP = 2, MDOCHECK = 4};
87 enum ttypes {OBSERVER = 1, LOOPER = 2};
88 bool trymode = true;
89 bool all_stop = false;
90 bool stop_on_failure = false;
91 bool reuse_addrs = true;
92 bool dosyscall = false;
93
94 pid_t cpid;
95 int sleepus;
96
97 pthread_t threads[MAX_THREADS];
98 uint32_t roles[MAX_THREADS];
99
100 void
usage(char ** a)101 usage(char **a)
102 {
103 exit(1);
104 }
105
106 void
set_enable(int val)107 set_enable(int val)
108 {
109 int mib[6];
110 size_t needed;
111
112 mib[0] = CTL_KERN;
113 mib[1] = KERN_KDEBUG;
114 mib[2] = KERN_KDENABLE;
115 mib[3] = val;
116 mib[4] = 0;
117 mib[5] = 0;
118
119 if (sysctl(mib, 4, NULL, &needed, NULL, 0) < 0) {
120 printf("trace facility failure, KERN_KDENABLE\n");
121 }
122 }
123
124 void
initialize_arena_element(int i)125 initialize_arena_element(int i)
126 {
127 __unused int sysret;
128 void *hint = reuse_addrs ? (void *)0x1000 : NULL;
129 parray[i].taddr = (uintptr_t)mmap(hint, mapping_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0);
130
131 if (parray[i].taddr == (uintptr_t)MAP_FAILED) {
132 perror("mmap");
133 exit(2);
134 }
135
136 #if !defined(__LP64__)
137 uint32_t pattern = parray[i].taddr;
138 pattern |= cpid & 0xFFF;
139 // memset_pattern4((void *)parray[i].taddr, &pattern, PAGE_SIZE); //
140 // uncomment to fill the whole page, but a sufficiently unique first word
141 // gets the job done without slowing down the test
142
143 #else
144 uint64_t pattern = parray[i].taddr;
145 pattern |= (cpid & 0xFFF);
146 // memset_pattern8(parray[i].taddr, &pattern, PAGE_SIZE);
147 #endif
148
149 uint64_t val = (*(uintptr_t *)parray[i].taddr);
150
151 if (val != 0) {
152 CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, 0ULL);
153 if (stop_on_failure) {
154 set_enable(0);
155 exit(5);
156 }
157 }
158 for (int k = 0; k < (mapping_size >> PAGE_SHIFT); k++) {
159 *(uintptr_t *)(parray[i].taddr + k * PAGE_SIZE) = pattern;
160 }
161
162 parray[i].tctime = mach_absolute_time();
163 parray[i].tstate = MTOUCHED;
164
165 if (dosyscall) {
166 sysret = syscall(MYSYS, MDOMAP, parray[i].taddr, pattern, i, mapping_size);
167 }
168 }
169
170 void
initialize_arena(void)171 initialize_arena(void)
172 {
173 for (int i = 0; i < arenasize; i++) {
174 initialize_arena_element(i);
175 }
176 }
177
178 void *
tlbexerciser(void * targs)179 tlbexerciser(void *targs)
180 {
181 uint32_t role = *(uint32_t *)targs;
182 __unused int sysret;
183 printf("Starting thread %p, role: %u\n", pthread_self(), role);
184
185 for (;;) {
186 for (int i = 0; i < arenasize; i++) {
187 if (all_stop) {
188 return NULL;
189 }
190
191 if (trymode) {
192 if (OSSpinLockTry(&parray[i].tlock) == false) {
193 continue;
194 }
195 } else {
196 OSSpinLockLock(&parray[i].tlock);
197 }
198
199 if (parray[i].tstate != UNMAPPED) {
200 uintptr_t ad;
201 ad = parray[i].taddr | (cpid & 0xFFF);
202 uintptr_t val = *(uintptr_t *)parray[i].taddr;
203
204 if (val != ad) {
205 if (stop_on_failure) {
206 all_stop = true;
207 }
208 syscall(180, 0x71BC0000, (ad >> 32), (ad & ~0), 0, 0, 0);
209 CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, (unsigned long long)ad);
210 if (stop_on_failure) {
211 set_enable(0);
212 exit(5);
213 }
214 }
215
216 if (dosyscall) {
217 sysret = syscall(MYSYS, MDOCHECK, parray[i].taddr, ad, i, 0);
218 }
219
220 if ((role != OBSERVER) && ((mach_absolute_time() - parray[i].tctime) > page_linger_time)) {
221 parray[i].tstate = UNMAPPED;
222 if (munmap((void *)parray[i].taddr, mapping_size) != 0) {
223 perror("munmap");
224 }
225
226 if (dosyscall) {
227 sysret = syscall(MYSYS, MDOUNMAP, parray[i].taddr, ad, i, mapping_size);
228 }
229 }
230 } else {
231 if (role != OBSERVER) {
232 initialize_arena_element(i);
233 }
234 }
235
236 parray[i].tlock = 0; //unlock
237
238 if (sleepus) {
239 usleep(sleepus);
240 }
241 }
242 }
243
244 return NULL;
245 }
246
247 int
main(int argc,char ** argv)248 main(int argc, char **argv)
249 {
250 extern char *optarg;
251 int arg;
252 unsigned nthreads = NTHREADS;
253
254 mapping_size = PAGE_SIZE;
255
256 while ((arg = getopt(argc, argv, "l:t:h:s:p:z:fry")) != -1) {
257 switch (arg) {
258 case 'l':
259 page_linger_time = strtoull(optarg, NULL, 0);
260 break;
261 case 't':
262 nthreads = atoi(optarg);
263 break;
264 case 's':
265 arenasize = atoi(optarg); // we typically want this to
266 // be sized < 2nd level TLB
267 break;
268 case 'f':
269 stop_on_failure = true;
270 break;
271 case 'r':
272 reuse_addrs = false;
273 break;
274 case 'p':
275 sleepus = atoi(optarg);
276 break;
277 case 'y':
278 dosyscall = true;
279 break;
280 case 'z':
281 mapping_size = atoi(optarg) * PAGE_SIZE;
282 break;
283 case 'h':
284 usage(argv);
285 }
286 }
287
288 if (optind != argc) {
289 usage(argv);
290 }
291
292 printf("page_linger_time: 0x%llx, nthreads: %u, arenasize: %u sleepus: %d reuse_addrs: %u, stop_on_failure: %u, dosyscall: %u, mappingsize: 0x%x\n", page_linger_time, nthreads, arenasize, sleepus, reuse_addrs, (unsigned) stop_on_failure, dosyscall, mapping_size);
293
294 parray = calloc(arenasize, sizeof(cpage));
295 cpid = getpid();
296
297 initialize_arena();
298
299 for (int dex = 0; dex < nthreads; dex++) {
300 roles[dex] = LOOPER;
301 if (dex == 0) {
302 roles[dex] = OBSERVER;
303 }
304 int result = pthread_create(&threads[dex], NULL, tlbexerciser, &roles[dex]);
305 if (result) {
306 printf("pthread_create: %d starting worker thread; aborting.\n", result);
307 return result;
308 }
309 }
310
311 for (int dex = 0; dex < nthreads; dex++) {
312 void *rtn;
313 int result = pthread_join(threads[dex], &rtn);
314
315 if (result) {
316 printf("pthread_join(): %d, aborting\n", result);
317 return result;
318 }
319
320 if (rtn) {
321 printf("***Aborting on worker error\n");
322 exit(1);
323 }
324 }
325 return 0;
326 }
327