1 /*
2 * Copyright (c) 2007-2018 Apple Inc. All rights reserved.
3 */
4 /*
5 * CDDL HEADER START
6 *
7 * The contents of this file are subject to the terms of the
8 * Common Development and Distribution License, Version 1.0 only
9 * (the "License"). You may not use this file except in compliance
10 * with the License.
11 *
12 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
13 * or http://www.opensolaris.org/os/licensing.
14 * See the License for the specific language governing permissions
15 * and limitations under the License.
16 *
17 * When distributing Covered Code, include this CDDL HEADER in each
18 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
19 * If applicable, add the following below this CDDL HEADER, with the
20 * fields enclosed by brackets "[]" replaced with your own identifying
21 * information: Portions Copyright [yyyy] [name of copyright owner]
22 *
23 * CDDL HEADER END
24 */
25 /*
26 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
28 */
29
30 #include <kern/thread.h>
31 #include <mach/thread_status.h>
32 #include <arm/proc_reg.h>
33 #include <arm/caches_internal.h>
34 #include <arm/thread.h>
35
36 #include <mach-o/loader.h>
37 #include <mach-o/nlist.h>
38 #include <libkern/kernel_mach_header.h>
39
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/errno.h>
43 #include <sys/stat.h>
44 #include <sys/ioctl.h>
45 #include <sys/conf.h>
46 #include <sys/fcntl.h>
47 #include <miscfs/devfs/devfs.h>
48
49 #include <sys/dtrace.h>
50 #include <sys/dtrace_impl.h>
51 #include <sys/fbt.h>
52
53 #include <sys/dtrace_glue.h>
54
55 #define DTRACE_INVOP_PUSH_LR 8
56 #define DTRACE_INVOP_BL 9
57 #define DTRACE_INVOP_POP_PC 10
58
59 #define DTRACE_INVOP_THUMB_NOP_SKIP 2
60 #define DTRACE_INVOP_POP_PC_SKIP 2
61 #define DTRACE_INVOP_THUMB_SET_R7_SKIP 2
62 #define DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP 2
63
64 #define FBT_IS_THUMB_PUSH_LR(x) (((x) & 0x0000ff00) == 0x0000b500)
65 #define FBT_IS_THUMB_POP_R7(x) (((x) & 0x0000ff80) == 0x0000bc80)
66 #define FBT_IS_THUMB32_POP_R7LR(x, y) (((x) == 0x0000e8bd) && (((y) & 0x00004080) == 0x00004080))
67 #define FBT_IS_THUMB_POP_PC(x) (((x) & 0x0000ff00) == 0x0000bd00)
68 #define FBT_IS_THUMB_SET_R7(x) (((x) & 0x0000ff00) == 0x0000af00)
69 #define FBT_IS_THUMB_MOV_SP_TO_R7(x) (((x) & 0x0000ffff) == 0x0000466f)
70 #define FBT_THUMB_SET_R7_OFFSET(x) (((x) & 0x000000ff) << 2)
71 #define FBT_IS_THUMB_LDR_PC(x) (((x) & 0x0000f800) == 0x00004800)
72 #define FBT_IS_THUMB32_LDR_PC(x, y) ((x) == 0x0000f8df) /* Only for positive offset PC relative loads */
73 #define FBT_THUMB_STACK_REGS(x) ((x) & 0x00FF)
74 #define FBT_IS_THUMB_BX_REG(x) (((x) & 0x0000ff87) == 0x00004700)
75
76 #define FBT_PATCHVAL 0xdefc
77 #define FBT_AFRAMES_ENTRY 8
78 #define FBT_AFRAMES_RETURN 6
79
80 #define FBT_ENTRY "entry"
81 #define FBT_RETURN "return"
82 #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
83
84 #define VFPSAVE_ALIGN_DTRACE 16 /* This value should come from VFPSAVE_ALIGN */
85
86 extern dtrace_provider_id_t fbt_id;
87 extern fbt_probe_t **fbt_probetab;
88 extern int fbt_probetab_mask;
89
90 kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int);
91
92 extern int dtrace_arm_condition_true(int cond, int cpsr);
93
94
95 /* Calculate the address of the ldr. (From the ARM Architecture reference) */
96 /* Does not check to see if it's really a load instruction, caller must do that */
97
98 static uint32_t
thumb_ldr_pc_address(uint32_t address)99 thumb_ldr_pc_address(uint32_t address)
100 {
101 return (address & 0xFFFFFFFC) + (*(uint16_t*) address & 0xFF) * 4 + 4;
102 }
103
104 static uint32_t
thumb32_ldr_pc_address(uint32_t address)105 thumb32_ldr_pc_address(uint32_t address)
106 {
107 return (address & 0xFFFFFFFC) + (*(uint16_t*) (address + 2) & 0xFFF) + 4;
108 }
109
110 /* Extract the current ITSTATE from the CPSR */
111 static uint32_t
get_itstate(uint32_t cpsr)112 get_itstate(uint32_t cpsr)
113 {
114 return
115 ((cpsr & 0x06000000) >> 25) |
116 ((cpsr & 0x0000FC00) >> 8);
117 }
118
119 static void
clear_itstate(uint32_t * cpsr)120 clear_itstate(uint32_t* cpsr)
121 {
122 *cpsr &= ~0x0600FC00;
123 }
124
125 int
fbt_invop(uintptr_t addr,uintptr_t * stack,uintptr_t rval)126 fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval)
127 {
128 fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
129
130 for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
131 if ((uintptr_t) fbt->fbtp_patchpoint == addr) {
132 if (0 == CPU->cpu_dtrace_invop_underway) {
133 CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on
134 * this per-cpu state */
135
136 struct arm_saved_state* regs = (struct arm_saved_state*) stack;
137 uintptr_t stack4 = *((uintptr_t*) regs->sp);
138
139 if ((regs->cpsr & PSR_MODE_MASK) == PSR_FIQ_MODE) {
140 /*
141 * We do not handle probes firing from FIQ context. We used to
142 * try to undo the patch and rerun the instruction, but
143 * most of the time we can't do that successfully anyway.
144 * Instead, we just panic now so we fail fast.
145 */
146 panic("dtrace: fbt: The probe at %08x was called from FIQ_MODE", (unsigned) addr);
147 }
148
149 /*
150 * If we are not outside an IT block, and are not executing the last instruction of an IT block,
151 * then that is an instrumentation error or a code gen error. Either way, we panic.
152 */
153 uint32_t itstate = get_itstate(regs->cpsr);
154 if ((itstate & 0x7) != 0) {
155 panic("dtrace: fbt: Instruction stream error: Middle of IT block at %08x", (unsigned) addr);
156 }
157
158 if (fbt->fbtp_roffset == 0) {
159 /*
160 * We need the frames to set up the backtrace, but we won't have the frame pointers
161 * until after the instruction is emulated. So here we calculate the address of the
162 * frame pointer from the saved instruction and put it in the stack. Yes, we end up
163 * repeating this work again when we emulate the instruction.
164 *
165 * This assumes that the frame area is immediately after the saved reg storage!
166 */
167 uint32_t offset = ((uint32_t) regs) + sizeof(struct arm_saved_state);
168 #if __ARM_VFP__
169 /* Match the stack alignment required for arm_vfpsaved_state */
170 offset &= ~(VFPSAVE_ALIGN_DTRACE - 1);
171 offset += VFPSAVE_ALIGN_DTRACE + sizeof(struct arm_vfpsaved_state);
172 #endif /* __ARM_VFP__ */
173 if (FBT_IS_THUMB_SET_R7(fbt->fbtp_savedval)) {
174 *((uint32_t*) offset) = regs->sp + FBT_THUMB_SET_R7_OFFSET(fbt->fbtp_savedval);
175 } else {
176 *((uint32_t*) offset) = regs->sp;
177 }
178
179 CPU->cpu_dtrace_caller = regs->lr;
180 dtrace_probe(fbt->fbtp_id, regs->r[0], regs->r[1], regs->r[2], regs->r[3], stack4);
181 CPU->cpu_dtrace_caller = 0;
182 } else {
183 /* Check to see if we're in the middle of an IT block. */
184 if (itstate != 0) {
185 /*
186 * We've already checked previously to see how far we are in the IT block.
187 * Here we must be getting ready to execute the last instruction.
188 */
189 int condition_it = (itstate & 0xF0) >> 4;
190
191 if (dtrace_arm_condition_true(condition_it, regs->cpsr) == 0) {
192 /* Condition wasn't true, so becomes a nop. */
193 clear_itstate(®s->cpsr);
194 CPU->cpu_dtrace_invop_underway = 0;
195 return DTRACE_INVOP_NOP;
196 }
197 }
198
199 dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0);
200 CPU->cpu_dtrace_caller = 0;
201
202 /* The dtrace script may access cpsr, so make sure to clear only after probe fired. */
203 clear_itstate(®s->cpsr);
204 }
205 CPU->cpu_dtrace_invop_underway = 0;
206 }
207
208 /*
209 * On other architectures, we return a DTRACE constant to let the callback function
210 * know what was replaced. On the ARM, since the function prologue/epilogue machine code
211 * can vary, we need the actual bytes of the instruction, so return the savedval instead.
212 */
213 return fbt->fbtp_savedval;
214 }
215 }
216
217 return 0;
218 }
219
220 #define IS_USER_TRAP(regs) (((regs)->cpsr & PSR_MODE_MASK) == PSR_USER_MODE)
221 #define T_INVALID_OPCODE EXC_BAD_INSTRUCTION
222 #define FBT_EXCEPTION_CODE T_INVALID_OPCODE
223
224 kern_return_t
fbt_perfCallback(int trapno,struct arm_saved_state * regs,__unused int unused1,__unused int unused2)225 fbt_perfCallback(
226 int trapno,
227 struct arm_saved_state * regs,
228 __unused int unused1,
229 __unused int unused2)
230 {
231 #pragma unused (unused1)
232 #pragma unused (unused2)
233 kern_return_t retval = KERN_FAILURE;
234
235 if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) {
236 boolean_t oldlevel = 0;
237 machine_inst_t emul = 0;
238
239 oldlevel = ml_set_interrupts_enabled(FALSE);
240
241 __asm__ volatile (
242 "Ldtrace_invop_callsite_pre_label:\n"
243 ".data\n"
244 ".private_extern _dtrace_invop_callsite_pre\n"
245 "_dtrace_invop_callsite_pre:\n"
246 " .long Ldtrace_invop_callsite_pre_label\n"
247 ".text\n"
248 );
249
250 emul = dtrace_invop(regs->pc, (uintptr_t*) regs, regs->r[0]);
251
252 __asm__ volatile (
253 "Ldtrace_invop_callsite_post_label:\n"
254 ".data\n"
255 ".private_extern _dtrace_invop_callsite_post\n"
256 "_dtrace_invop_callsite_post:\n"
257 " .long Ldtrace_invop_callsite_post_label\n"
258 ".text\n"
259 );
260
261 /*
262 * The following emulation code does not execute properly if we are in the middle of
263 * an IT block. IT blocks need to be handled in the dtrace_invop function. If we do
264 * manage to get here and we are inside an IT block, then we missed a case somewhere
265 * prior to this point.
266 */
267 uint32_t itstate = get_itstate(regs->cpsr);
268 if (itstate != 0) {
269 panic("dtrace: fbt: Not emulated: Middle of IT block at %08x", (unsigned) regs->pc);
270 }
271
272 if (emul == DTRACE_INVOP_NOP) {
273 regs->pc += DTRACE_INVOP_THUMB_NOP_SKIP;
274 retval = KERN_SUCCESS;
275 } else if (FBT_IS_THUMB_SET_R7(emul)) {
276 regs->r[7] = regs->sp + FBT_THUMB_SET_R7_OFFSET(emul);
277 regs->pc += DTRACE_INVOP_THUMB_SET_R7_SKIP;
278 retval = KERN_SUCCESS;
279 } else if (FBT_IS_THUMB_MOV_SP_TO_R7(emul)) {
280 regs->r[7] = regs->sp;
281 regs->pc += DTRACE_INVOP_THUMB_MOV_SP_TO_R7_SKIP;
282 retval = KERN_SUCCESS;
283 } else if (FBT_IS_THUMB_POP_PC(emul)) {
284 uintptr_t* sp = (uintptr_t*) regs->sp;
285
286 machine_inst_t mask = 0x0001;
287 int regnum = 0;
288 while (mask & 0x00ff) {
289 if (emul & mask) {
290 /* Pop this register */
291 regs->r[regnum] = *sp++;
292 }
293 mask <<= 1;
294 regnum++;
295 }
296
297 regs->pc = *sp++;
298 regs->sp = (uintptr_t) sp;
299 if (regs->pc & 1) {
300 regs->cpsr |= PSR_TF;
301 } else {
302 regs->cpsr &= ~PSR_TF;
303 }
304
305 retval = KERN_SUCCESS;
306 } else if (FBT_IS_THUMB_BX_REG(emul)) {
307 regs->pc = regs->r[(emul >> 3) & 0xF];
308
309 if (regs->pc & 1) {
310 regs->cpsr |= PSR_TF;
311 } else {
312 regs->cpsr &= ~PSR_TF;
313 }
314
315 retval = KERN_SUCCESS;
316 } else if (emul == FBT_PATCHVAL) {
317 /* Means we encountered an error but handled it, try same inst again */
318 retval = KERN_SUCCESS;
319 } else {
320 retval = KERN_FAILURE;
321 }
322
323 ml_set_interrupts_enabled(oldlevel);
324 }
325
326 return retval;
327 }
328
329 void
fbt_provide_probe(struct modctl * ctl,const char * modname,const char * symbolName,machine_inst_t * symbolStart,machine_inst_t * instrHigh)330 fbt_provide_probe(struct modctl *ctl, const char *modname, const char* symbolName, machine_inst_t* symbolStart, machine_inst_t *instrHigh)
331 {
332 unsigned int j;
333 int doenable = 0;
334 dtrace_id_t thisid;
335
336 fbt_probe_t *newfbt, *retfbt, *entryfbt;
337 machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr;
338 int foundPushLR, savedRegs;
339
340 /*
341 * Guard against null symbols
342 */
343 if (!symbolStart || !instrHigh || instrHigh < symbolStart) {
344 kprintf("dtrace: %s has an invalid address\n", symbolName);
345 return;
346 }
347
348 /*
349 * Assume the compiler doesn't schedule instructions in the prologue.
350 */
351 foundPushLR = 0;
352 savedRegs = -1;
353 limit = (machine_inst_t *)instrHigh;
354 for (j = 0, instr = symbolStart, theInstr = 0;
355 (j < 8) && instr < instrHigh; j++, instr++) {
356 theInstr = *instr;
357 if (FBT_IS_THUMB_PUSH_LR(theInstr)) {
358 foundPushLR = 1;
359 /* Keep track of what registers we pushed. Compare this against the pop later. */
360 savedRegs = FBT_THUMB_STACK_REGS(theInstr);
361 pushinstr = instr;
362 }
363 if (foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr))) {
364 /* Guard against a random setting of r7 from sp, we make sure we found the push first */
365 break;
366 }
367 if (FBT_IS_THUMB_BX_REG(theInstr)) { /* We've gone too far, bail. */
368 break;
369 }
370 if (FBT_IS_THUMB_POP_PC(theInstr)) { /* We've gone too far, bail. */
371 break;
372 }
373
374 /* Check for 4 byte thumb instruction */
375 if (dtrace_instr_size(theInstr, 1) == 4) {
376 instr++;
377 }
378 }
379
380 if (!(foundPushLR && (FBT_IS_THUMB_SET_R7(theInstr) || FBT_IS_THUMB_MOV_SP_TO_R7(theInstr)))) {
381 return;
382 }
383
384 thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY);
385 newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
386 newfbt->fbtp_next = NULL;
387 strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
388
389 if (thisid != 0) {
390 /*
391 * The dtrace_probe previously existed, so we have to hook
392 * the newfbt entry onto the end of the existing fbt's
393 * chain.
394 * If we find an fbt entry that was previously patched to
395 * fire, (as indicated by the current patched value), then
396 * we want to enable this newfbt on the spot.
397 */
398 entryfbt = dtrace_probe_arg(fbt_id, thisid);
399 ASSERT(entryfbt != NULL);
400 for (; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) {
401 if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) {
402 doenable++;
403 }
404
405 if (entryfbt->fbtp_next == NULL) {
406 entryfbt->fbtp_next = newfbt;
407 newfbt->fbtp_id = entryfbt->fbtp_id;
408 break;
409 }
410 }
411 } else {
412 /*
413 * The dtrace_probe did not previously exist, so we
414 * create it and hook in the newfbt. Since the probe is
415 * new, we obviously do not need to enable it on the spot.
416 */
417 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt);
418 doenable = 0;
419 }
420
421 newfbt->fbtp_patchpoint = instr;
422 newfbt->fbtp_ctl = ctl;
423 newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
424 newfbt->fbtp_rval = DTRACE_INVOP_PUSH_LR;
425 newfbt->fbtp_savedval = theInstr;
426 newfbt->fbtp_patchval = FBT_PATCHVAL;
427 newfbt->fbtp_currentval = 0;
428 newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
429 fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
430
431 if (doenable) {
432 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
433 }
434
435 /*
436 * The fbt entry chain is in place, one entry point per symbol.
437 * The fbt return chain can have multiple return points per
438 * symbol.
439 * Here we find the end of the fbt return chain.
440 */
441
442 doenable = 0;
443
444 thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN);
445
446 if (thisid != 0) {
447 /* The dtrace_probe previously existed, so we have to
448 * find the end of the existing fbt chain. If we find
449 * an fbt return that was previously patched to fire,
450 * (as indicated by the currrent patched value), then
451 * we want to enable any new fbts on the spot.
452 */
453 retfbt = dtrace_probe_arg(fbt_id, thisid);
454 ASSERT(retfbt != NULL);
455 for (; retfbt != NULL; retfbt = retfbt->fbtp_next) {
456 if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) {
457 doenable++;
458 }
459 if (retfbt->fbtp_next == NULL) {
460 break;
461 }
462 }
463 } else {
464 doenable = 0;
465 retfbt = NULL;
466 }
467
468 /*
469 * Go back to the start of the function, in case
470 * the compiler emitted pcrel data loads
471 * before R7 was adjusted.
472 */
473 instr = pushinstr + 1;
474 again:
475 if (instr >= limit) {
476 return;
477 }
478
479 /*
480 * We (desperately) want to avoid erroneously instrumenting a
481 * jump table. To determine if we're looking at a true instruction
482 * or an inline jump table that happens to contain the same
483 * byte sequences, we resort to some heuristic sleeze: we
484 * treat this instruction as being contained within a pointer,
485 * and see if that pointer points to within the body of the
486 * function. If it does, we refuse to instrument it.
487 */
488 if (((uintptr_t)instr & 0x3) == 0) {
489 machine_inst_t *ptr = *(machine_inst_t **)(void *)instr;
490
491 if (ptr >= (machine_inst_t *)symbolStart && ptr < limit) {
492 /* kprintf("dtrace: fbt: Found jump table in %s, at %08x\n",symbolName,(unsigned)instr); */
493 instr++;
494 goto again;
495 }
496 }
497
498 /*
499 * OK, it's an instruction.
500 */
501 theInstr = *instr;
502
503 /* Walked onto the start of the next routine? If so, bail out from this function */
504 if (FBT_IS_THUMB_PUSH_LR(theInstr)) {
505 if (!retfbt) {
506 kprintf("dtrace: fbt: No return probe for %s, walked to next routine at %08x\n", symbolName, (unsigned)instr);
507 }
508 return;
509 }
510
511 /* The PC relative data should be stored after the end of the function. If
512 * we see a PC relative load, assume the address to load from is the new end
513 * of the function. */
514 if (FBT_IS_THUMB_LDR_PC(theInstr)) {
515 uint32_t newlimit = thumb_ldr_pc_address((uint32_t) instr);
516 if (newlimit < (uint32_t) limit) {
517 limit = (machine_inst_t*) newlimit;
518 }
519 }
520 if ((instr + 1) < limit && FBT_IS_THUMB32_LDR_PC(*instr, *(instr + 1))) {
521 uint32_t newlimit = thumb32_ldr_pc_address((uint32_t) instr);
522 if (newlimit < (uint32_t) limit) {
523 limit = (machine_inst_t*) newlimit;
524 }
525 }
526
527 /* Look for the 1. pop { ..., pc } or 2. pop { ..., r7 } ... bx reg or 3. ldmia.w sp!, { ..., r7, lr } ... bx reg */
528 if (!FBT_IS_THUMB_POP_PC(theInstr) &&
529 !FBT_IS_THUMB_POP_R7(theInstr) &&
530 !FBT_IS_THUMB32_POP_R7LR(theInstr, *(instr + 1))) {
531 instr++;
532 if (dtrace_instr_size(theInstr, 1) == 4) {
533 instr++;
534 }
535 goto again;
536 }
537
538 if (FBT_IS_THUMB_POP_PC(theInstr)) {
539 if (savedRegs != FBT_THUMB_STACK_REGS(theInstr)) {
540 /* What we're popping doesn't match what we're pushing, assume that we've
541 * gone too far in the function. Bail.
542 */
543 kprintf("dtrace: fbt: No return probe for %s, popped regs don't match at %08x\n", symbolName, (unsigned)instr);
544 return;
545 }
546 } else {
547 /* Scan ahead for the bx */
548 for (j = 0; (j < 4) && (instr < limit); j++, instr++) {
549 theInstr = *instr;
550 if (FBT_IS_THUMB_BX_REG(theInstr)) {
551 break;
552 }
553 if (dtrace_instr_size(theInstr, 1) == 4) {
554 instr++;
555 }
556 }
557
558 if (!FBT_IS_THUMB_BX_REG(theInstr)) {
559 return;
560 }
561 }
562
563 /*
564 * pop { ..., pc}, bx reg -- We have a winner!
565 */
566
567 newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP);
568 newfbt->fbtp_next = NULL;
569 strlcpy((char *)&(newfbt->fbtp_name), symbolName, MAX_FBTP_NAME_CHARS );
570
571 if (retfbt == NULL) {
572 newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
573 symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt);
574 } else {
575 retfbt->fbtp_next = newfbt;
576 newfbt->fbtp_id = retfbt->fbtp_id;
577 }
578
579 retfbt = newfbt;
580 newfbt->fbtp_patchpoint = instr;
581 newfbt->fbtp_ctl = ctl;
582 newfbt->fbtp_loadcnt = ctl->mod_loadcnt;
583
584 ASSERT(FBT_IS_THUMB_POP_PC(theInstr) || FBT_IS_THUMB_BX_REG(theInstr));
585 newfbt->fbtp_rval = DTRACE_INVOP_POP_PC;
586 newfbt->fbtp_roffset =
587 (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart);
588 newfbt->fbtp_savedval = theInstr;
589 newfbt->fbtp_patchval = FBT_PATCHVAL;
590 newfbt->fbtp_currentval = 0;
591 newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
592 fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt;
593
594 if (doenable) {
595 fbt_enable(NULL, newfbt->fbtp_id, newfbt);
596 }
597
598 instr++;
599 goto again;
600 }
601