1*aca3beaaSApple OSS Distributions""" 2*aca3beaaSApple OSS Distributions Triage Macros for zone related panics 3*aca3beaaSApple OSS Distributions 4*aca3beaaSApple OSS Distributions Supported panic strings from xnu/osfmk/kern/zalloc.c: 5*aca3beaaSApple OSS Distributions "a freed zone element has been modified in zone %s: expected %p but found %p, bits changed %p, at offset %d of %d in element %p, cookies %p %p" and 6*aca3beaaSApple OSS Distributions "zalloc: zone map exhausted while allocating from zone %s, likely due to memory leak in zone %s (%lu total bytes, %d elements allocated)" 7*aca3beaaSApple OSS Distributions These macros are dependant on the above panic strings. If the strings are modified in any way, this script must be updated to reflect the change. 8*aca3beaaSApple OSS Distributions 9*aca3beaaSApple OSS Distributions To support more zone panic strings: 10*aca3beaaSApple OSS Distributions 1. Add the panic string regex to the globals and include in the named capture group 'zone' (the zone to be 11*aca3beaaSApple OSS Distributions logged) as well as any other info necessary to parse out of the panic string. 12*aca3beaaSApple OSS Distributions 2. Add a check for the panic string regex in ZoneTriage(), which then calls into the function you create. 13*aca3beaaSApple OSS Distributions 3. Add a check for the panic string regex in CheckZoneBootArgs() which sets the variable panic_string_regex to your 14*aca3beaaSApple OSS Distributions panic string regex if found. 15*aca3beaaSApple OSS Distributions 4. Create a function that can be called either through the zonetriage macro ZoneTriage() or using its own macro. 16*aca3beaaSApple OSS Distributions This function should handle all lldb commands you want to run for this type of zone panic. 17*aca3beaaSApple OSS Distributions""" 18*aca3beaaSApple OSS Distributionsfrom __future__ import absolute_import, print_function 19*aca3beaaSApple OSS Distributions 20*aca3beaaSApple OSS Distributionsfrom xnu import * 21*aca3beaaSApple OSS Distributionsimport sys, shlex 22*aca3beaaSApple OSS Distributionsfrom utils import * 23*aca3beaaSApple OSS Distributionsimport xnudefines 24*aca3beaaSApple OSS Distributionsimport re 25*aca3beaaSApple OSS Distributionsimport os.path 26*aca3beaaSApple OSS Distributions 27*aca3beaaSApple OSS Distributions## Globals 28*aca3beaaSApple OSS Distributionspanic_string = None 29*aca3beaaSApple OSS Distributions## If the following panic strings are modified in xnu/osfmk/kern/zalloc.c, they must be updated here to reflect the change. 30*aca3beaaSApple OSS Distributionszone_element_modified = ".*a freed zone element has been modified in zone (?P<zone>.+): expected (0x)?([0-9A-Fa-f]*)? but found (0x)?([0-9A-Fa-f]*)?, bits changed (0x)?([0-9A-Fa-f]*)?, at offset ([0-9]*)? of ([0-9]*)? in element (?P<element>0x[0-9A-Fa-f]*), cookies (0x)?([0-9A-Fa-f]*)? (0x)?([0-9A-Fa-f]*)?.*" 31*aca3beaaSApple OSS Distributionszone_map_exhausted = ".*zalloc: zone map exhausted while allocating from zone .+, likely due to memory leak in zone (?P<zone>.+) \(([0-9]*)? total bytes, ([0-9]*)? elements allocated\).*" 32*aca3beaaSApple OSS Distributions 33*aca3beaaSApple OSS Distributions# Macro: zonetriage, zonetriage_freedelement, zonetriage_memoryleak 34*aca3beaaSApple OSS Distributions@lldb_command('zonetriage') 35*aca3beaaSApple OSS Distributionsdef ZoneTriage(cmd_args=None): 36*aca3beaaSApple OSS Distributions """ Calls function specific to type of zone panic based on the panic string 37*aca3beaaSApple OSS Distributions """ 38*aca3beaaSApple OSS Distributions global panic_string 39*aca3beaaSApple OSS Distributions if panic_string is None: 40*aca3beaaSApple OSS Distributions try: 41*aca3beaaSApple OSS Distributions panic_string = lldb_run_command("paniclog").split('\n', 1)[0] 42*aca3beaaSApple OSS Distributions except: 43*aca3beaaSApple OSS Distributions return 44*aca3beaaSApple OSS Distributions if re.match(zone_element_modified, panic_string) is not None: 45*aca3beaaSApple OSS Distributions ZoneTriageFreedElement() 46*aca3beaaSApple OSS Distributions elif re.match(zone_map_exhausted, panic_string) is not None: 47*aca3beaaSApple OSS Distributions ZoneTriageMemoryLeak() 48*aca3beaaSApple OSS Distributions else: 49*aca3beaaSApple OSS Distributions print("zonetriage does not currently support this panic string.") 50*aca3beaaSApple OSS Distributions 51*aca3beaaSApple OSS Distributions@lldb_command('zonetriage_freedelement') 52*aca3beaaSApple OSS Distributionsdef ZoneTriageFreedElement(cmd_args=None): 53*aca3beaaSApple OSS Distributions """ Runs zstack_findelem on the element and zone being logged based on the panic string regex 54*aca3beaaSApple OSS Distributions """ 55*aca3beaaSApple OSS Distributions global panic_string 56*aca3beaaSApple OSS Distributions if panic_string is None: 57*aca3beaaSApple OSS Distributions try: 58*aca3beaaSApple OSS Distributions panic_string = lldb_run_command("paniclog").split('\n', 1)[0] 59*aca3beaaSApple OSS Distributions except: 60*aca3beaaSApple OSS Distributions return 61*aca3beaaSApple OSS Distributions CheckZoneBootArgs() 62*aca3beaaSApple OSS Distributions ## Run showzonesbeinglogged. 63*aca3beaaSApple OSS Distributions print("(lldb) zstack_showzonesbeinglogged\n%s\n" % lldb_run_command("zstack_showzonesbeinglogged")) 64*aca3beaaSApple OSS Distributions ## Capture zone and element from panic string. 65*aca3beaaSApple OSS Distributions values = re.search(zone_element_modified, panic_string) 66*aca3beaaSApple OSS Distributions if values is None or 'zone' not in values.group() or 'element' not in values.group(): 67*aca3beaaSApple OSS Distributions return 68*aca3beaaSApple OSS Distributions element = values.group('element') 69*aca3beaaSApple OSS Distributions zone = values.group('zone') 70*aca3beaaSApple OSS Distributions btlog = FindZoneBTLog(zone) 71*aca3beaaSApple OSS Distributions if btlog is not None: 72*aca3beaaSApple OSS Distributions print("(lldb) zstack_findelem " + btlog + " " + element) 73*aca3beaaSApple OSS Distributions findelem_output = lldb_run_command("zstack_findelem " + btlog + " " + element) 74*aca3beaaSApple OSS Distributions findelem_output = re.sub('Scanning is ongoing. [0-9]* items scanned since last check.\n', '', findelem_output) 75*aca3beaaSApple OSS Distributions print(findelem_output) 76*aca3beaaSApple OSS Distributions 77*aca3beaaSApple OSS Distributions@lldb_command('zonetriage_memoryleak') 78*aca3beaaSApple OSS Distributionsdef ZoneTriageMemoryLeak(cmd_args=None): 79*aca3beaaSApple OSS Distributions """ Runs zstack_findtop and zstack_findleak on all zones being logged 80*aca3beaaSApple OSS Distributions """ 81*aca3beaaSApple OSS Distributions global kern 82*aca3beaaSApple OSS Distributions CheckZoneBootArgs() 83*aca3beaaSApple OSS Distributions ## Run showzonesbeinglogged. 84*aca3beaaSApple OSS Distributions print("(lldb) zstack_showzonesbeinglogged\n%s\n" % lldb_run_command("zstack_showzonesbeinglogged")) 85*aca3beaaSApple OSS Distributions for zval, _ in kern.zones: 86*aca3beaaSApple OSS Distributions btlog = getattr(zval, 'z_btlog', None) 87*aca3beaaSApple OSS Distributions if btlog: 88*aca3beaaSApple OSS Distributions print('%s:' % zval.z_name) 89*aca3beaaSApple OSS Distributions print("(lldb) zstack_findtop -N 5 0x%lx" % btlog) 90*aca3beaaSApple OSS Distributions print(lldb_run_command("zstack_findtop -N 5 0x%lx" % btlog)) 91*aca3beaaSApple OSS Distributions print("(lldb) zstack_findleak 0x%lx" % btlog) 92*aca3beaaSApple OSS Distributions print(lldb_run_command("zstack_findleak 0x%lx" % btlog)) 93*aca3beaaSApple OSS Distributions 94*aca3beaaSApple OSS Distributionsdef CheckZoneBootArgs(cmd_args=None): 95*aca3beaaSApple OSS Distributions """ Check boot args to see if zone is being logged, if not, suggest new boot args 96*aca3beaaSApple OSS Distributions """ 97*aca3beaaSApple OSS Distributions global panic_string 98*aca3beaaSApple OSS Distributions if panic_string is None: 99*aca3beaaSApple OSS Distributions try: 100*aca3beaaSApple OSS Distributions panic_string = lldb_run_command("paniclog").split('\n', 1)[0] 101*aca3beaaSApple OSS Distributions except: 102*aca3beaaSApple OSS Distributions return 103*aca3beaaSApple OSS Distributions panic_string_regex = "" 104*aca3beaaSApple OSS Distributions if re.match(zone_element_modified, panic_string) is not None: 105*aca3beaaSApple OSS Distributions panic_string_regex = zone_element_modified 106*aca3beaaSApple OSS Distributions if re.match(zone_map_exhausted, panic_string) is not None: 107*aca3beaaSApple OSS Distributions panic_string_regex = zone_map_exhausted 108*aca3beaaSApple OSS Distributions values = re.search(panic_string_regex, panic_string) 109*aca3beaaSApple OSS Distributions if values is None or 'zone' not in values.group(): 110*aca3beaaSApple OSS Distributions return 111*aca3beaaSApple OSS Distributions zone = values.group('zone') 112*aca3beaaSApple OSS Distributions bootargs = lldb_run_command("showbootargs") 113*aca3beaaSApple OSS Distributions correct_boot_args = re.search('zlog([1-9]|10)?=' + re.sub(' ', '.', zone), bootargs) 114*aca3beaaSApple OSS Distributions if correct_boot_args is None: 115*aca3beaaSApple OSS Distributions print("Current boot-args:\n" + bootargs) 116*aca3beaaSApple OSS Distributions print("You may need to include: -zc -zp zlog([1-9]|10)?=" + re.sub(' ', '.', zone)) 117*aca3beaaSApple OSS Distributions 118*aca3beaaSApple OSS Distributionsdef FindZoneBTLog(zone): 119*aca3beaaSApple OSS Distributions """ Returns the btlog address in the format 0x%lx for the zone name passed as a parameter 120*aca3beaaSApple OSS Distributions """ 121*aca3beaaSApple OSS Distributions global kern 122*aca3beaaSApple OSS Distributions for zval, _ in kern.zones: 123*aca3beaaSApple OSS Distributions btlog = getattr(zval, 'z_btlog', None) 124*aca3beaaSApple OSS Distributions if btlog: 125*aca3beaaSApple OSS Distributions if zone == "%s" % zval.z_name: 126*aca3beaaSApple OSS Distributions return "0x%lx" % btlog 127*aca3beaaSApple OSS Distributions return None 128*aca3beaaSApple OSS Distributions# EndMacro: zonetriage, zonetriage_freedelement, zonetriage_memoryleak 129