1from xnu import ( 2 kern, ArgumentError, unsigned, lldb_command, header, GetEnumValue, 3 GetEnumValues, GetEnumName, GetThreadName, GetProcStartAbsTimeForTask, 4 GetRecentTimestamp, GetProcNameForTask, FindTasksByName, IterateQueue) 5 6 7def validate_args(opts, valid_flags): 8 valid_flags = set(valid_flags) 9 for k in opts.keys(): 10 if k[1:] not in valid_flags: 11 raise ArgumentError('-{} not supported in subcommand'.format(k)) 12 13 14@lldb_command('recount', 'AF:MT', fancy=True) 15def Recount(cmd_args=None, cmd_options={}, O=None): # noqa: E741 16 """ Inspect counters maintained by the Recount subsystem on various resource 17 aggregators, like tasks or threads. 18 19 recount task [-TM] <task_t> [...] | -F <task_name> 20 recount thread [-M] <thread_t> [...] 21 recount coalition [-M] <coalition_t> [...] 22 recount processor [-ATM] [<processor_t-or-cpu-id>] [...] 23 24 Options: 25 -T : break out active threads for a task or processor 26 -M : show times in the Mach timebase 27 -A : show all processors 28 29 Diagnostic macros: 30 recount diagnose task <task_t> 31 - Ensure resource accounting consistency in a task. 32 recount triage 33 - Print out statistics useful for general panic triage. 34 35 """ 36 if not cmd_args: 37 raise ArgumentError('subcommand required') 38 39 if cmd_args[0] == 'coalition': 40 validate_args(cmd_options, ['M']) 41 RecountCoalition(cmd_args[1:], cmd_options=cmd_options, O=O) 42 elif cmd_args[0] == 'task': 43 validate_args(cmd_options, ['F', 'M', 'T']) 44 RecountTask(cmd_args[1:], cmd_options=cmd_options, O=O) 45 elif cmd_args[0] == 'thread': 46 validate_args(cmd_options, ['M']) 47 RecountThread(cmd_args[1:], cmd_options=cmd_options, O=O) 48 elif cmd_args[0] == 'processor': 49 validate_args(cmd_options, ['A', 'M', 'T']) 50 RecountProcessor(cmd_args[1:], cmd_options=cmd_options, O=O) 51 elif cmd_args[0] == 'diagnose': 52 RecountDiagnose(cmd_args[1:], cmd_options=cmd_options, O=O) 53 elif cmd_args[0] == 'triage': 54 validate_args(cmd_options, []) 55 RecountTriage(cmd_options=cmd_options, O=O) 56 else: 57 raise ArgumentError('{}: invalid subcommand'.format(cmd_args[0])) 58 59 60def scale_suffix(val, unit=''): 61 si_units = [ 62 (1e21, 'Z'), (1e18, 'E'), (1e15, 'P'), (1e12, 'T'), (1e9, 'B'), 63 (1e6, 'M'), (1e3, 'k'), (1, ' '), (1e-3, 'm'), (1e-6, 'u'), 64 (1e-9, 'n')] 65 scale, sfx = (1, '') 66 for (si_scale, si_sfx) in si_units: 67 if val >= si_scale: 68 scale, sfx = (si_scale, si_sfx) 69 break 70 return '{:>7.3f}{:<1s}{}'.format(val / scale, sfx, unit) 71 72 73class RecountSum(object): 74 """ 75 Accumulate usage counters. 76 """ 77 78 def __init__(self, mach_times=False): 79 self._mach_times = mach_times 80 self._levels = RecountPlan.levels() 81 self._times_mach = [0] * len(self._levels) 82 self._instructions = [0] * len(self._levels) 83 self._cycles = [0] * len(self._levels) 84 self._energy_nj = 0 85 self._valid_count = 0 86 87 def add_usage(self, usage): 88 for (_, level) in self._levels: 89 metrics = usage.ru_metrics[level] 90 self._times_mach[level] += unsigned(metrics.rm_time_mach) 91 if hasattr(metrics, 'rm_cycles'): 92 self._instructions[level] += unsigned(metrics.rm_instructions) 93 self._cycles[level] += unsigned(metrics.rm_cycles) 94 if unsigned(metrics.rm_cycles) != 0: 95 self._valid_count += 1 96 if hasattr(usage, 'ru_energy_nj'): 97 self._energy_nj += unsigned(usage.ru_energy_nj) 98 99 def user_sys_times(self): 100 user_level = GetEnumValue('recount_level_t', 'RCT_LVL_USER') 101 user_time = self._times_mach[user_level] 102 return (user_time, sum(self._times_mach) - user_time) 103 104 def div_valid(self, numer, denom): 105 if self._valid_count == 0 or denom == 0: 106 return 0 107 return numer / denom 108 109 def _convert_time(self, time): 110 if self._mach_times: 111 return time 112 return kern.GetNanotimeFromAbstime(time) / 1e9 113 114 def time(self): 115 time = sum(self._times_mach) 116 if self._mach_times: 117 return time 118 return kern.GetNanotimeFromAbstime(time) 119 120 def fmt_args(self): 121 level_args = [[ 122 level_name, 123 self._convert_time(self._times_mach[level]), 124 scale_suffix(self._cycles[level]), 125 self.div_valid( 126 self._cycles[level], 127 kern.GetNanotimeFromAbstime(self._times_mach[level])), 128 scale_suffix(self._instructions[level]), 129 self.div_valid(self._cycles[level], self._instructions[level]), 130 '-', 131 '-'] for (level_name, level) in 132 RecountPlan.levels()] 133 134 total_time_ns = kern.GetNanotimeFromAbstime(sum(self._times_mach)) 135 total_cycles = sum(self._cycles) 136 total_insns = sum(self._instructions) 137 power_w = self._energy_nj / total_time_ns if total_time_ns != 0 else 0 138 level_args.append([ 139 '*', 140 total_time_ns / 1e9, scale_suffix(total_cycles), 141 self.div_valid(total_cycles, total_time_ns), 142 scale_suffix(total_insns), 143 self.div_valid(total_cycles, total_insns), 144 scale_suffix(self._energy_nj / 1e9, 'J'), 145 scale_suffix(power_w, 'W')]) 146 return level_args 147 148 def fmt_basic_args(self): 149 return [[ 150 level_name, 151 self._convert_time(self._times_mach[level]), 152 self._cycles[level], 153 self._instructions[level], 154 '-'] for (level_name, level) in 155 RecountPlan.levels()] 156 157 158class RecountPlan(object): 159 """ 160 Format tracks and usage according to a plan. 161 """ 162 163 def __init__(self, name, mach_times=False): 164 self._mach_times = mach_times 165 self._group_names = [] 166 self._group_column = None 167 168 plan = kern.GetGlobalVariable('recount_' + name + '_plan') 169 topo = plan.rpl_topo 170 if topo == GetEnumValue('recount_topo_t', 'RCT_TOPO_CPU'): 171 self._group_column = 'cpu' 172 self._group_count = unsigned(kern.globals.real_ncpus) 173 self._group_names = [ 174 'cpu-{}'.format(i) for i in range(self._group_count)] 175 elif topo == GetEnumValue('recount_topo_t', 'RCT_TOPO_CPU_KIND'): 176 if kern.arch.startswith('arm64'): 177 self._group_column = 'cpu-kind' 178 cluster_mask = int(kern.globals.topology_info.cluster_types) 179 self._group_count = bin(cluster_mask).count('1') 180 self._group_names = [ 181 GetEnumName('recount_cpu_kind_t', i)[8:][:4] 182 for i in range(self._group_count)] 183 else: 184 self._group_count = 1 185 elif topo == GetEnumValue('recount_topo_t', 'RCT_TOPO_SYSTEM'): 186 self._group_count = 1 187 else: 188 raise RuntimeError('{}: Unexpected recount topography', topo) 189 190 def time_fmt(self): 191 return '{:>12d}' if self._mach_times else '{:>12.05f}' 192 193 def _usage_fmt(self): 194 prefix = '{n}{{:>6s}} {t} '.format( 195 t=self.time_fmt(), n='{:>8s} ' if self._group_column else '') 196 return prefix + '{:>8s} {:>7.3g} {:>8s} {:>5.03f} {:>9s} {:>9s}' 197 198 def usages(self, usages): 199 for i in range(self._group_count): 200 yield usages[i] 201 202 def track_usages(self, tracks): 203 for i in range(self._group_count): 204 yield tracks[i].rt_usage 205 206 def usage_header(self): 207 fmt = '{:>6s} {:>12s} {:>8s} {:>7s} {:>8s} {:>5s} {:>9s} {:>9s}'.format( # noqa: E501 208 'level', 'time', 'cycles', 'GHz', 'insns', 209 'CPI', 'energy', 'power',) 210 if self._group_column: 211 fmt = '{:>8s} '.format(self._group_column) + fmt 212 return fmt 213 214 def levels(): 215 names = ['kernel', 'user'] 216 levels = list(zip(names, GetEnumValues('recount_level_t', [ 217 'RCT_LVL_' + name.upper() for name in names]))) 218 try: 219 levels.append(('secure', 220 GetEnumValue('recount_level_t', 'RCT_LVL_SECURE'))) 221 except KeyError: 222 # RCT_LVL_SECURE is not defined on this system. 223 pass 224 return levels 225 226 def format_usage(self, usage, name=None, sum=None, O=None): 227 rows = [] 228 229 levels = RecountPlan.levels() 230 total_time = 0 231 total_time_ns = 0 232 total_cycles = 0 233 total_insns = 0 234 for (level_name, level) in levels: 235 metrics = usage.ru_metrics[level] 236 time = unsigned(metrics.rm_time_mach) 237 time_ns = kern.GetNanotimeFromAbstime(time) 238 total_time_ns += time_ns 239 if not self._mach_times: 240 time = time_ns / 1e9 241 total_time += time 242 if hasattr(metrics, 'rm_cycles'): 243 cycles = unsigned(metrics.rm_cycles) 244 total_cycles += cycles 245 freq = cycles / time_ns if time_ns != 0 else 0 246 insns = unsigned(metrics.rm_instructions) 247 total_insns += insns 248 cpi = cycles / insns if insns != 0 else 0 249 else: 250 cycles = 0 251 freq = 0 252 insns = 0 253 cpi = 0 254 rows.append([ 255 level_name, time, scale_suffix(cycles), freq, 256 scale_suffix(insns), cpi, '-', '-']) 257 258 if hasattr(usage, 'ru_energy_nj'): 259 energy_nj = unsigned(usage.ru_energy_nj) 260 if total_time_ns != 0: 261 power_w = energy_nj / total_time_ns 262 else: 263 power_w = 0 264 else: 265 energy_nj = 0 266 power_w = 0 267 if total_insns != 0: 268 total_freq = total_cycles / total_time_ns if total_time_ns != 0 else 0 269 total_cpi = total_cycles / total_insns 270 else: 271 total_freq = 0 272 total_cpi = 0 273 274 rows.append([ 275 '*', total_time, scale_suffix(total_cycles), total_freq, 276 scale_suffix(total_insns), total_cpi, 277 scale_suffix(energy_nj / 1e9, 'J'), 278 scale_suffix(power_w, 'W')]) 279 280 if sum: 281 sum.add_usage(usage) 282 283 if self._group_column: 284 for row in rows: 285 row.insert(0, name) 286 287 return [O.format(self._usage_fmt(), *row) for row in rows] 288 289 def format_sum(self, sum, O=None): 290 lines = [] 291 for line in sum.fmt_args(): 292 lines.append(O.format(self._usage_fmt(), '*', *line)) 293 return lines 294 295 def format_usages(self, usages, O=None): # noqa: E741 296 sum = RecountSum(self._mach_times) if self._group_count > 1 else None 297 str = '' 298 for (i, usage) in enumerate(self.usages(usages)): 299 name = self._group_names[i] if i < len(self._group_names) else None 300 lines = self.format_usage(usage, name=name, sum=sum, O=O) 301 str += '\n'.join(lines) + '\n' 302 if sum: 303 str += '\n'.join(self.format_sum(sum, O=O)) 304 return str 305 306 def format_tracks(self, tracks, O=None): # noqa: E741 307 sum = RecountSum(self._mach_times) if self._group_count > 1 else None 308 str = '' 309 for (i, usage) in enumerate(self.track_usages(tracks)): 310 name = self._group_names[i] if i < len(self._group_names) else None 311 lines = self.format_usage(usage, name=name, sum=sum, O=O) 312 str += '\n'.join(lines) + '\n' 313 if sum: 314 str += '\n'.join(self.format_sum(sum, O=O)) 315 return str 316 317 def sum_usages(self, usages, sum=None): 318 if sum is None: 319 sum = RecountSum(mach_times=self._mach_times) 320 for usage in self.usages(usages): 321 sum.add_usage(usage) 322 return sum 323 324 def sum_tracks(self, tracks, sum=None): 325 if sum is None: 326 sum = RecountSum(mach_times=self._mach_times) 327 for usage in self.track_usages(tracks): 328 sum.add_usage(usage) 329 return sum 330 331 332def GetTaskTerminatedUserSysTime(task): 333 plan = RecountPlan('task_terminated') 334 sum = RecountSum() 335 for usage in plan.usages(task.tk_recount.rtk_terminated): 336 sum.add_usage(usage) 337 return sum.user_sys_times() 338 339 340def GetThreadUserSysTime(thread): 341 plan = RecountPlan('thread') 342 sum = RecountSum() 343 for usage in plan.track_usages(thread.th_recount.rth_lifetime): 344 sum.add_usage(usage) 345 return sum.user_sys_times() 346 347 348def print_threads(plan, thread_ptrs, indent=False, O=None): # noqa: E741 349 for thread_ptr in thread_ptrs: 350 thread = kern.GetValueFromAddress(thread_ptr, 'thread_t') 351 print('{}thread 0x{:x} 0x{:x} {}'.format( 352 ' ' if indent else '', unsigned(thread.thread_id), 353 unsigned(thread), GetThreadName(thread))) 354 with O.table(plan.usage_header(), indent=indent): 355 print(plan.format_tracks(thread.th_recount.rth_lifetime, O=O)) 356 357 358def RecountThread( 359 thread_ptrs, cmd_options={}, indent=False, O=None): # noqa: E741 360 plan = RecountPlan('thread', mach_times='-M' in cmd_options) 361 print_threads(plan, thread_ptrs, indent=indent, O=O) 362 363 364def get_task_age_ns(task): 365 start_abs = GetProcStartAbsTimeForTask(task) 366 if start_abs is not None: 367 return kern.GetNanotimeFromAbstime(GetRecentTimestamp() - start_abs) 368 return None 369 370 371def print_task_description(task): 372 task_name = GetProcNameForTask(task) 373 task_age_ns = get_task_age_ns(task) 374 if task_age_ns is not None: 375 duration_desc = '{:.3f}s'.format(task_age_ns / 1e9) 376 else: 377 duration_desc = '-s' 378 print('task 0x{:x} {} ({} old)'.format( 379 unsigned(task), task_name, duration_desc)) 380 return task_name 381 382 383def RecountTask(task_ptrs, cmd_options={}, O=None): # noqa: E741 384 if '-F' in cmd_options: 385 tasks = FindTasksByName(cmd_options['-F']) 386 else: 387 tasks = [kern.GetValueFromAddress(t, 'task_t') for t in task_ptrs] 388 mach_times = '-M' in cmd_options 389 plan = RecountPlan('task', mach_times=mach_times) 390 terminated_plan = RecountPlan('task_terminated', mach_times=mach_times) 391 active_threads = '-T' in cmd_options 392 if active_threads: 393 thread_plan = RecountPlan('thread', mach_times=mach_times) 394 for task in tasks: 395 task_name = print_task_description(task) 396 with O.table(plan.usage_header()): 397 print(plan.format_tracks(task.tk_recount.rtk_lifetime, O=O)) 398 if active_threads: 399 threads = [unsigned(t) for t in IterateQueue( 400 task.threads, 'thread *', 'task_threads')] 401 print_threads(thread_plan, threads, indent=True, O=O) 402 print('task (terminated threads) 0x{:x} {}'.format( 403 unsigned(task), task_name)) 404 with O.table(terminated_plan.usage_header()): 405 print(terminated_plan.format_usages( 406 task.tk_recount.rtk_terminated, O=O)) 407 408 409def RecountCoalition(coal_ptrs, cmd_options={}, O=None): # noqa: E741 410 plan = RecountPlan('coalition', mach_times='-M' in cmd_options) 411 coals = [kern.GetValueFromAddress(c, 'coalition_t') for c in coal_ptrs] 412 for coal in coals: 413 print('coalition 0x{:x} {}'.format(unsigned(coal), unsigned(coal.id))) 414 with O.table(plan.usage_header()): 415 print(plan.format_usages(coal.r.co_recount.rco_exited, O=O)) 416 417 418def get_processor(ptr_or_id): 419 ptr_or_id = unsigned(ptr_or_id) 420 if ptr_or_id < 1024: 421 processor_list = kern.GetGlobalVariable('processor_list') 422 current_processor = processor_list 423 while unsigned(current_processor) > 0: 424 if unsigned(current_processor.cpu_id) == ptr_or_id: 425 return current_processor 426 current_processor = current_processor.processor_list 427 raise ArgumentError('no processor found with CPU ID {}'.format( 428 ptr_or_id)) 429 else: 430 return kern.GetValueFromAddress(ptr_or_id, 'processor_t') 431 432 433def get_all_processors(): 434 processors = [] 435 processor_list = kern.GetGlobalVariable('processor_list') 436 current_processor = processor_list 437 while unsigned(current_processor) > 0: 438 processors.append(current_processor) 439 current_processor = current_processor.processor_list 440 return sorted(processors, key=lambda p: p.cpu_id) 441 442 443def RecountProcessor(pr_ptrs_or_ids, cmd_options={}, O=None): # noqa: E741 444 mach_times = '-M' in cmd_options 445 plan = RecountPlan('processor', mach_times=mach_times) 446 if '-A' in cmd_options: 447 prs = get_all_processors() 448 else: 449 prs = [get_processor(p) for p in pr_ptrs_or_ids] 450 active_threads = '-T' in cmd_options 451 if active_threads: 452 thread_plan = RecountPlan('thread', mach_times=mach_times) 453 hdr_prefix = '{:>18s} {:>4s} {:>4s} '.format('processor', 'cpu', 'kind',) 454 header_fmt = ' {:>12s} {:>12s} {:>8s}' 455 hdr_suffix = header_fmt.format('idle-time', 'total-time', 'idle-pct') 456 null_suffix = header_fmt.format('-', '-', '-') 457 levels = RecountPlan.levels() 458 with O.table(hdr_prefix + plan.usage_header() + hdr_suffix): 459 for pr in prs: 460 usage = pr.pr_recount.rpr_active.rt_usage 461 idle_time = pr.pr_recount.rpr_idle_time_mach 462 times = [usage.ru_metrics[i].rm_time_mach for (_, i) in levels] 463 total_time = sum(times) + idle_time 464 if not mach_times: 465 idle_time = kern.GetNanotimeFromAbstime(idle_time) / 1e9 466 total_time = kern.GetNanotimeFromAbstime(total_time) / 1e9 467 kind = 'SMP' 468 pset = pr.processor_set 469 if unsigned(pset.pset_cluster_type) == 1: 470 kind = 'E' 471 elif unsigned(pset.pset_cluster_type) == 2: 472 kind = 'P' 473 474 prefix = '{:<#018x} {:>4d} {:>4s} '.format( 475 unsigned(pr), pr.cpu_id, kind) 476 suffix = ( 477 ' ' + plan.time_fmt().format(idle_time) + ' ' + 478 plan.time_fmt().format(total_time) + 479 ' {:>7.2f}%'.format(idle_time / total_time * 100)) 480 usage_lines = plan.format_usage(usage, O=O) 481 for (i, line) in enumerate(usage_lines): 482 line_suffix = null_suffix 483 if i + 1 == len(usage_lines): 484 line_suffix = suffix 485 O.write(prefix + line + line_suffix + '\n') 486 if active_threads: 487 active_thread = unsigned(pr.active_thread) 488 if active_thread != 0: 489 print_threads( 490 thread_plan, [active_thread], indent=True, O=O) 491 492 493@header('{:>4s} {:>20s} {:>20s} {:>20s}'.format( 494 'cpu', 'time-mach', 'cycles', 'insns')) 495def GetRecountSnapshot(cpu, snap, O=None): 496 (insns, cycles) = (0, 0) 497 if hasattr(snap, 'rsn_cycles'): 498 (insns, cycles) = (snap.rsn_insns, snap.rsn_cycles) 499 return O.format( 500 '{:4d} {:20d} {:20d} {:20d}', cpu, snap.rsn_time_mach, 501 cycles, insns) 502 503 504def GetRecountProcessorState(pr): 505 state_time = pr.pr_recount.rpr_state_last_abs_time 506 state = state_time >> 63 507 return ( 508 pr.pr_recount.rpr_snap, 509 'I' if state == 1 else 'A', 510 state_time & ~(0x1 << 63)) 511 512 513@header('{:>20s} {:>4s} {:>6s} {:>18s} {:>18s} {:>18s} {:>18s} {:>18s}'.format( 514 'processor', 'cpu', 'state', 'last-idle-change', 'last-user-change', 515 'last-disp', 'since-idle-change', 'since-user-change')) 516def GetRecountProcessorDiagnostics(pr, cur_time, O=None): 517 (snap, state, time) = GetRecountProcessorState(pr) 518 cpu_id = unsigned(pr.cpu_id) 519 last_usrchg = snap.rsn_time_mach 520 since_usrchg = cur_time - last_usrchg 521 last_disp = '{}{:>d}'.format( 522 '*' if cur_time == unsigned(pr.last_dispatch) else '', 523 pr.last_dispatch) 524 return O.format( 525 '{:>#20x} {:4d} {:>6s} {:>18d} {:>18d} {:>18s} {:>18d} {:>18d}', 526 unsigned(pr), cpu_id, state, time, last_usrchg, last_disp, 527 cur_time - time, since_usrchg) 528 529 530@header('{:>12s} {:>6s} {:>12s} {:>20s} {:>20s}'.format( 531 'group', 'level', 'time', 'cycles', 'insns')) 532def RecountDiagnoseTask(task_ptrs, cmd_options={}, O=None): # noqa: E74 533 if '-F' in cmd_options: 534 tasks = FindTasksByName(cmd_options['-F']) 535 else: 536 tasks = [kern.GetValueFromAddress(t, 'task_t') for t in task_ptrs] 537 538 line_fmt = '{:20s} = {:10.3f}' 539 row_fmt = '{:>12s} {:>6s} {:>12.3f} {:>20d} {:>20d}' 540 541 task_plan = RecountPlan('task', mach_times=False) 542 term_plan = RecountPlan('task_terminated', mach_times=False) 543 for task in tasks: 544 print_task_description(task) 545 with O.table(RecountDiagnoseTask.header): 546 task_sum = task_plan.sum_tracks(task.tk_recount.rtk_lifetime) 547 for line in task_sum.fmt_basic_args(): 548 line = line[:-1] 549 print(O.format(row_fmt, 'task', *line)) 550 551 term_sum = term_plan.sum_usages(task.tk_recount.rtk_terminated) 552 for line in term_sum.fmt_basic_args(): 553 print(O.format(row_fmt, 'terminated', *line)) 554 term_sum_ns = term_sum.time() 555 556 threads_sum = RecountSum(mach_times=True) 557 threads_time_mach = threads_sum.time() 558 for thread in IterateQueue( 559 task.threads, 'thread *', 'task_threads'): 560 usr_time, sys_time = GetThreadUserSysTime(thread) 561 threads_time_mach += usr_time + sys_time 562 563 threads_sum_ns = kern.GetNanotimeFromAbstime(threads_time_mach) 564 print(line_fmt.format('threads CPU', threads_sum_ns / 1e9)) 565 566 all_threads_sum_ns = threads_sum_ns + term_sum_ns 567 print(line_fmt.format('all threads CPU', all_threads_sum_ns / 1e9)) 568 569 print(line_fmt.format( 570 'discrepancy', task_sum.time() - all_threads_sum_ns)) 571 572 573def RecountDiagnose(cmd_args=[], cmd_options={}, O=None): # noqa: E741 574 if not cmd_args: 575 raise ArgumentError('diagnose subcommand required') 576 577 if cmd_args[0] == 'task': 578 validate_args(cmd_options, ['F']) 579 RecountDiagnoseTask(cmd_args[1:], cmd_options=cmd_options, O=O) 580 else: 581 raise ArgumentError('{}: invalid diagnose subcommand'.format( 582 cmd_args[0])) 583 584 585def RecountTriage(cmd_options={}, O=None): # noqa: E741 586 prs = get_all_processors() 587 print('processors') 588 with O.table(GetRecountProcessorDiagnostics.header, indent=True): 589 max_dispatch = max([unsigned(pr.last_dispatch) for pr in prs]) 590 for pr in prs: 591 print(GetRecountProcessorDiagnostics( 592 pr, cur_time=max_dispatch, O=O)) 593 594 print('snapshots') 595 with O.table(GetRecountSnapshot.header, indent=True): 596 for (i, pr) in enumerate(prs): 597 print(GetRecountSnapshot(i, pr.pr_recount.rpr_snap, O=O)) 598