xref: /xnu-11215.41.3/tools/format_vm_parameter_validation.py (revision 33de042d024d46de5ff4e89f2471de6608e37fa4)

#!/usr/bin/python3

# format_vm_parameter_validation.py
# Pretty-print the output of tests/vm/vm_parameter_validation.c
#
# usage:
#     vm_parameter_validation | format_vm_parameter_validation.py

import re
import sys
import copy
import itertools

# magic return values used for in-band signalling
# fixme duplicated in vm_parameter_validation.c
# fixme also duplicated in other_return_values below
RESULT_SUCCESS  = 0
RESULT_BUSTED   = -99
RESULT_IGNORED  = -98
RESULT_ZEROSIZE = -97
RESULT_PANIC    = -96
RESULT_GUARD    = -95
RESULT_MISMATCH = -94
RESULT_OUT_PARAM_BAD = -93
RESULT_MACH_SEND_INVALID_MEMORY = 0x1000000c

# output formatting
format_result = {
    RESULT_SUCCESS  : '  .',
    RESULT_BUSTED   : ' **',
    RESULT_MISMATCH : ' ##',
    RESULT_IGNORED  : '   ',
    RESULT_ZEROSIZE : '  o',
    RESULT_PANIC    : ' pp',
    RESULT_GUARD    : ' gg',
    RESULT_OUT_PARAM_BAD: ' ot',
    RESULT_MACH_SEND_INVALID_MEMORY : ' mi',
}

format_default = '%3d'
format_col_width = 3
format_empty_col = format_col_width * ' '
format_indent_width = 4
format_indent = format_indent_width * ' '


# record the result of one trial:
# ret: the return value from the tested function
# parameters: array of the input parameter names for that trial
#   (for example ["start PGSZ-2", "size -1"])
class Result:
    def __init__(self, new_ret, new_parameters):
        self.ret = new_ret
        self.parameters = new_parameters
    def __repr__(self):
        return str(self.ret) + " = " + str(self.parameters)

# record the results of all trials in one test
# testname: the name of the test (including the function being tested)
# config: a string describing OS, CPU, etc
# compat: code for error compatibility
# results: an array of Result, one per trial
class Test:
    def __init__(self, new_name, new_config, new_compat, new_results = []):
        self.testname = new_name
        self.config = new_config
        self.compat = new_compat
        self.results = new_results

# print column labels under some output
# example output given indent=2 col_width=4 labels=[foo,bar,baz,qux]:
#  |   |   |   |
#  |   |   |   qux
#  |   |   baz
#  |   bar
#  foo
def print_column_labels(labels, indent_width, col_width):
    indent = indent_width * ' '
    empty_column = '|' + (col_width-1) * ' '

    unprinted = len(labels)
    print(indent + unprinted*empty_column)

    for label in reversed(labels):
        unprinted -= 1
        print(indent + unprinted*empty_column + label)

# pretty-print one function return code
def print_one_result(ret):
    if ret in format_result:
        print(format_result[ret], end='')
    else:
        print(format_default % (ret), end='')

# choose the appropriate error code table for a test
# (either errno_return_values or kern_return_values)
def error_code_values_for_test(test):
    errno_fns = ['mprotect', 'msync', 'minherit', 'mincore', 'mlock', 'munlock',
                 'mmap', 'munmap', 'mremap_encrypted', 'vslock', 'vsunlock',
                 'madvise', 'useracc']
    for fn in errno_fns:
        if test.testname.startswith(fn):
            return errno_return_values
    else:
        return kern_return_values

# print a helpful description of the return values seen in results
# fixme these won't include RESULT_MISMATCH
def print_legend(test):
    # find all error codes represented in the results
    codes = {}
    for result in test.results:
        codes[result.ret] = True

    known_return_values = error_code_values_for_test(test)

    # print the names of the detected error codes
    output = []
    for code in sorted(codes.keys()):
        if code in known_return_values:
            output.append(str(code) + ': ' + known_return_values[code])
        elif code in other_return_values:
            output.append(other_return_values[code])
        elif code != 0:
            output.append(str(code) + ': ????')

    print(format_indent + '(' + ', '.join(output) + ')')

# display names for error codes returned in errno
errno_return_values = {
    1: 'EPERM',
    9: 'EBADF',
    12: 'ENOMEM',
    13: 'EACCES',
    14: 'EFAULT',
    22: 'EINVAL',
}

# display names for error codes returned in kern_return_t
kern_return_values = {
    1: 'KERN_INVALID_ADDRESS',
    2: 'KERN_PROTECTION_FAILURE',
    3: 'KERN_NO_SPACE',
    4: 'KERN_INVALID_ARGUMENT',
    5: 'KERN_FAILURE',
    6: 'KERN_RESOURCE_SHORTAGE',
    7: 'KERN_NOT_RECEIVER',
    8: 'KERN_NO_ACCESS',
    9: 'KERN_MEMORY_FAILURE',
    10: 'KERN_MEMORY_ERROR',
    11: 'KERN_ALREADY_IN_SET',
    12: 'KERN_NOT_IN_SET',
    13: 'KERN_NAME_EXISTS',
    14: 'KERN_ABORTED',
    15: 'KERN_INVALID_NAME',
    16: 'KERN_INVALID_TASK',
    17: 'KERN_INVALID_RIGHT',
    18: 'KERN_INVALID_VALUE',
    19: 'KERN_UREFS_OVERFLOW',
    20: 'KERN_INVALID_CAPABILITY',
    21: 'KERN_RIGHT_EXISTS',
    22: 'KERN_INVALID_HOST',
    23: 'KERN_MEMORY_PRESENT',
    24: 'KERN_MEMORY_DATA_MOVED',
    25: 'KERN_MEMORY_RESTART_COPY',
    26: 'KERN_INVALID_PROCESSOR_SET',
    27: 'KERN_POLICY_LIMIT',
    28: 'KERN_INVALID_POLICY',
    29: 'KERN_INVALID_OBJECT',
    30: 'KERN_ALREADY_WAITING',
    31: 'KERN_DEFAULT_SET',
    32: 'KERN_EXCEPTION_PROTECTED',
    33: 'KERN_INVALID_LEDGER',
    34: 'KERN_INVALID_MEMORY_CONTROL',
    35: 'KERN_INVALID_SECURITY',
    36: 'KERN_NOT_DEPRESSED',
    37: 'KERN_TERMINATED',
    38: 'KERN_LOCK_SET_DESTROYED',
    39: 'KERN_LOCK_UNSTABLE',
    40: 'KERN_LOCK_OWNED',
    41: 'KERN_LOCK_OWNED_SELF',
    42: 'KERN_SEMAPHORE_DESTROYED',
    43: 'KERN_RPC_SERVER_TERMINATED',
    44: 'KERN_RPC_TERMINATE_ORPHAN',
    45: 'KERN_RPC_CONTINUE_ORPHAN',
    46: 'KERN_NOT_SUPPORTED',
    47: 'KERN_NODE_DOWN',
    48: 'KERN_NOT_WAITING',
    49: 'KERN_OPERATION_TIMED_OUT',
    50: 'KERN_CODESIGN_ERROR',
    51: 'KERN_POLICY_STATIC',
    52: 'KERN_INSUFFICIENT_BUFFER_SIZE',
    53: 'KERN_DENIED',
    54: 'KERN_MISSING_KC',
    55: 'KERN_INVALID_KC',
    56: 'KERN_NOT_FOUND',
    100: 'KERN_RETURN_MAX',
    -304: 'MIG_BAD_ARGUMENTS (server type check failure)',
    0x1000000c : 'MACH_SEND_INVALID_MEMORY',
}

# display names for the special return values used by the test machinery
other_return_values = {
    RESULT_BUSTED: format_result[RESULT_BUSTED].lstrip() + ': trial broken, not performed',
    RESULT_IGNORED: '<empty> trial ignored, not performed',
    RESULT_ZEROSIZE: format_result[RESULT_ZEROSIZE].lstrip() + ': size == 0',
    RESULT_PANIC: format_result[RESULT_PANIC].lstrip() + ': trial is believed to panic, not performed',
    RESULT_GUARD: format_result[RESULT_GUARD].lstrip() + ': trial is believed to throw EXC_GUARD, not performed',
    RESULT_OUT_PARAM_BAD: format_result[RESULT_OUT_PARAM_BAD].lstrip() + ': trial set incorrect values to out parameters',
}

# inside line, replace 'return 123' with 'return ERR_CODE_NAME'
def replace_error_code_return(test, line):
    known_return_values = error_code_values_for_test(test)
    for code, name in known_return_values.items():
        line = line.replace('return ' + str(code) + ';', 'return ' + name + ';')
    return line

def dimensions(results):
    if len(results) == 0:
        return 0
    return len(results[0].parameters)

# given one k-dimensional results
# return a list of k counts that is the size of each dimension
def count_each_dimension(results):
    if len(results) == 0:
        return []
    first = results[0].parameters
    k = dimensions(results)
    counts = []
    step = 1
    for dim in range(k-1, -1, -1):
        count = round(len(results) / step)
        for i in range(0, len(results), step):
            cur = results[i].parameters
            if i != 0 and cur[dim] == first[dim]:
                count = round(i / step)
                break;
        step *= count
        counts.append(count)

    counts.reverse()
    return counts;

# Reduce one k-dimensional results to many (k-1) dimensional results
# Yields a sequence of [results, name] pairs
# where results has k-1 dimensions
# and name is the parameter name from the removed dimension
def iterate_dimension(results, dim = 0):
    if len(results) == 0:
        return

    k = dimensions(results)
    dim_counts = count_each_dimension(results)

    inner_count = 1
    for d in range(dim+1, k):
        inner_count *= dim_counts[d]

    outer_step = len(results)
    for d in range(0, dim):
        outer_step = int(outer_step / dim_counts[d])

    for r in range(dim_counts[dim]):
        start = r * inner_count
        name = results[start].parameters[dim]
        new_results = []
        for i in range(start, len(results), outer_step):
            for j in range(inner_count):
                new_result = copy.deepcopy(results[i+j])
                del new_result.parameters[dim]
                new_results.append(new_result)
        yield [new_results, name]

# Print the results of a test that has two parameters (for example a test of start/size)
# If overrides!=None, use any non-SUCCESS return values from override in place of the other results.
def print_results_2D(results, overrides=None):
    # complain if results and override have different dimensions
    if overrides:
        if len(overrides) != len(results):
            print("WARNING: override results have a different height; overrides ignored")
        for i, result in enumerate(results):
            if len(overrides[i].parameters) != len(result.parameters):
                print("WARNING: override results have a different width; overrides ignored")

    columns = []
    prev_row_label = ''
    first_row_label = ''
    for i, result in enumerate(results):
        if overrides: override = overrides[i].ret

        if first_row_label == '':
            # record first row's name so we can use it to find columns
            # (assumes every row has the same column labels)
            first_row_label = result.parameters[0]

        if result.parameters[0] == first_row_label:
            # record column names in the first row
            columns.append(result.parameters[1])

        if result.parameters[0] != prev_row_label:
            # new row
            if prev_row_label != '': print(format_indent + prev_row_label)
            print(format_indent, end='')
            prev_row_label = result.parameters[0]

        if overrides and override != RESULT_SUCCESS:
            print_one_result(override)
        else:
            print_one_result(result.ret)

    if prev_row_label: print(format_indent + prev_row_label)
    print_column_labels(columns, format_indent_width + format_col_width - 1, format_col_width)

def print_results_2D_try_condensed(results):
    if 0 == len(results):
        return
    singleton = results[0].ret
    if any([result.ret != singleton for result in results]):
        print_results_2D(results)
        return
    # will print as condensed
    rows = set()
    cols = set()
    for result in results:
        rows.add(result.parameters[0].split()[1])
        cols.add(result.parameters[1].split()[1])
    print_one_result(result.ret)
    print(" for all pairs")

def print_results_3D(results, testname):
    # foreach parameter[1], print 2D table of parameter[0] and parameter[2]
    for results2D, name in iterate_dimension(results, 1):
        print(testname + ': ' + name)
        print_results_2D(results2D)

    # foreach parameter[0], print 2D table of parameter[1] and parameter[2]
    # This is redundant but can be useful for human readers.
    for results2D, name in iterate_dimension(results, 0):
        print(testname + ': ' + name)
        print_results_2D(results2D)

def print_results_4D(results):
    x, y, z = '', '', ''
    # Make a map[{3rd_param, 4th_param, ...}] = {all options}
    # For now, we print 2d tables of 1st, 2nd param for each possible combination of remaining values

    map_of_results = {}
    for _, result in enumerate(results):
        k = tuple(result.parameters[2:])

        if k not in map_of_results:
            map_of_results[k] = [result]
        else:
            map_of_results[k].append(result)

    # prepare to iterate
    prev_matrix = []
    iterable = []
    for k, result_list in map_of_results.items():
        one_2d_result = []
        matrix = []
        for result in result_list:
            x = result.parameters[0]
            y = result.parameters[1]
            repl_result = Result(result.ret, (x, y))
            one_2d_result.append(repl_result)
            matrix.append(result.ret)
        if matrix == prev_matrix:
            # if the return codes are the same everywhere, we will print successive tables only once
            # note that this assumes that the sets of 2D labels are the same everywhere, and doesn't check that assumption
            iterable[-1][0].append(k)
        else:
            iterable.append(([k], one_2d_result))
        prev_matrix = matrix

    # print
    for iter in iterable:
        print(iter[0])
        print_results_2D_try_condensed(iter[1])


# Print the results of a test that has two parameters
# (for example a test of addr only, or size only)
# If overrides!=None, use any non-SUCCESS return values from override in place of the other results.
def print_results_1D(results, overrides=None):
    # complain if results and overrides have different dimensions
    if overrides:
        if len(overrides) != len(results):
            print("WARNING: override results have a different height; overrides ignored")
        for i, result in enumerate(results):
            if len(overrides[i].parameters) != len(result.parameters):
                print("WARNING: override results have a different width; overrides ignored")

    for i, result in enumerate(results):
        if overrides: override = overrides[i].ret

        # indent, value, indent, label
        print(format_indent, end='')
        if overrides and override != RESULT_SUCCESS:
            print_one_result(override)
        else:
            print_one_result(result.ret)
        print(format_indent + result.parameters[0])

def print_results_nD(results, testname, overrides=None):
    if (dimensions(results) == 1):
        print_results_1D(results, overrides)
    elif (dimensions(results) == 2):
        print_results_2D(results, overrides)
    elif dimensions(results) == 3:
        print_results_3D(results, testname)
    elif dimensions(results) == 4:
        print_results_4D(results)
    else:
        print(format_indent + 'too many dimensions')


def main():
    data = sys.stdin.readlines()


    # remove any lines that don't start with "TESTNAME" or "TESTCONFIG" or "RESULT"
    # (including darwintest output like "PASS" or "FAIL")
    # and print them now
    # Also verify that the counts of "TEST BEGIN" == "TEST END"
    # (they will mismatch if a test suite crashed)
    testbegincount = 0
    testendcount = 0
    testlines = []
    for line in data:
        unmodified_line = line
        # count TEST BEGIN and TEST END
        if ('TEST BEGIN' in line):
            testbegincount += 1
        if ('TEST END' in line):
            testendcount += 1
        # remove any T_LOG() timestamp prefixes and KTEST prefixes
        line = re.sub('^\s*\d+:\d+:\d+ ', '', line)
        line = re.sub('^\[KTEST\]\s+[A-Z]+\s+\d+\s+(\d+\s+)?\S+\s+\d+\s+', '', line)
        line = line.lstrip()

        if (line.startswith('TESTNAME') or line.startswith('RESULT')
            or line.startswith('TESTCONFIG') or line.startswith('TESTCOMPAT')):
            testlines.append(line)  # line is test output
        elif line == '':
            pass # ignore empty lines
        else:
            print(unmodified_line, end='')  # line is other output

    # parse test output into Test and Result objects

    testnum = 0
    def group_by_test(line):
        nonlocal testnum
        if line.startswith('TESTNAME '):
            testnum = testnum+1
        return testnum

    tests = []
    for _, group in itertools.groupby(testlines, group_by_test):
        lines = list(group)

        name = lines.pop(0).removeprefix('TESTNAME ').rstrip()
        config = lines.pop(0).removeprefix('TESTCONFIG ').rstrip()
        compat = []
        results = []
        for line in lines:
            if line.startswith('RESULT'):
                components = line.removeprefix('RESULT ').rstrip().split(', ')
                ret = int(components.pop(0))
                results.append(Result(ret, components))

        tests.append(Test(name, config, compat, results))

    print('found %d tests' % (len(tests)))

    # stats to print at the end
    test_count = len(tests)
    all_configurations = set()

    # print test output
    for test in tests:
        # print test name and test config on separate lines
        # `diff` handles this better than putting both on the same line
        print('test ' + test.testname)

        print(format_indent + 'config ' + test.config)
        all_configurations.add(test.config)

        if len(test.results) == 0:
            print(format_indent + 'no results')
        else:
            print_legend(test)
            print_results_nD(test.results, test.testname)


        print('end  ' + test.testname)

    print()
    print(str(test_count) + ' test(s) performed')

    if (testbegincount != testendcount):
        print('### error: %d TEST BEGINs, %d TEST ENDs - some tests may have crashed'
              % (testbegincount, testendcount))

    print(str(len(all_configurations)) + ' configuration(s) tested:')
    for config in sorted(all_configurations):
        print(format_indent + '[' + config + ']')


main()