From 3ce152f61cd397ac6c0b10ea77afefcd6070e560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Tue, 22 Jan 2019 00:00:00 +0000 Subject: [PATCH] test-gdb: move to pure python calls --- build | 2 +- build-baremetal | 9 +++-- cli_function.py | 65 ++++++++++++++++++++++++----------- common.py | 55 +++++++++++++++++++++++------- getvar | 7 ++-- run | 18 ++++++---- run-gdb | 5 ++- run-gdb-user | 2 +- shell_helpers.py | 31 ++++++++++------- test-gdb | 88 +++++++++++++++++++++++++++++------------------- 10 files changed, 182 insertions(+), 100 deletions(-) diff --git a/build b/build index a35c2811..2399d5c6 100755 --- a/build +++ b/build @@ -86,7 +86,7 @@ buildroot_component = Component( name_to_component_map = { # Leaves without dependencies. 'baremetal-qemu': Component( - lambda arch: run_cmd(['build-baremetal', '--qemu'], arch), + lambda arch: run_cmd(['build-baremetal', '--emulator', 'qemu'], arch), supported_archs=kwargs['crosstool_ng_supported_archs'], ), 'baremetal-gem5': Component( diff --git a/build-baremetal b/build-baremetal index c37bc7d6..f0fd95c3 100755 --- a/build-baremetal +++ b/build-baremetal @@ -92,10 +92,9 @@ Build the baremetal examples with crosstool-NG. bootloader_obj=bootloader_obj, common_objs=common_objs, ) - arch_dir = os.path.join('arch', self.env['arch']) - if os.path.isdir(os.path.join(self.env['baremetal_src_dir'], arch_dir)): + if os.path.isdir(os.path.join(self.env['baremetal_src_arch_dir'])): self._build_dir( - arch_dir, + self.env['baremetal_src_arch_subpath'], gcc=gcc, cflags=cflags, entry_address=entry_address, @@ -127,12 +126,12 @@ Build the baremetal examples with crosstool-NG. common_objs, bootloader=True ): - """ + ''' Build all .c and .S files in a given subpath of the baremetal source directory non recursively. Place outputs on the same subpath or the output directory. - """ + ''' in_dir = os.path.join(self.env['baremetal_src_dir'], subpath) out_dir = os.path.join(self.env['baremetal_build_dir'], subpath) os.makedirs(out_dir, exist_ok=True) diff --git a/cli_function.py b/cli_function.py index e92e1136..ca90f245 100755 --- a/cli_function.py +++ b/cli_function.py @@ -4,7 +4,7 @@ import argparse import imp import os -class Argument: +class _Argument: def __init__( self, long_or_short_1, @@ -14,28 +14,22 @@ class Argument: nargs=None, **kwargs ): - if long_or_short_2 is None: - shortname = None - longname = long_or_short_1 - else: - shortname = long_or_short_1 - longname = long_or_short_2 self.args = [] # argparse is crappy and cannot tell us if arguments were given or not. # We need that information to decide if the config file should override argparse or not. # So we just use None as a sentinel. self.kwargs = {'default': None} + shortname, longname, key, is_option = self.get_key( + long_or_short_1, + long_or_short_2 + ) if shortname is not None: self.args.append(shortname) - if longname[0] == '-': + if is_option: self.args.append(longname) - self.key = longname.lstrip('-').replace('-', '_') - self.is_option = True else: - self.key = longname.replace('-', '_') - self.args.append(self.key) + self.args.append(key) self.kwargs['metavar'] = longname - self.is_option = False if default is not None and nargs is None: self.kwargs['nargs'] = '?' if nargs is not None: @@ -53,22 +47,51 @@ class Argument: if self.is_bool and not 'action' in kwargs: self.kwargs['action'] = bool_action if help is not None: - if not self.is_bool and default: - help += ' Default: {}'.format(default) + if default is not None: + if help[-1] == '\n': + if '\n\n' in help[:-1]: + help += '\n' + elif help[-1] == ' ': + pass + else: + help += ' ' + help += 'Default: {}'.format(default) self.kwargs['help'] = help self.optional = ( default is not None or self.is_bool or - self.is_option or + is_option or nargs in ('?', '*', '+') ) self.kwargs.update(kwargs) self.default = default self.longname = longname + self.key = key + self.is_option = is_option def __str__(self): return str(self.args) + ' ' + str(self.kwargs) + @staticmethod + def get_key( + long_or_short_1, + long_or_short_2=None, + **kwargs + ): + if long_or_short_2 is None: + shortname = None + longname = long_or_short_1 + else: + shortname = long_or_short_1 + longname = long_or_short_2 + if longname[0] == '-': + key = longname.lstrip('-').replace('-', '_') + is_option = True + else: + key = longname.replace('-', '_') + is_option = False + return shortname, longname, key, is_option + class CliFunction: ''' Represent a function that can be called either from Python code, or @@ -141,7 +164,7 @@ class CliFunction: *args, **kwargs ): - argument = Argument(*args, **kwargs) + argument = _Argument(*args, **kwargs) self._arguments.append(argument) self._all_keys.add(argument.key) @@ -170,6 +193,10 @@ class CliFunction: args = parser.parse_args(args=cli_args) return self(**vars(args)) + @staticmethod + def get_key(*args, **kwargs): + return _Argument.get_key(*args, **kwargs) + def main(self, **kwargs): ''' Do the main function call work. @@ -217,7 +244,7 @@ amazing function! 'args_star': [] } - # Default CLI call. + # Default CLI call with programmatic CLI arguments. out = one_cli_function.cli(['1']) assert out == default @@ -258,5 +285,5 @@ amazing function! # Force a boolean value set on the config to be False on CLI. assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False - # CLI call. + # CLI call with argv command line arguments. print(one_cli_function.cli()) diff --git a/common.py b/common.py index 217ae1fb..93b7cbef 100644 --- a/common.py +++ b/common.py @@ -92,6 +92,7 @@ consts['obj_ext'] = '.o' consts['config_file'] = os.path.join(consts['data_dir'], 'config.py') consts['magic_fail_string'] = b'lkmc_test_fail' consts['baremetal_lib_basename'] = 'lib' +consts['emulators'] = ['qemu', 'gem5'] class LkmcCliFunction(cli_function.CliFunction): ''' @@ -100,9 +101,15 @@ class LkmcCliFunction(cli_function.CliFunction): * command timing * some common flags, e.g.: --arch, --dry-run, --verbose ''' - def __init__(self, *args, do_print_time=True, **kwargs): + def __init__(self, *args, defaults=None, **kwargs): + ''' + :ptype defaults: Dict[str,Any] + :param defaults: override the default value of an argument + ''' kwargs['config_file'] = consts['config_file'] - self._do_print_time = do_print_time + if defaults is None: + defaults = {} + self._defaults = defaults super().__init__(*args, **kwargs) # Args for all scripts. @@ -122,6 +129,10 @@ Bash equivalents even for actions taken directly in Python without shelling out. mkdir are generally omitted since those are obvious ''' ) + self.add_argument( + '--print-time', default=True, + help='Print how long it took to run the command at the end.' + ) self.add_argument( '-v', '--verbose', default=False, help='Show full compilation commands when they are not shown by default.' @@ -266,27 +277,29 @@ instances in parallel. Default: the run ID (-n) if that is an integer, otherwise # Misc. self.add_argument( - '-g', '--gem5', default=False, - help='Use gem5 instead of QEMU.' + '--emulator', choices=consts['emulators'], + help='''\ +Set the emulator to use. Ignore --gem5. +''' ) self.add_argument( - '--qemu', default=False, + '-g', '--gem5', default=False, help='''\ -Use QEMU as the emulator. This option exists in addition to --gem5 -to allow overriding configs from the CLI. +Use gem5 instead of QEMU. Shortcut for `--emulator gem5`. ''' ) def _init_env(self, env): ''' - Update the kwargs from the command line with derived arguments. + Update the kwargs from the command line with values derived from them. ''' def join(*paths): return os.path.join(*paths) - if env['qemu'] or not env['gem5']: - env['emulator'] = 'qemu' - else: - env['emulator'] = 'gem5' + if env['emulator'] is None: + if env['gem5']: + env['emulator'] = 'gem5' + else: + env['emulator'] = 'qemu' if env['arch'] in env['arch_short_to_long_dict']: env['arch'] = env['arch_short_to_long_dict'][env['arch']] if env['userland_build_id'] is None: @@ -481,6 +494,8 @@ to allow overriding configs from the CLI. # Baremetal. env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal') + env['baremetal_src_arch_subpath'] = join('arch', env['arch']) + env['baremetal_src_arch_dir'] = join(env['baremetal_src_dir'], env['baremetal_src_arch_subpath']) env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename']) if env['emulator'] == 'gem5': env['simulator_name'] = 'gem5' @@ -534,6 +549,12 @@ to allow overriding configs from the CLI. env['image'] = path self.env = env + def add_argument(self, *args, **kwargs): + shortname, longname, key, is_option = self.get_key(*args, **kwargs) + if key in self._defaults: + kwargs['default'] = self._defaults[key] + super().add_argument(*args, **kwargs) + def assert_crosstool_ng_supports_arch(self, arch): if arch not in self.env['crosstool_ng_supported_archs']: raise Exception('arch not yet supported: ' + arch) @@ -644,6 +665,14 @@ to allow overriding configs from the CLI. _json = {} return _json + @staticmethod + def import_path(path): + ''' + https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension + https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do + ''' + return imp.load_source(os.path.split(path)[1].replace('-', '_'), path) + @staticmethod def log_error(msg): print('error: {}'.format(msg), file=sys.stderr) @@ -687,7 +716,7 @@ to allow overriding configs from the CLI. return False def _print_time(self, ellapsed_seconds): - if self._do_print_time: + if self.env['print_time']: hours, rem = divmod(ellapsed_seconds, 3600) minutes, seconds = divmod(rem, 60) print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) diff --git a/getvar b/getvar index da7a86e3..3f62b10d 100755 --- a/getvar +++ b/getvar @@ -1,13 +1,13 @@ #!/usr/bin/env python3 -import types - import common -from shell_helpers import LF class Main(common.LkmcCliFunction): def __init__(self): super().__init__( + defaults={ + 'print_time': False, + }, description='''\ Print the value of a self.env['py'] variable. @@ -28,7 +28,6 @@ List all available variables: ./%(prog)s .... ''', - do_print_time=False, ) self.add_argument('variable', nargs='?') diff --git a/run b/run index e2c53c76..830ab0bf 100755 --- a/run +++ b/run @@ -40,6 +40,10 @@ https://superuser.com/questions/1373226/how-to-redirect-qemu-serial-output-to-bo '--debug-vm-args', default='', help='Pass arguments to GDB.' ) + self.add_argument( + '--dp650', default=False, + help='Use the ARM DP650 display processor instead of the default HDLCD on gem5.' + ) self.add_argument( '--dtb', help='''\ @@ -372,15 +376,15 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: dtb = None if self.env['dtb'] is not None: dtb = self.env['dtb'] - elif args.dp650: - dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, args.cpus)), common.Newline, + elif self.env['dp650']: + dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, self.env['cpus'])), LF, if dtb is None: - cmd.extend(['--generate-dtb', common.Newline]) + if not self.env['baremetal']: + cmd.extend(['--generate-dtb', LF]) else: - cmd.extend(['--dtb-filename', dtb, common.Newline]) + cmd.extend(['--dtb-filename', dtb, LF]) if self.env['baremetal'] is None: - cmd.extend([ - '--param', 'system.panic_on_panic = True', LF]) + cmd.extend(['--param', 'system.panic_on_panic = True', LF ]) else: cmd.extend([ '--bare-metal', LF, @@ -488,7 +492,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with: vnc ) if self.env['dtb'] is not None: - cmd.extend(['-dtb', self.env['dtb'], common.Newline]) + cmd.extend(['-dtb', self.env['dtb'], LF]) if not qemu_executable_prebuilt: cmd.extend(qemu_user_and_system_options) if self.env['initrd']: diff --git a/run-gdb b/run-gdb index 379b4069..164b8cbe 100755 --- a/run-gdb +++ b/run-gdb @@ -23,8 +23,6 @@ class GdbTestcase: ''' self.prompt = '\(gdb\) ' self.source_path = source_path - self.print_cmd(cmd) - cmd = self.strip_newlines(cmd) import pexpect self.child = pexpect.spawn( cmd[0], @@ -198,10 +196,11 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-s cmd.extend(['-ex', 'lx-symbols {}'.format(self.env['kernel_modules_build_subdir']), LF]) cmd.extend(after) if self.env['test']: + self.sh.print_cmd(cmd) GdbTestcase( self.env['source_path'], test_script_path, - cmd, + self.sh.strip_newlines(cmd), verbose=self.env['verbose'], ) else: diff --git a/run-gdb-user b/run-gdb-user index 44884d1f..14c5bdf3 100755 --- a/run-gdb-user +++ b/run-gdb-user @@ -5,7 +5,7 @@ import os import sys import common -rungdb = imp.load_source('rungdb', os.path.join(kwargs['root_dir'], 'run-gdb')) +rungdb = imp.load_source('run_gdb', os.path.join(kwargs['root_dir'], 'run-gdb')) parser = self.get_argparse(argparse_args={ 'description': '''GDB step debug guest userland processes without gdbserver. diff --git a/shell_helpers.py b/shell_helpers.py index 040b1758..28d4f312 100644 --- a/shell_helpers.py +++ b/shell_helpers.py @@ -9,6 +9,7 @@ import signal import stat import subprocess import sys +import threading class LF: ''' @@ -178,18 +179,21 @@ class ShellHelpers: if show_cmd: self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths) - # Otherwise Ctrl + C gives: - # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). - # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec - sigint_old = signal.getsignal(signal.SIGINT) - signal.signal(signal.SIGINT, signal.SIG_IGN) + # Otherwise, if called from a non-main thread: + # ValueError: signal only works in main thread + if threading.current_thread() == threading.main_thread(): + # Otherwise Ctrl + C gives: + # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine). + # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec + sigint_old = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, signal.SIG_IGN) - # Otherwise BrokenPipeError when piping through | grep - # But if I do this_module, my terminal gets broken at the end. Why, why, why. - # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python - # Ignoring the exception is not enough as it prints a warning anyways. - #sigpipe_old = signal.getsignal(signal.SIGPIPE) - #signal.signal(signal.SIGPIPE, signal.SIG_DFL) + # Otherwise BrokenPipeError when piping through | grep + # But if I do this_module, my terminal gets broken at the end. Why, why, why. + # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python + # Ignoring the exception is not enough as it prints a warning anyways. + #sigpipe_old = signal.getsignal(signal.SIGPIPE) + #signal.signal(signal.SIGPIPE, signal.SIG_DFL) cmd = self.strip_newlines(cmd) if not self.dry_run: @@ -211,8 +215,9 @@ class ShellHelpers: logfile.write(byte) else: break - signal.signal(signal.SIGINT, sigint_old) - #signal.signal(signal.SIGPIPE, sigpipe_old) + if threading.current_thread() == threading.main_thread(): + signal.signal(signal.SIGINT, sigint_old) + #signal.signal(signal.SIGPIPE, sigpipe_old) returncode = proc.returncode if returncode != 0 and raise_on_failure: raise Exception('Command exited with status: {}'.format(returncode)) diff --git a/test-gdb b/test-gdb index 3840fe4d..b3c49dbd 100755 --- a/test-gdb +++ b/test-gdb @@ -1,35 +1,55 @@ -#!/usr/bin/env bash -set -eux -for emulator in --qemu --gem5; do - # Userland. - # TODO make work. - #./run --arch x86_64 --background --userland add "$emulator" --wait-gdb & - #./run-gdb --arch x86_64 --userland add "$emulator" --test "$@" - #wait +#!/usr/bin/env python3 - # Baremetal. - ./run --arch arm --background --baremetal add "$emulator" --wait-gdb & - ./run-gdb --arch arm --baremetal add "$emulator" --test "$@" - wait - ./run --arch arm --background --baremetal arch/arm/add "$emulator" --wait-gdb & - ./run-gdb --arch arm --baremetal arch/arm/add "$emulator" --test "$@" - wait - ./run --arch arm --background --baremetal arch/arm/regs "$emulator" --wait-gdb & - ./run-gdb --arch arm --baremetal arch/arm/regs "$emulator" --test "$@" - wait - ./run --arch aarch64 --background --baremetal add "$emulator" --wait-gdb & - ./run-gdb --arch aarch64 --baremetal add "$emulator" --test "$@" - wait - ./run --arch aarch64 --background --baremetal arch/aarch64/add "$emulator" --wait-gdb & - ./run-gdb --arch aarch64 --baremetal arch/aarch64/add "$emulator" --test "$@" - wait - ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb & - ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@" - wait - ./run --arch aarch64 --background --baremetal arch/aarch64/fadd "$emulator" --wait-gdb & - ./run-gdb --arch aarch64 --baremetal arch/aarch64/fadd "$emulator" --test "$@" - wait - ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb & - ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@" - wait -done +import functools +import threading +import os + +import common + +def output_reader(proc, file): + while True: + byte = proc.stdout.read(1) + if byte: + sys.stdout.buffer.write(byte) + sys.stdout.flush() + file.buffer.write(byte) + else: + break + +class Main(common.LkmcCliFunction): + def timed_main(self): + run = self.import_path('run').Main() + run_gdb = self.import_path('run-gdb').Main() + for emulator in self.env['emulators']: + for arch in self.env['crosstool_ng_supported_archs']: + test_scripts_noext = [] + for f in os.listdir(self.env['baremetal_src_dir']): + base, ext = os.path.splitext(f) + if ext == '.py': + test_scripts_noext.append(base) + for root, dirs, files in os.walk(os.path.join(self.env['baremetal_src_dir'], 'arch', arch)): + for f in files: + base, ext = os.path.splitext(f) + if ext == '.py': + full_path = os.path.join(root, base) + relpath = os.path.relpath(full_path, self.env['baremetal_src_dir']) + test_scripts_noext.append(relpath) + for test_script_noext in test_scripts_noext: + run_thread = threading.Thread(target=lambda: run( + arch=arch, + background=True, + baremetal=test_script_noext, + print_time=False, + emulator=emulator, + wait_gdb=True + )) + gdb_thread = threading.Thread(target=lambda: run_gdb( + arch=arch, baremetal=test_script_noext, print_time=False, emulator=emulator, test=True + )) + run_thread.start() + gdb_thread.start() + run_thread.join() + gdb_thread.join() + +if __name__ == '__main__': + Main().cli()