test-gdb: move to pure python calls

build

@@ -86,7 +86,7 @@ buildroot_component = Component(
 name_to_component_map = {
     # Leaves without dependencies.
     'baremetal-qemu': Component(
-        lambda arch: run_cmd(['build-baremetal', '--qemu'], arch),
+        lambda arch: run_cmd(['build-baremetal', '--emulator', 'qemu'], arch),
         supported_archs=kwargs['crosstool_ng_supported_archs'],
     ),
     'baremetal-gem5': Component(

build-baremetal

@@ -92,10 +92,9 @@ Build the baremetal examples with crosstool-NG.
             bootloader_obj=bootloader_obj,
             common_objs=common_objs,
         )
-        arch_dir = os.path.join('arch', self.env['arch'])
-        if os.path.isdir(os.path.join(self.env['baremetal_src_dir'], arch_dir)):
+        if os.path.isdir(os.path.join(self.env['baremetal_src_arch_dir'])):
             self._build_dir(
-                arch_dir,
+                self.env['baremetal_src_arch_subpath'],
                 gcc=gcc,
                 cflags=cflags,
                 entry_address=entry_address,
@@ -127,12 +126,12 @@ Build the baremetal examples with crosstool-NG.
             common_objs,
             bootloader=True
         ):
-        """
+        '''
         Build all .c and .S files in a given subpath of the baremetal source
         directory non recursively.
 
         Place outputs on the same subpath or the output directory.
-        """
+        '''
         in_dir = os.path.join(self.env['baremetal_src_dir'], subpath)
         out_dir = os.path.join(self.env['baremetal_build_dir'], subpath)
         os.makedirs(out_dir, exist_ok=True)

cli_function.py

@@ -4,7 +4,7 @@ import argparse
 import imp
 import os
 
-class Argument:
+class _Argument:
     def __init__(
             self,
             long_or_short_1,
@@ -14,28 +14,22 @@ class Argument:
             nargs=None,
             **kwargs
         ):
-            if long_or_short_2 is None:
-                shortname = None
-                longname = long_or_short_1
-            else:
-                shortname = long_or_short_1
-                longname = long_or_short_2
             self.args = []
             # argparse is crappy and cannot tell us if arguments were given or not.
             # We need that information to decide if the config file should override argparse or not.
             # So we just use None as a sentinel.
             self.kwargs = {'default': None}
+            shortname, longname, key, is_option = self.get_key(
+                long_or_short_1,
+                long_or_short_2
+            )
             if shortname is not None:
                 self.args.append(shortname)
-            if longname[0] == '-':
+            if is_option:
                 self.args.append(longname)
-                self.key = longname.lstrip('-').replace('-', '_')
-                self.is_option = True
             else:
-                self.key = longname.replace('-', '_')
-                self.args.append(self.key)
+                self.args.append(key)
                 self.kwargs['metavar'] = longname
-                self.is_option = False
                 if default is not None and nargs is None:
                     self.kwargs['nargs'] = '?'
             if nargs is not None:
@@ -53,22 +47,51 @@ class Argument:
             if self.is_bool and not 'action' in kwargs:
                 self.kwargs['action'] = bool_action
             if help is not None:
-                if not self.is_bool and default:
-                    help += ' Default: {}'.format(default)
+                if default is not None:
+                    if help[-1] == '\n':
+                        if '\n\n' in help[:-1]:
+                            help += '\n'
+                    elif help[-1] == ' ':
+                        pass
+                    else:
+                        help += ' '
+                    help += 'Default: {}'.format(default)
                 self.kwargs['help'] = help
             self.optional = (
                 default is not None or
                 self.is_bool or
-                self.is_option or
+                is_option or
                 nargs in ('?', '*', '+')
             )
             self.kwargs.update(kwargs)
             self.default = default
             self.longname = longname
+            self.key = key
+            self.is_option = is_option
 
     def __str__(self):
         return str(self.args) + ' ' + str(self.kwargs)
 
+    @staticmethod
+    def get_key(
+        long_or_short_1,
+        long_or_short_2=None,
+        **kwargs
+    ):
+        if long_or_short_2 is None:
+            shortname = None
+            longname = long_or_short_1
+        else:
+            shortname = long_or_short_1
+            longname = long_or_short_2
+        if longname[0] == '-':
+            key = longname.lstrip('-').replace('-', '_')
+            is_option = True
+        else:
+            key = longname.replace('-', '_')
+            is_option = False
+        return shortname, longname, key, is_option
+
 class CliFunction:
     '''
     Represent a function that can be called either from Python code, or
@@ -141,7 +164,7 @@ class CliFunction:
             *args,
             **kwargs
         ):
-            argument = Argument(*args, **kwargs)
+            argument = _Argument(*args, **kwargs)
             self._arguments.append(argument)
             self._all_keys.add(argument.key)
 
@@ -170,6 +193,10 @@ class CliFunction:
         args = parser.parse_args(args=cli_args)
         return self(**vars(args))
 
+    @staticmethod
+    def get_key(*args, **kwargs):
+        return _Argument.get_key(*args, **kwargs)
+
     def main(self, **kwargs):
         '''
         Do the main function call work.
@@ -217,7 +244,7 @@ amazing function!
         'args_star': []
     }
 
-    # Default CLI call.
+    # Default CLI call with programmatic CLI arguments.
     out = one_cli_function.cli(['1'])
     assert out == default
 
@@ -258,5 +285,5 @@ amazing function!
     # Force a boolean value set on the config to be False on CLI.
     assert one_cli_function.cli(['--no-bool-cli', '1'])['bool_cli'] is False
 
-    # CLI call.
+    # CLI call with argv command line arguments.
     print(one_cli_function.cli())

common.py

@@ -92,6 +92,7 @@ consts['obj_ext'] = '.o'
 consts['config_file'] = os.path.join(consts['data_dir'], 'config.py')
 consts['magic_fail_string'] = b'lkmc_test_fail'
 consts['baremetal_lib_basename'] = 'lib'
+consts['emulators'] = ['qemu', 'gem5']
 
 class LkmcCliFunction(cli_function.CliFunction):
     '''
@@ -100,9 +101,15 @@ class LkmcCliFunction(cli_function.CliFunction):
     * command timing
     * some common flags, e.g.: --arch, --dry-run, --verbose
     '''
-    def __init__(self, *args, do_print_time=True, **kwargs):
+    def __init__(self, *args, defaults=None, **kwargs):
+        '''
+        :ptype defaults: Dict[str,Any]
+        :param defaults: override the default value of an argument
+        '''
         kwargs['config_file'] = consts['config_file']
-        self._do_print_time = do_print_time
+        if defaults is None:
+            defaults = {}
+        self._defaults = defaults
         super().__init__(*args, **kwargs)
 
         # Args for all scripts.
@@ -122,6 +129,10 @@ Bash equivalents even for actions taken directly in Python without shelling out.
 mkdir are generally omitted since those are obvious
 '''
         )
+        self.add_argument(
+            '--print-time', default=True,
+            help='Print how long it took to run the command at the end.'
+        )
         self.add_argument(
             '-v', '--verbose', default=False,
             help='Show full compilation commands when they are not shown by default.'
@@ -266,27 +277,29 @@ instances in parallel. Default: the run ID (-n) if that is an integer, otherwise
 
         # Misc.
         self.add_argument(
-            '-g', '--gem5', default=False,
-            help='Use gem5 instead of QEMU.'
+            '--emulator', choices=consts['emulators'],
+            help='''\
+Set the emulator to use. Ignore --gem5.
+'''
         )
         self.add_argument(
-            '--qemu', default=False,
+            '-g', '--gem5', default=False,
             help='''\
-Use QEMU as the emulator. This option exists in addition to --gem5
-to allow overriding configs from the CLI.
+Use gem5 instead of QEMU. Shortcut for `--emulator gem5`.
 '''
         )
 
     def _init_env(self, env):
         '''
-        Update the kwargs from the command line with derived arguments.
+        Update the kwargs from the command line with values derived from them.
         '''
         def join(*paths):
             return os.path.join(*paths)
-        if env['qemu'] or not env['gem5']:
-            env['emulator'] = 'qemu'
-        else:
-            env['emulator'] = 'gem5'
+        if env['emulator'] is None:
+            if env['gem5']:
+                env['emulator'] = 'gem5'
+            else:
+                env['emulator'] = 'qemu'
         if env['arch'] in env['arch_short_to_long_dict']:
             env['arch'] = env['arch_short_to_long_dict'][env['arch']]
         if env['userland_build_id'] is None:
@@ -481,6 +494,8 @@ to allow overriding configs from the CLI.
 
         # Baremetal.
         env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal')
+        env['baremetal_src_arch_subpath'] = join('arch', env['arch'])
+        env['baremetal_src_arch_dir'] = join(env['baremetal_src_dir'], env['baremetal_src_arch_subpath'])
         env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename'])
         if env['emulator'] == 'gem5':
             env['simulator_name'] = 'gem5'
@@ -534,6 +549,12 @@ to allow overriding configs from the CLI.
             env['image'] = path
         self.env = env
 
+    def add_argument(self, *args, **kwargs):
+        shortname, longname, key, is_option = self.get_key(*args, **kwargs)
+        if key in self._defaults:
+            kwargs['default'] = self._defaults[key]
+        super().add_argument(*args, **kwargs)
+
     def assert_crosstool_ng_supports_arch(self, arch):
         if arch not in self.env['crosstool_ng_supported_archs']:
             raise Exception('arch not yet supported: ' + arch)
@@ -644,6 +665,14 @@ to allow overriding configs from the CLI.
             _json = {}
         return _json
 
+    @staticmethod
+    def import_path(path):
+        '''
+        https://stackoverflow.com/questions/2601047/import-a-python-module-without-the-py-extension
+        https://stackoverflow.com/questions/31773310/what-does-the-first-argument-of-the-imp-load-source-method-do
+        '''
+        return imp.load_source(os.path.split(path)[1].replace('-', '_'), path)
+
     @staticmethod
     def log_error(msg):
         print('error: {}'.format(msg), file=sys.stderr)
@@ -687,7 +716,7 @@ to allow overriding configs from the CLI.
         return False
 
     def _print_time(self, ellapsed_seconds):
-        if self._do_print_time:
+        if self.env['print_time']:
             hours, rem = divmod(ellapsed_seconds, 3600)
             minutes, seconds = divmod(rem, 60)
             print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))

getvar

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 
-import types
-
 import common
-from shell_helpers import LF
 
 class Main(common.LkmcCliFunction):
     def __init__(self):
         super().__init__(
+            defaults={
+                'print_time': False,
+            },
             description='''\
 Print the value of a self.env['py'] variable.
 
@@ -28,7 +28,6 @@ List all available variables:
 ./%(prog)s
 ....
 ''',
-            do_print_time=False,
         )
         self.add_argument('variable', nargs='?')
 

run

@@ -40,6 +40,10 @@ https://superuser.com/questions/1373226/how-to-redirect-qemu-serial-output-to-bo
             '--debug-vm-args', default='',
             help='Pass arguments to GDB.'
         )
+        self.add_argument(
+            '--dp650', default=False,
+            help='Use the ARM DP650 display processor instead of the default HDLCD on gem5.'
+        )
         self.add_argument(
             '--dtb',
             help='''\
@@ -372,15 +376,15 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
                         dtb = None
                         if self.env['dtb'] is not None:
                             dtb = self.env['dtb']
-                        elif args.dp650:
-                            dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, args.cpus)), common.Newline,
+                        elif self.env['dp650']:
+                            dtb = os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv{}_gem5_v1_{}{}cpu.dtb'.format(common.armv, dp650_cmd, self.env['cpus'])), LF,
                         if dtb is None:
-                            cmd.extend(['--generate-dtb', common.Newline])
+                            if not self.env['baremetal']:
+                                cmd.extend(['--generate-dtb', LF])
                         else:
-                            cmd.extend(['--dtb-filename', dtb, common.Newline])
+                            cmd.extend(['--dtb-filename', dtb, LF])
                         if self.env['baremetal'] is None:
-                            cmd.extend([
-                                '--param', 'system.panic_on_panic = True', LF])
+                            cmd.extend(['--param', 'system.panic_on_panic = True', LF ])
                         else:
                             cmd.extend([
                                 '--bare-metal', LF,
@@ -488,7 +492,7 @@ Run QEMU with VNC instead of the default SDL. Connect to it with:
                     vnc
                 )
                 if self.env['dtb'] is not None:
-                    cmd.extend(['-dtb', self.env['dtb'], common.Newline])
+                    cmd.extend(['-dtb', self.env['dtb'], LF])
                 if not qemu_executable_prebuilt:
                     cmd.extend(qemu_user_and_system_options)
                 if self.env['initrd']:

run-gdb

@@ -23,8 +23,6 @@ class GdbTestcase:
         '''
         self.prompt = '\(gdb\) '
         self.source_path = source_path
-        self.print_cmd(cmd)
-        cmd = self.strip_newlines(cmd)
         import pexpect
         self.child = pexpect.spawn(
             cmd[0],
@@ -198,10 +196,11 @@ See: https://github.com/cirosantilli/linux-kernel-module-cheat#gdb-builtin-cpu-s
                 cmd.extend(['-ex', 'lx-symbols {}'.format(self.env['kernel_modules_build_subdir']), LF])
         cmd.extend(after)
         if self.env['test']:
+            self.sh.print_cmd(cmd)
             GdbTestcase(
                 self.env['source_path'],
                 test_script_path,
-                cmd,
+                self.sh.strip_newlines(cmd),
                 verbose=self.env['verbose'],
             )
         else:

run-gdb-user

@@ -5,7 +5,7 @@ import os
 import sys
 
 import common
-rungdb = imp.load_source('rungdb', os.path.join(kwargs['root_dir'], 'run-gdb'))
+rungdb = imp.load_source('run_gdb', os.path.join(kwargs['root_dir'], 'run-gdb'))
 
 parser = self.get_argparse(argparse_args={
     'description': '''GDB step debug guest userland processes without gdbserver.

shell_helpers.py

@@ -9,6 +9,7 @@ import signal
 import stat
 import subprocess
 import sys
+import threading
 
 class LF:
     '''
@@ -178,18 +179,21 @@ class ShellHelpers:
         if show_cmd:
             self.print_cmd(cmd, cwd=cwd, cmd_file=cmd_file, extra_env=extra_env, extra_paths=extra_paths)
 
-        # Otherwise Ctrl + C gives:
-        # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
-        # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
-        sigint_old = signal.getsignal(signal.SIGINT)
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        # Otherwise, if called from a non-main thread:
+        # ValueError: signal only works in main thread
+        if threading.current_thread() == threading.main_thread():
+            # Otherwise Ctrl + C gives:
+            # - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
+            # - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
+            sigint_old = signal.getsignal(signal.SIGINT)
+            signal.signal(signal.SIGINT, signal.SIG_IGN)
 
-        # Otherwise BrokenPipeError when piping through | grep
-        # But if I do this_module, my terminal gets broken at the end. Why, why, why.
-        # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
-        # Ignoring the exception is not enough as it prints a warning anyways.
-        #sigpipe_old = signal.getsignal(signal.SIGPIPE)
-        #signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+            # Otherwise BrokenPipeError when piping through | grep
+            # But if I do this_module, my terminal gets broken at the end. Why, why, why.
+            # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python
+            # Ignoring the exception is not enough as it prints a warning anyways.
+            #sigpipe_old = signal.getsignal(signal.SIGPIPE)
+            #signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
         cmd = self.strip_newlines(cmd)
         if not self.dry_run:
@@ -211,8 +215,9 @@ class ShellHelpers:
                                 logfile.write(byte)
                             else:
                                 break
-            signal.signal(signal.SIGINT, sigint_old)
-            #signal.signal(signal.SIGPIPE, sigpipe_old)
+            if threading.current_thread() == threading.main_thread():
+                signal.signal(signal.SIGINT, sigint_old)
+                #signal.signal(signal.SIGPIPE, sigpipe_old)
             returncode = proc.returncode
             if returncode != 0 and raise_on_failure:
                 raise Exception('Command exited with status: {}'.format(returncode))

test-gdb

@@ -1,35 +1,55 @@
-#!/usr/bin/env bash
-set -eux
-for emulator in --qemu --gem5; do
-  # Userland.
-  # TODO make work.
-  #./run --arch x86_64 --background --userland add "$emulator" --wait-gdb &
-  #./run-gdb --arch x86_64 --userland add "$emulator" --test "$@"
-  #wait
+#!/usr/bin/env python3
 
-  # Baremetal.
-  ./run --arch arm --background --baremetal add "$emulator" --wait-gdb &
-  ./run-gdb --arch arm --baremetal add "$emulator" --test "$@"
-  wait
-  ./run --arch arm --background --baremetal arch/arm/add "$emulator" --wait-gdb &
-  ./run-gdb --arch arm --baremetal arch/arm/add "$emulator" --test "$@"
-  wait
-  ./run --arch arm --background --baremetal arch/arm/regs "$emulator" --wait-gdb &
-  ./run-gdb --arch arm --baremetal arch/arm/regs "$emulator" --test "$@"
-  wait
-  ./run --arch aarch64 --background --baremetal add "$emulator" --wait-gdb &
-  ./run-gdb --arch aarch64 --baremetal add "$emulator" --test "$@"
-  wait
-  ./run --arch aarch64 --background --baremetal arch/aarch64/add "$emulator" --wait-gdb &
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/add "$emulator" --test "$@"
-  wait
-  ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb &
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@"
-  wait
-  ./run --arch aarch64 --background --baremetal arch/aarch64/fadd "$emulator" --wait-gdb &
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/fadd "$emulator" --test "$@"
-  wait
-  ./run --arch aarch64 --background --baremetal arch/aarch64/regs "$emulator" --wait-gdb &
-  ./run-gdb --arch aarch64 --baremetal arch/aarch64/regs "$emulator" --test "$@"
-  wait
-done
+import functools
+import threading
+import os
+
+import common
+
+def output_reader(proc, file):
+    while True:
+        byte = proc.stdout.read(1)
+        if byte:
+            sys.stdout.buffer.write(byte)
+            sys.stdout.flush()
+            file.buffer.write(byte)
+        else:
+            break
+
+class Main(common.LkmcCliFunction):
+    def timed_main(self):
+        run = self.import_path('run').Main()
+        run_gdb = self.import_path('run-gdb').Main()
+        for emulator in self.env['emulators']:
+            for arch in self.env['crosstool_ng_supported_archs']:
+                test_scripts_noext = []
+                for f in os.listdir(self.env['baremetal_src_dir']):
+                    base, ext = os.path.splitext(f)
+                    if ext == '.py':
+                        test_scripts_noext.append(base)
+                for root, dirs, files in os.walk(os.path.join(self.env['baremetal_src_dir'], 'arch', arch)):
+                    for f in files:
+                        base, ext = os.path.splitext(f)
+                        if ext == '.py':
+                            full_path = os.path.join(root, base)
+                            relpath = os.path.relpath(full_path, self.env['baremetal_src_dir'])
+                            test_scripts_noext.append(relpath)
+                for test_script_noext in test_scripts_noext:
+                    run_thread = threading.Thread(target=lambda: run(
+                        arch=arch,
+                        background=True,
+                        baremetal=test_script_noext,
+                        print_time=False,
+                        emulator=emulator,
+                        wait_gdb=True
+                    ))
+                    gdb_thread = threading.Thread(target=lambda: run_gdb(
+                        arch=arch, baremetal=test_script_noext, print_time=False, emulator=emulator, test=True
+                    ))
+                    run_thread.start()
+                    gdb_thread.start()
+                    run_thread.join()
+                    gdb_thread.join()
+
+if __name__ == '__main__':
+    Main().cli()