From 2a77df690cdd8df3896665f59d112e9d60bbd8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Thu, 18 Oct 2018 00:00:00 +0000 Subject: [PATCH] common: create a Component class to factor out builds Not yet finished factoring, but half way there, do for all build- --- README.adoc | 5 +- build | 4 +- build-crosstool-ng | 3 +- build-gem5 | 179 +++++++++--------- build-linux | 162 +++++++++-------- build-m5 | 54 +++--- build-modules | 205 +++++++++++---------- build-qemu | 88 ++++----- build-userland | 105 +++++------ common.py | 440 ++++++++++++++++++++++++--------------------- 10 files changed, 650 insertions(+), 595 deletions(-) diff --git a/README.adoc b/README.adoc index cb546cca..c7398cc6 100644 --- a/README.adoc +++ b/README.adoc @@ -144,7 +144,7 @@ It is super easy to build for different CPU architectures, just use the `--arch` ./run --arch aarch64 .... -But to avoid typing `--arch aarch64` so many times, set the default arch as explained at: <> +To avoid typing `--arch aarch64` so many times, set the default arch as explained at: <> See also: <>. @@ -395,11 +395,12 @@ If you haven't built Buildroot yet for <>, you can build f .... ./download-dependencies --gem5 -./build --gem5 --no-qemu +./build --gem5 --m5 --no-qemu ./run --gem5 .... `--no-qemu` is optional, but it makes the build slightly faster TODO: after first build: + .... ./download-dependencies --gem5 ./build-gem5 diff --git a/build b/build index 7cd69d91..440de427 100755 --- a/build +++ b/build @@ -36,7 +36,7 @@ name_to_component_map = { 'baremetal': BaremetalComponent(False), 'buildroot': Component( True, - lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch, '--gem5'], dry_run=dry_run), + lambda arch, dry_run: run_cmd(['build-buildroot', '--arch', arch], dry_run=dry_run), ), 'gem5': Component( False, @@ -51,7 +51,7 @@ name_to_component_map = { lambda arch, dry_run: run_cmd(['build-modules', '--arch', arch], dry_run=dry_run) ), 'm5': Component( - True, + False, lambda arch, dry_run: run_cmd(['build-m5', '--arch', arch], dry_run=dry_run) ), 'qemu': Component( diff --git a/build-crosstool-ng b/build-crosstool-ng index 5a061c24..ffa37f2b 100755 --- a/build-crosstool-ng +++ b/build-crosstool-ng @@ -2,7 +2,6 @@ import multiprocessing import os -import shutil import sys import time @@ -43,7 +42,7 @@ def main(args, extra_args=None): ) == 0 # Build the toolchain. - shutil.copy2( + common.cp( os.path.join(common.root_dir, 'crosstool_ng_config', args.arch), defconfig_dest ) diff --git a/build-gem5 b/build-gem5 index 7d577051..2a6935ce 100755 --- a/build-gem5 +++ b/build-gem5 @@ -4,100 +4,99 @@ import glob import multiprocessing import os import pathlib -import shutil import subprocess -import time import common -parser = common.get_argparse() -common.add_build_arguments(parser) -parser.add_argument( - 'extra_scons_args', - default=[], - metavar='extra-scons-args', - nargs='*' -) -args = common.setup(parser) -binaries_dir = os.path.join(common.gem5_system_dir, 'binaries') -disks_dir = os.path.join(common.gem5_system_dir, 'disks') -if args.clean: - common.rmrf(common.gem5_build_dir) -else: - start_time = time.time() - os.makedirs(binaries_dir, exist_ok=True) - os.makedirs(disks_dir, exist_ok=True) - if args.gem5_source_dir is None: - if not os.path.exists(os.path.join(common.gem5_src_dir, '.git')): - if common.gem5_src_dir == common.gem5_default_src_dir: - raise Exception('gem5 submodule not checked out') - assert common.run_cmd([ - 'git', - '-C', common.gem5_default_src_dir, - 'worktree', 'add', - '-b', os.path.join('wt', args.gem5_build_id), - common.gem5_src_dir - ]) == 0 - if args.verbose: - verbose = ['--verbose'] - else: - verbose = [] - if args.arch == 'x86_64': - dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img') - with open(dummy_img_path, 'wb') as dummy_img_file: - zeroes = b'\x00' * (2 ** 16) - for i in range(2 ** 10): - dummy_img_file.write(zeroes) - assert common.run_cmd(['mkswap', dummy_img_path]) == 0 - with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): - # This file must always be present, despite --kernel overriding that default and selecting the kernel. - # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. - pass - elif args.arch == 'arm' or args.arch == 'aarch64': - gem5_system_src_dir = os.path.join(common.gem5_src_dir, 'system') +class Gem5Component(common.Component): + def add_parser_arguments(self, parser): + parser.add_argument( + 'extra_scons_args', + default=[], + metavar='extra-scons-args', + nargs='*' + ) - # dtb - dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') - dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt') - assert common.run_cmd(['make', '-C', dt_src_dir]) == 0 - os.makedirs(dt_build_dir, exist_ok=True) - for dt in glob.glob(os.path.join(dt_src_dir, '*.dtb')): - shutil.copy2(dt, dt_build_dir) + def do_build(self, args): + build_dir = self.get_build_dir(args) + binaries_dir = os.path.join(common.gem5_system_dir, 'binaries') + disks_dir = os.path.join(common.gem5_system_dir, 'disks') + os.makedirs(binaries_dir, exist_ok=True) + os.makedirs(disks_dir, exist_ok=True) + if args.gem5_source_dir is None: + if not os.path.exists(os.path.join(common.gem5_src_dir, '.git')): + if common.gem5_src_dir == common.gem5_default_src_dir: + raise Exception('gem5 submodule not checked out') + assert common.run_cmd([ + 'git', + '-C', common.gem5_default_src_dir, + 'worktree', 'add', + '-b', os.path.join('wt', args.gem5_build_id), + common.gem5_src_dir + ]) == 0 + if args.verbose: + verbose = ['--verbose'] + else: + verbose = [] + if args.arch == 'x86_64': + dummy_img_path = os.path.join(disks_dir, 'linux-bigswap2.img') + with open(dummy_img_path, 'wb') as dummy_img_file: + zeroes = b'\x00' * (2 ** 16) + for i in range(2 ** 10): + dummy_img_file.write(zeroes) + assert common.run_cmd(['mkswap', dummy_img_path]) == 0 + with open(os.path.join(binaries_dir, 'x86_64-vmlinux-2.6.22.9'), 'w'): + # This file must always be present, despite --kernel overriding that default and selecting the kernel. + # I'm not even joking. No one has ever built x86 gem5 without the magic dist dir present. + pass + elif args.arch == 'arm' or args.arch == 'aarch64': + gem5_system_src_dir = os.path.join(common.gem5_src_dir, 'system') - # Bootloader 32. - bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader') - # TODO use the buildroot cross compiler here, and remove the dependencies from configure. - assert common.run_cmd(['make', '-C', bootloader32_dir]) == 0 - # bootloader - shutil.copy2(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) + # dtb + dt_src_dir = os.path.join(gem5_system_src_dir, 'arm', 'dt') + dt_build_dir = os.path.join(common.gem5_system_dir, 'arm', 'dt') + assert common.run_cmd(['make', '-C', dt_src_dir]) == 0 + os.makedirs(dt_build_dir, exist_ok=True) + for dt in glob.glob(os.path.join(dt_src_dir, '*.dtb')): + common.cp(dt, dt_build_dir) - # Bootloader 64. - bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') - # TODO cross_compile is ignored because the make does not use CC... - assert common.run_cmd(['make', '-C', bootloader64_dir]) == 0 - shutil.copy2(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) - assert common.run_cmd( - ( - [ - 'scons', - '-j', str(multiprocessing.cpu_count()), - '--ignore-style', - common.gem5_executable - ] + - verbose + - args.extra_scons_args - ), - cwd=common.gem5_src_dir, - extra_paths=[common.ccache_dir], - ) == 0 - term_src_dir = os.path.join(common.gem5_src_dir, 'util/term') - m5term_build = os.path.join(term_src_dir, 'm5term') - assert common.run_cmd(['make', '-C', term_src_dir]) == 0 - if os.path.exists(common.gem5_m5term): - # Otherwise shutil.copy2 would fail with "Text file busy" if you - # tried to rebuild while running m5term: - # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512 - os.unlink(common.gem5_m5term) - shutil.copy2(m5term_build, common.gem5_m5term) - end_time = time.time() - common.print_time(end_time - start_time) + # Bootloader 32. + bootloader32_dir = os.path.join(gem5_system_src_dir, 'arm', 'simple_bootloader') + # TODO use the buildroot cross compiler here, and remove the dependencies from configure. + assert common.run_cmd(['make', '-C', bootloader32_dir]) == 0 + # bootloader + common.cp(os.path.join(bootloader32_dir, 'boot_emm.arm'), binaries_dir) + + # Bootloader 64. + bootloader64_dir = os.path.join(gem5_system_src_dir, 'arm', 'aarch64_bootloader') + # TODO cross_compile is ignored because the make does not use CC... + assert common.run_cmd(['make', '-C', bootloader64_dir]) == 0 + common.cp(os.path.join(bootloader64_dir, 'boot_emm.arm64'), binaries_dir) + assert common.run_cmd( + ( + [ + 'scons', + '-j', str(multiprocessing.cpu_count()), + '--ignore-style', + common.gem5_executable + ] + + verbose + + args.extra_scons_args + ), + cwd=common.gem5_src_dir, + extra_paths=[common.ccache_dir], + ) == 0 + term_src_dir = os.path.join(common.gem5_src_dir, 'util/term') + m5term_build = os.path.join(term_src_dir, 'm5term') + assert common.run_cmd(['make', '-C', term_src_dir]) == 0 + if os.path.exists(common.gem5_m5term): + # Otherwise common.cp would fail with "Text file busy" if you + # tried to rebuild while running m5term: + # https://stackoverflow.com/questions/16764946/what-generates-the-text-file-busy-message-in-unix/52427512#52427512 + os.unlink(common.gem5_m5term) + common.cp(m5term_build, common.gem5_m5term) + + def get_build_dir(self, args): + return common.gem5_build_dir + +Gem5Component().build() diff --git a/build-linux b/build-linux index b356f76d..a81f5457 100755 --- a/build-linux +++ b/build-linux @@ -3,86 +3,90 @@ import multiprocessing import os import shutil -import subprocess -import time import common -parser = common.get_argparse() -common.add_build_arguments(parser) -parser.add_argument( - 'extra_make_args', - default=[], - metavar='extra-make-args', - nargs='*' -) -args = common.setup(parser) -if args.clean: - common.rmrf(common.linux_build_dir) -else: - if not common.dry_run: - start_time = time.time() - os.makedirs(common.linux_build_dir, exist_ok=True) - shutil.copy2( - os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch)), - os.path.join(common.linux_build_dir, '.config'), - ) - tool = 'gcc' - gcc = common.get_toolchain_tool(tool) - prefix = gcc[:-len(tool)] - common_args = { - 'cwd': common.linux_src_dir, - } - ccache = shutil.which('ccache') - if ccache is not None: - cc = '{} {}'.format(ccache, gcc) - else: - cc = gcc - common_make_args = [ - 'ARCH={}'.format(common.linux_arch), - 'CROSS_COMPILE={}'.format(prefix), - 'CC={}'.format(cc), - 'O={}'.format(common.linux_build_dir), - ] - if args.verbose: - verbose = ['V=1'] - else: - verbose = [] - assert common.run_cmd( - [ - os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), - '-m', - '-O', common.linux_build_dir, - os.path.join(common.linux_build_dir, '.config'), - os.path.join(common.linux_config_dir, 'min'), - os.path.join(common.linux_config_dir, 'default'), - ], - ) == 0 - assert common.run_cmd( - ( +class LinuxComponent(common.Component): + def add_parser_arguments(self, parser): + parser.add_argument( + 'extra_make_args', + default=[], + metavar='extra-make-args', + nargs='*' + ) + + def do_build(self, args): + build_dir = self.get_build_dir(args) + os.makedirs(build_dir, exist_ok=True) + common.cp( + os.path.join(common.linux_config_dir, 'buildroot-{}'.format(args.arch)), + os.path.join(build_dir, '.config'), + ) + tool = 'gcc' + gcc = common.get_toolchain_tool(tool) + prefix = gcc[:-len(tool)] + common_args = { + 'cwd': common.linux_src_dir, + } + ccache = shutil.which('ccache') + if ccache is not None: + cc = '{} {}'.format(ccache, gcc) + else: + cc = gcc + common_make_args = [ + 'ARCH={}'.format(common.linux_arch), + 'CROSS_COMPILE={}'.format(prefix), + 'CC={}'.format(cc), + 'O={}'.format(build_dir), + ] + if args.verbose: + verbose = ['V=1'] + else: + verbose = [] + assert common.run_cmd( [ - 'make', - '-j', str(multiprocessing.cpu_count()), - ] + - common_make_args + - [ - 'olddefconfig', - ] - ), - **common_args, - ) == 0 - assert common.run_cmd( - ( - [ - 'make', - '-j', str(multiprocessing.cpu_count()), - ] + - common_make_args + - verbose + - args.extra_make_args - ), - **common_args, - ) == 0 - if not common.dry_run: - end_time = time.time() - common.print_time(end_time - start_time) + os.path.join(common.linux_src_dir, 'scripts', 'kconfig', 'merge_config.sh'), + '-m', + '-O', build_dir, + os.path.join(build_dir, '.config'), + os.path.join(common.linux_config_dir, 'min'), + os.path.join(common.linux_config_dir, 'default'), + ], + ) == 0 + assert common.run_cmd( + ( + [ + 'make', + '-j', str(multiprocessing.cpu_count()), + ] + + common_make_args + + [ + 'olddefconfig', + ] + ), + **common_args, + ) == 0 + assert common.run_cmd( + ( + [ + 'make', + '-j', str(multiprocessing.cpu_count()), + ] + + common_make_args + + verbose + + args.extra_make_args + ), + **common_args, + ) == 0 + + def get_argparse_args(self): + return { + 'description': '''\ +Build the Linux kernel. +''' + } + + def get_build_dir(self, args): + return common.linux_build_dir + +LinuxComponent().build() diff --git a/build-m5 b/build-m5 index fa25af10..6445bedd 100755 --- a/build-m5 +++ b/build-m5 @@ -3,29 +3,21 @@ import multiprocessing import os import platform -import shutil import subprocess import time import common -parser = common.get_argparse(argparse_args={ - 'description': 'Build the m5 executable', -}) -common.add_build_arguments(parser) -args = common.setup(parser) -start_time = time.time() -os.makedirs(common.gem5_m5_build_dir, exist_ok=True) -allowed_toolchains = ['buildroot'] -cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) -ld = common.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains) -if args.arch == 'x86_64': - arch = 'x86' -else: - arch = args.arch -assert common.run_cmd( - ( - [ +class M5Component(common.Component): + def get_make_cmd(self, args): + allowed_toolchains = ['buildroot'] + cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) + ld = common.get_toolchain_tool('ld', allowed_toolchains=allowed_toolchains) + if args.arch == 'x86_64': + arch = 'x86' + else: + arch = args.arch + return [ 'make', '-j', str(multiprocessing.cpu_count()), '-f', 'Makefile.{}'.format(arch), @@ -33,11 +25,21 @@ assert common.run_cmd( 'LD={}'.format(ld), 'PWD={}'.format(common.gem5_m5_src_dir), ] - ), - cwd=common.gem5_m5_src_dir, -) == 0 -print(common.out_rootfs_overlay_bin_dir) -os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) -shutil.copy2(os.path.join(common.gem5_m5_src_dir, 'm5'), common.out_rootfs_overlay_bin_dir) -end_time = time.time() -common.print_time(end_time - start_time) + + def do_build(self, args): + os.makedirs(common.gem5_m5_build_dir, exist_ok=True) + assert common.run_cmd( + self.get_make_cmd(args), + cwd=common.gem5_m5_src_dir, + ) == 0 + print(common.out_rootfs_overlay_bin_dir) + os.makedirs(common.out_rootfs_overlay_bin_dir, exist_ok=True) + common.cp(os.path.join(common.gem5_m5_src_dir, 'm5'), common.out_rootfs_overlay_bin_dir) + + def clean(self, args): + assert common.run_cmd( + self.get_make_cmd(args) + ['clean'], + cwd=common.gem5_m5_src_dir, + ) == 0 + +M5Component().build() diff --git a/build-modules b/build-modules index 291265ad..8ff3cdfa 100755 --- a/build-modules +++ b/build-modules @@ -5,108 +5,115 @@ import multiprocessing import os import platform import shutil -import subprocess -import time import common -parser = common.get_argparse(argparse_args={ - 'description': '''\ +class ModulesComponent(common.Component): + def add_parser_arguments(self, parser): + parser.add_argument( + 'extra_make_args', + default=[], + metavar='extra-make-args', + nargs='*' + ) + parser.add_argument( + '--host', + action='store_true', + default=False, + help='Build the Linux kernel modules for the host instead of guest', + ) + parser.add_argument( + 'kernel_modules', + default=[], + help='Which kernel modules to build. Default: build all', + metavar='kernel-modules', + nargs='*', + ) + + def do_build(self, args): + build_dir = self.get_build_dir(args) + os.makedirs(build_dir, exist_ok=True) + # I kid you not, out-of-tree build is not possible, O= does not work as for the kernel build: + # + # * https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory + # * https://stackoverflow.com/questions/12244979/build-kernel-module-into-a-specific-directory + # * https://stackoverflow.com/questions/18386182/out-of-tree-kernel-modules-multiple-module-single-makefile-same-source-file + # + # This copies only modified files as per: + # https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory + distutils.dir_util.copy_tree( + common.lkmc_package_src_dir, + os.path.join(build_dir), + update=1, + ) + all_kernel_modules = [] + for basename in os.listdir(common.kernel_modules_src_dir): + src = os.path.join(common.kernel_modules_src_dir, basename) + if os.path.isfile(src): + noext, ext = os.path.splitext(basename) + if ext == common.c_ext: + all_kernel_modules.append(noext) + if args.kernel_modules == []: + kernel_modules = all_kernel_modules + else: + kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], args.kernel_modules) + object_files = map(lambda x: x + common.obj_ext, kernel_modules) + tool = 'gcc' + if args.host: + allowed_toolchains = ['host'] + else: + allowed_toolchains = None + gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains) + prefix = gcc[:-len(tool)] + ccache = shutil.which('ccache') + if ccache is not None: + cc = '{} {}'.format(ccache, gcc) + else: + cc = gcc + if args.verbose: + verbose = ['V=1'] + else: + verbose = [] + if args.host: + linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') + else: + linux_dir = common.linux_build_dir + build_subdir = os.path.join(build_dir, common.kernel_modules_subdir) + assert common.run_cmd( + ( + [ + 'make', + '-j', str(multiprocessing.cpu_count()), + 'ARCH={}'.format(common.linux_arch), + 'CC={}'.format(cc), + 'CROSS_COMPILE={}'.format(prefix), + 'LINUX_DIR={}'.format(linux_dir), + 'M={}'.format(build_subdir), + 'OBJECT_FILES={}'.format(' '.join(object_files)), + ] + + verbose + ), + cwd=os.path.join(build_subdir), + ) == 0 + common.copy_dir_if_update_non_recursive( + srcdir=build_subdir, + destdir=common.out_rootfs_overlay_dir, + filter_ext=common.kernel_module_ext, + ) + + def get_argparse_args(self): + return { + 'description': '''\ Build our Linux kernel modules without using Buildroot. -See also:https://github.com/cirosantilli/linux-kernel-module-cheat#host +See also: https://github.com/cirosantilli/linux-kernel-module-cheat#host ''' -}) -common.add_build_arguments(parser) -parser.add_argument( - '--host', - action='store_true', - default=False, - help='Build the Linux kernel modules for the host instead of guest', -) -parser.add_argument( - 'kernel_modules', - default=[], - help='Which kernel modules to build. Default: build all', - metavar='kernel-modules', - nargs='*', -) -args = common.setup(parser) -if args.host: - build_dir = os.path.join(common.kernel_modules_build_host_dir) -else: - build_dir = os.path.join(common.kernel_modules_build_dir) -if args.clean: - common.rmrf(build_dir) -else: - start_time = time.time() - os.makedirs(build_dir, exist_ok=True) - # I kid you not, out-of-tree build is not possible, O= does not work as for the kernel build: - # - # * https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory - # * https://stackoverflow.com/questions/12244979/build-kernel-module-into-a-specific-directory - # * https://stackoverflow.com/questions/18386182/out-of-tree-kernel-modules-multiple-module-single-makefile-same-source-file - # - # This copies only modified files as per: - # https://stackoverflow.com/questions/5718899/building-an-out-of-tree-linux-kernel-module-in-a-separate-object-directory - distutils.dir_util.copy_tree( - common.lkmc_package_src_dir, - os.path.join(build_dir), - update=1, - ) - all_kernel_modules = [] - for basename in os.listdir(common.kernel_modules_src_dir): - src = os.path.join(common.kernel_modules_src_dir, basename) - if os.path.isfile(src): - noext, ext = os.path.splitext(basename) - if ext == common.c_ext: - all_kernel_modules.append(noext) - if args.kernel_modules == []: - kernel_modules = all_kernel_modules - else: - kernel_modules = map(lambda x: os.path.splitext(os.path.split(x)[1])[0], args.kernel_modules) - object_files = map(lambda x: x + common.obj_ext, kernel_modules) - tool = 'gcc' - if args.host: - allowed_toolchains = ['host'] - else: - allowed_toolchains = None - gcc = common.get_toolchain_tool(tool, allowed_toolchains=allowed_toolchains) - prefix = gcc[:-len(tool)] - ccache = shutil.which('ccache') - if ccache is not None: - cc = '{} {}'.format(ccache, gcc) - else: - cc = gcc - if args.verbose: - verbose = ['V=1'] - else: - verbose = [] - if args.host: - linux_dir = os.path.join('/lib', 'modules', platform.uname().release, 'build') - else: - linux_dir = common.linux_build_dir - build_subdir = os.path.join(build_dir, common.kernel_modules_subdir) - assert common.run_cmd( - ( - [ - 'make', - '-j', str(multiprocessing.cpu_count()), - 'ARCH={}'.format(common.linux_arch), - 'CC={}'.format(cc), - 'CROSS_COMPILE={}'.format(prefix), - 'LINUX_DIR={}'.format(linux_dir), - 'M={}'.format(build_subdir), - 'OBJECT_FILES={}'.format(' '.join(object_files)), - ] + - verbose - ), - cwd=os.path.join(build_subdir), - ) == 0 - common.copy_dir_if_update_non_recursive( - srcdir=build_subdir, - destdir=common.out_rootfs_overlay_dir, - filter_ext=common.kernel_module_ext, - ) - end_time = time.time() - common.print_time(end_time - start_time) + } + + def get_build_dir(self, args): + if args.host: + return os.path.join(common.kernel_modules_build_host_dir) + else: + return os.path.join(common.kernel_modules_build_dir) + +ModulesComponent().build() diff --git a/build-qemu b/build-qemu index 31ee691a..c5767666 100755 --- a/build-qemu +++ b/build-qemu @@ -3,52 +3,52 @@ import multiprocessing import os import subprocess -import time import common -parser = common.get_argparse() -common.add_build_arguments(parser) -parser.add_argument( - 'extra_config_args', - default=[], - metavar='extra-config-args', - nargs='*' -) -args = common.setup(parser) -if args.clean: - common.rmrf(common.qemu_build_dir) -else: - start_time = time.time() - os.makedirs(common.qemu_build_dir, exist_ok=True) - if args.verbose: - verbose = ['V=1'] - else: - verbose = [] - assert common.run_cmd( - [ - os.path.join(common.qemu_src_dir, 'configure'), - '--enable-debug', - '--enable-trace-backends=simple', - '--target-list={}-softmmu'.format(args.arch), - '--enable-sdl', - '--with-sdlabi=2.0', - ] + - args.extra_config_args, - extra_paths=[common.ccache_dir], - cwd=common.qemu_build_dir - ) == 0 - assert common.run_cmd( - ( - [ - 'make', - '-j', str(multiprocessing.cpu_count()), +class QemuComponent(common.Component): + def add_parser_arguments(self, parser): + parser.add_argument( + 'extra_config_args', + default=[], + metavar='extra-config-args', + nargs='*' + ) + def do_build(self, args): + build_dir = self.get_build_dir(args) + os.makedirs(build_dir, exist_ok=True) + if args.verbose: + verbose = ['V=1'] + else: + verbose = [] + assert common.run_cmd( + [ + os.path.join(common.qemu_src_dir, 'configure'), + '--enable-debug', + '--enable-trace-backends=simple', + '--target-list={}-softmmu'.format(args.arch), + '--enable-sdl', + '--with-sdlabi=2.0', ] + - verbose - ), - cwd=common.qemu_build_dir, - extra_paths=[common.ccache_dir], - ) == 0 - end_time = time.time() - common.print_time(end_time - start_time) + args.extra_config_args, + extra_paths=[common.ccache_dir], + cwd=build_dir + ) == 0 + assert common.run_cmd( + ( + [ + 'make', + '-j', str(multiprocessing.cpu_count()), + + ] + + verbose + ), + cwd=build_dir, + extra_paths=[common.ccache_dir], + ) == 0 + + def get_build_dir(self, args): + return None + +QemuComponent().build() diff --git a/build-userland b/build-userland index 797fed7b..4cac4675 100755 --- a/build-userland +++ b/build-userland @@ -5,65 +5,68 @@ import os import platform import shutil import subprocess -import time import common -parser = common.get_argparse(argparse_args={ - 'description': 'Build our compiled userland examples', -}) -common.add_build_arguments(parser) -parser.add_argument( - '--has-package', - action='append', - default=[], - help='''\ +class UserlandComponent(common.Component): + def add_parser_arguments(self, parser): + parser.add_argument( + '--has-package', + action='append', + default=[], + help='''\ Indicate that a given package is present in the root filesystem, which allows us to build examples that rely on it. ''', -) -parser.add_argument( - 'targets', - default=[], - help='''\ + ) + parser.add_argument( + 'targets', + default=[], + help='''\ Build only the given userland programs. Default: build all examples that have their package dependencies met. For example, an OpenBLAS example can only be built if the target root filesystem has the OpenBLAS libraries and headers installed. ''', - metavar='programs', - nargs='*', -) -args = common.setup(parser) -if args.clean: - common.rmrf(common.userland_build_dir) -else: - start_time = time.time() - os.makedirs(common.userland_build_dir, exist_ok=True) - allowed_toolchains = ['buildroot'] - cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) - cxx = common.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains) - assert common.run_cmd( - ( - [ - 'make', - '-j', str(multiprocessing.cpu_count()), - 'CC={}'.format(cc), - 'CXX={}'.format(cxx), - 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), - 'STAGING_DIR={}'.format(common.buildroot_staging_dir), - 'OUT_DIR={}'.format(common.userland_build_dir), - ] + - ['HAS_{}=y'.format(package.upper()) for package in args.has_package] + - [os.path.join(common.userland_build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.executable_ext for target in args.targets] - ), - cwd=common.userland_src_dir, - extra_paths=[common.ccache_dir], - ) == 0 - common.copy_dir_if_update_non_recursive( - srcdir=common.userland_build_dir, - destdir=common.out_rootfs_overlay_dir, - filter_ext=common.executable_ext, - ) - end_time = time.time() - common.print_time(end_time - start_time) + metavar='programs', + nargs='*', + ) + + def do_build(self, args): + build_dir = self.get_build_dir(args) + os.makedirs(build_dir, exist_ok=True) + allowed_toolchains = ['buildroot'] + cc = common.get_toolchain_tool('gcc', allowed_toolchains=allowed_toolchains) + cxx = common.get_toolchain_tool('g++', allowed_toolchains=allowed_toolchains) + assert common.run_cmd( + ( + [ + 'make', + '-j', str(multiprocessing.cpu_count()), + 'CC={}'.format(cc), + 'CXX={}'.format(cxx), + 'PKG_CONFIG={}'.format(common.buildroot_pkg_config), + 'STAGING_DIR={}'.format(common.buildroot_staging_dir), + 'OUT_DIR={}'.format(build_dir), + ] + + ['HAS_{}=y'.format(package.upper()) for package in args.has_package] + + [os.path.join(build_dir, os.path.splitext(os.path.split(target)[1])[0]) + common.executable_ext for target in args.targets] + ), + cwd=common.userland_src_dir, + extra_paths=[common.ccache_dir], + ) == 0 + common.copy_dir_if_update_non_recursive( + srcdir=build_dir, + destdir=common.out_rootfs_overlay_dir, + filter_ext=common.executable_ext, + ) + + def get_argparse_args(self): + return { + 'description': 'Build our compiled userland examples', + } + + def get_build_dir(self, args): + return common.userland_build_dir + +UserlandComponent().build() diff --git a/common.py b/common.py index 8bc7756b..46c48578 100644 --- a/common.py +++ b/common.py @@ -21,7 +21,7 @@ import time import urllib import urllib.request -this = sys.modules[__name__] +this_module = sys.modules[__name__] root_dir = os.path.dirname(os.path.abspath(__file__)) data_dir = os.path.join(root_dir, 'data') p9_dir = os.path.join(data_dir, '9p') @@ -29,17 +29,17 @@ gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5') out_dir = os.path.join(root_dir, 'out') bench_boot = os.path.join(out_dir, 'bench-boot.txt') packages_dir = os.path.join(root_dir, 'packages') -lkmc_package_src_dir = os.path.join(this.packages_dir, 'lkmc') +lkmc_package_src_dir = os.path.join(this_module.packages_dir, 'lkmc') kernel_modules_subdir = 'kernel_modules' -kernel_modules_src_dir = os.path.join(this.lkmc_package_src_dir, this.kernel_modules_subdir) +kernel_modules_src_dir = os.path.join(this_module.lkmc_package_src_dir, this_module.kernel_modules_subdir) userland_subdir = 'userland' -userland_src_dir = os.path.join(this.lkmc_package_src_dir, this.userland_subdir) +userland_src_dir = os.path.join(this_module.lkmc_package_src_dir, this_module.userland_subdir) submodules_dir = os.path.join(root_dir, 'submodules') buildroot_src_dir = os.path.join(submodules_dir, 'buildroot') crosstool_ng_src_dir = os.path.join(submodules_dir, 'crosstool-ng') linux_src_dir = os.path.join(submodules_dir, 'linux') -linux_config_dir = os.path.join(this.root_dir, 'kernel_config') -rootfs_overlay_dir = os.path.join(this.root_dir, 'rootfs_overlay') +linux_config_dir = os.path.join(this_module.root_dir, 'kernel_config') +rootfs_overlay_dir = os.path.join(this_module.root_dir, 'rootfs_overlay') extract_vmlinux = os.path.join(linux_src_dir, 'scripts', 'extract-vmlinux') qemu_src_dir = os.path.join(submodules_dir, 'qemu') parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark') @@ -52,14 +52,14 @@ arch_short_to_long_dict = collections.OrderedDict([ ]) all_archs = [arch_short_to_long_dict[k] for k in arch_short_to_long_dict] arch_choices = [] -for key in this.arch_short_to_long_dict: +for key in this_module.arch_short_to_long_dict: arch_choices.append(key) - arch_choices.append(this.arch_short_to_long_dict[key]) + arch_choices.append(this_module.arch_short_to_long_dict[key]) default_arch = 'x86_64' gem5_cpt_prefix = '^cpt\.' sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip() -release_dir = os.path.join(this.out_dir, 'release') -release_zip_file = os.path.join(this.release_dir, 'lkmc-{}.zip'.format(this.sha)) +release_dir = os.path.join(this_module.out_dir, 'release') +release_zip_file = os.path.join(this_module.release_dir, 'lkmc-{}.zip'.format(this_module.sha)) github_repo_id = 'cirosantilli/linux-kernel-module-cheat' asm_ext = '.S' c_ext = '.c' @@ -73,6 +73,40 @@ if os.path.exists(config_file): config = imp.load_source('config', config_file) configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')} +class Component: + def __init__(self): + pass + + def build(self): + parser = this_module.get_argparse(argparse_args=self.get_argparse_args()) + self.add_parser_arguments(parser) + this_module.add_build_arguments(parser) + args = this_module.setup(parser) + if not this_module.dry_run: + start_time = time.time() + if args.clean: + self.clean(args) + else: + self.do_build(args) + if not this_module.dry_run: + end_time = time.time() + this_module.print_time(end_time - start_time) + + def add_parser_arguments(self, parser): + pass + + def clean(self, args): + this_module.rmrf(self.get_build_dir(args)) + + def do_build(self, args): + raise NotImplementedError() + + def get_argparse_args(self): + return {} + + def get_build_dir(self, args): + raise NotImplementedError() + def add_build_arguments(parser): parser.add_argument( '--clean', @@ -105,10 +139,10 @@ def gem_list_checkpoint_dirs(): ''' List checkpoint directory, oldest first. ''' - global this - prefix_re = re.compile(this.gem5_cpt_prefix) - files = list(filter(lambda x: os.path.isdir(os.path.join(this.m5out_dir, x)) and prefix_re.search(x), os.listdir(this.m5out_dir))) - files.sort(key=lambda x: os.path.getmtime(os.path.join(this.m5out_dir, x))) + global this_module + prefix_re = re.compile(this_module.gem5_cpt_prefix) + files = list(filter(lambda x: os.path.isdir(os.path.join(this_module.m5out_dir, x)) and prefix_re.search(x), os.listdir(this_module.m5out_dir))) + files.sort(key=lambda x: os.path.getmtime(os.path.join(this_module.m5out_dir, x))) return files def get_argparse(default_args=None, argparse_args=None): @@ -118,7 +152,7 @@ def get_argparse(default_args=None, argparse_args=None): :type default_args: Dict[str,str] :type argparse_args: Dict ''' - global this + global this_module if default_args is None: default_args = {} if argparse_args is None: @@ -127,9 +161,9 @@ def get_argparse(default_args=None, argparse_args=None): formatter_class=argparse.RawTextHelpFormatter, **argparse_args ) - this.add_dry_run_argument(parser) + this_module.add_dry_run_argument(parser) parser.add_argument( - '-a', '--arch', choices=this.arch_choices, default=this.default_arch, + '-a', '--arch', choices=this_module.arch_choices, default=this_module.default_arch, help='CPU architecture. Default: %(default)s' ) parser.add_argument( @@ -232,8 +266,8 @@ Default: the run ID (-n) if that is an integer, otherwise 0. '-v', '--verbose', default=False, action='store_true', help='Show full compilation commands when they are not shown by default.' ) - if hasattr(this, 'configs'): - defaults = this.configs.copy() + if hasattr(this_module, 'configs'): + defaults = this_module.configs.copy() else: defaults = {} defaults.update(default_args) @@ -245,9 +279,9 @@ Default: the run ID (-n) if that is an integer, otherwise 0. return parser def get_elf_entry(elf_file_path): - global this + global this_module readelf_header = subprocess.check_output([ - this.get_toolchain_tool('readelf'), + this_module.get_toolchain_tool('readelf'), '-h', elf_file_path ]) @@ -259,11 +293,11 @@ def get_elf_entry(elf_file_path): return int(addr, 0) def get_stats(stat_re=None, stats_file=None): - global this + global this_module if stat_re is None: stat_re = '^system.cpu[0-9]*.numCycles$' if stats_file is None: - stats_file = this.stats_file + stats_file = this_module.stats_file stat_re = re.compile(stat_re) ret = [] with open(stats_file, 'r') as statfile: @@ -275,11 +309,11 @@ def get_stats(stat_re=None, stats_file=None): return ret def get_toolchain_prefix(tool, allowed_toolchains=None): - buildroot_full_prefix = os.path.join(this.host_bin_dir, this.buildroot_toolchain_prefix) + buildroot_full_prefix = os.path.join(this_module.host_bin_dir, this_module.buildroot_toolchain_prefix) buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool)) - crosstool_ng_full_prefix = os.path.join(this.crosstool_ng_bin_dir, this.crosstool_ng_toolchain_prefix) + crosstool_ng_full_prefix = os.path.join(this_module.crosstool_ng_bin_dir, this_module.crosstool_ng_toolchain_prefix) crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool)) - host_tool = '{}-{}'.format(this.ubuntu_toolchain_prefix, tool) + host_tool = '{}-{}'.format(this_module.ubuntu_toolchain_prefix, tool) host_path = shutil.which(host_tool) if host_path is not None: host_exists = True @@ -293,7 +327,7 @@ def get_toolchain_prefix(tool, allowed_toolchains=None): 'host': (host_exists, host_full_prefix), } if allowed_toolchains is None: - if this.baremetal is None: + if this_module.baremetal is None: allowed_toolchains = ['buildroot', 'crosstool-ng', 'host'] else: allowed_toolchains = ['crosstool-ng', 'buildroot', 'host'] @@ -306,7 +340,7 @@ def get_toolchain_prefix(tool, allowed_toolchains=None): raise Exception('Tool not found. Tried:\n' + '\n'.join(tried)) def get_toolchain_tool(tool, allowed_toolchains=None): - return '{}-{}'.format(this.get_toolchain_prefix(tool, allowed_toolchains), tool) + return '{}-{}'.format(this_module.get_toolchain_prefix(tool, allowed_toolchains), tool) def github_make_request( authenticate=False, @@ -317,7 +351,7 @@ def github_make_request( url_params=None, **extra_request_args ): - global this + global this_module if extra_headers is None: extra_headers = {} headers = {'Accept': 'application/vnd.github.v3+json'} @@ -343,20 +377,20 @@ def log_error(msg): print('error: {}'.format(msg), file=sys.stderr) def make_build_dirs(): - global this - os.makedirs(this.build_dir, exist_ok=True) - os.makedirs(this.gem5_build_dir, exist_ok=True) - os.makedirs(this.out_rootfs_overlay_dir, exist_ok=True) + global this_module + os.makedirs(this_module.build_dir, exist_ok=True) + os.makedirs(this_module.gem5_build_dir, exist_ok=True) + os.makedirs(this_module.out_rootfs_overlay_dir, exist_ok=True) def make_run_dirs(): ''' Make directories required for the run. The user could nuke those anytime between runs to try and clean things up. ''' - global this - os.makedirs(this.gem5_run_dir, exist_ok=True) - os.makedirs(this.p9_dir, exist_ok=True) - os.makedirs(this.qemu_run_dir, exist_ok=True) + global this_module + os.makedirs(this_module.gem5_run_dir, exist_ok=True) + os.makedirs(this_module.p9_dir, exist_ok=True) + os.makedirs(this_module.qemu_run_dir, exist_ok=True) def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): ''' @@ -379,7 +413,7 @@ def print_cmd(cmd, cwd=None, cmd_file=None, extra_env=None, extra_paths=None): for arg in cmd: out.append(shlex.quote(arg) + newline_separator) out = ' '.join(out) + ';\n' - print(this.command_prefix + out, end='') + print(this_module.command_prefix + out, end='') if cmd_file is not None: with open(cmd_file, 'w') as f: f.write('#!/usr/bin/env bash\n') @@ -393,15 +427,15 @@ def print_time(ellapsed_seconds): print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) def raw_to_qcow2(prebuilt=False, reverse=False): - global this + global this_module if prebuilt: - qemu_img_executable = this.qemu_img_basename + qemu_img_executable = this_module.qemu_img_basename else: - qemu_img_executable = this.qemu_img_executable + qemu_img_executable = this_module.qemu_img_executable infmt = 'raw' outfmt = 'qcow2' - infile = this.rootfs_raw_file - outfile = this.qcow2_file + infile = this_module.rootfs_raw_file + outfile = this_module.qcow2_file if reverse: tmp = infmt infmt = outfmt @@ -409,7 +443,7 @@ def raw_to_qcow2(prebuilt=False, reverse=False): tmp = infile infile = outfile outfile = tmp - assert this.run_cmd([ + assert this_module.run_cmd([ qemu_img_executable, # Prevent qemu-img from generating trace files like QEMU. Disgusting. '-T', 'pr_manager_run,file=/dev/null', @@ -431,8 +465,14 @@ def resolve_args(defaults, args, extra_args): argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items())) return argcopy +def cp(src, dest): + print_cmd(['cp', src, dest]) + if not this_module.dry_run: + shutil.copy2(src, dest) + def rmrf(path): - if os.path.exists(path): + print_cmd(['rm', '-r', '-f', path]) + if not this_module.dry_run and os.path.exists(path): shutil.rmtree(path) def run_cmd( @@ -508,13 +548,13 @@ def run_cmd( signal.signal(signal.SIGINT, signal.SIG_IGN) # Otherwise BrokenPipeError when piping through | grep - # But if I do this, my terminal gets broken at the end. Why, why, why. + # But if I do this_module, my terminal gets broken at the end. Why, why, why. # https://stackoverflow.com/questions/14207708/ioerror-errno-32-broken-pipe-python # Ignoring the exception is not enough as it prints a warning anyways. #sigpipe_old = signal.getsignal(signal.SIGPIPE) #signal.signal(signal.SIGPIPE, signal.SIG_DFL) - if not dry_run and not this.dry_run: + if not dry_run and not this_module.dry_run: # https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802 with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc: if out_file is not None: @@ -544,10 +584,10 @@ def setup(parser): Parse the command line arguments, and setup several variables based on them. Typically done after getting inputs from the command line arguments. ''' - global this + global this_module args = parser.parse_args() - if args.arch in this.arch_short_to_long_dict: - args.arch = this.arch_short_to_long_dict[args.arch] + if args.arch in this_module.arch_short_to_long_dict: + args.arch = this_module.arch_short_to_long_dict[args.arch] if args.gem5_build_id is None: args.gem5_build_id = default_build_id gem5_build_id_given = False @@ -555,144 +595,144 @@ def setup(parser): gem5_build_id_given = True if args.gem5_worktree is not None and not gem5_build_id_given: args.gem5_build_id = args.gem5_worktree - this.machine = args.machine - this.dry_run = args.dry_run + this_module.machine = args.machine + this_module.dry_run = args.dry_run if args.arch == 'arm': - this.armv = 7 - this.gem5_arch = 'ARM' - this.mcpu = 'cortex-a15' - this.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf' - this.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi' - this.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf' + this_module.armv = 7 + this_module.gem5_arch = 'ARM' + this_module.mcpu = 'cortex-a15' + this_module.buildroot_toolchain_prefix = 'arm-buildroot-linux-uclibcgnueabihf' + this_module.crosstool_ng_toolchain_prefix = 'arm-unknown-eabi' + this_module.ubuntu_toolchain_prefix = 'arm-linux-gnueabihf' if args.gem5: - if this.machine is None: - this.machine = 'VExpress_GEM5_V1' + if this_module.machine is None: + this_module.machine = 'VExpress_GEM5_V1' else: - if this.machine is None: - this.machine = 'virt' + if this_module.machine is None: + this_module.machine = 'virt' elif args.arch == 'aarch64': - this.armv = 8 - this.gem5_arch = 'ARM' - this.mcpu = 'cortex-a57' - this.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc' - this.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf' - this.ubuntu_toolchain_prefix = 'aarch64-linux-gnu' + this_module.armv = 8 + this_module.gem5_arch = 'ARM' + this_module.mcpu = 'cortex-a57' + this_module.buildroot_toolchain_prefix = 'aarch64-buildroot-linux-uclibc' + this_module.crosstool_ng_toolchain_prefix = 'aarch64-unknown-elf' + this_module.ubuntu_toolchain_prefix = 'aarch64-linux-gnu' if args.gem5: - if this.machine is None: - this.machine = 'VExpress_GEM5_V1' + if this_module.machine is None: + this_module.machine = 'VExpress_GEM5_V1' else: - if this.machine is None: - this.machine = 'virt' + if this_module.machine is None: + this_module.machine = 'virt' elif args.arch == 'x86_64': - this.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf' - this.gem5_arch = 'X86' - this.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc' - this.ubuntu_toolchain_prefix = 'x86_64-linux-gnu' + this_module.crosstool_ng_toolchain_prefix = 'x86_64-unknown-elf' + this_module.gem5_arch = 'X86' + this_module.buildroot_toolchain_prefix = 'x86_64-buildroot-linux-uclibc' + this_module.ubuntu_toolchain_prefix = 'x86_64-linux-gnu' if args.gem5: - if this.machine is None: - this.machine = 'TODO' + if this_module.machine is None: + this_module.machine = 'TODO' else: - if this.machine is None: - this.machine = 'pc' - this.buildroot_out_dir = os.path.join(this.out_dir, 'buildroot') - this.buildroot_build_dir = os.path.join(this.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch) - this.buildroot_download_dir = os.path.join(this.buildroot_out_dir, 'download') - this.buildroot_config_file = os.path.join(this.buildroot_build_dir, '.config') - this.build_dir = os.path.join(this.buildroot_build_dir, 'build') - this.qemu_build_dir = os.path.join(this.out_dir, 'qemu', args.qemu_build_id) - this.qemu_executable_basename = 'qemu-system-{}'.format(args.arch) - this.qemu_executable = os.path.join(this.qemu_build_dir, '{}-softmmu'.format(args.arch), this.qemu_executable_basename) - this.qemu_img_basename = 'qemu-img' - this.qemu_img_executable = os.path.join(this.qemu_build_dir, this.qemu_img_basename) - this.qemu_guest_build_dir = os.path.join(this.build_dir, 'qemu-custom') - this.host_dir = os.path.join(this.buildroot_build_dir, 'host') - this.host_bin_dir = os.path.join(this.host_dir, 'usr', 'bin') - this.buildroot_pkg_config = os.path.join(this.host_bin_dir, 'pkg-config') - this.buildroot_images_dir = os.path.join(this.buildroot_build_dir, 'images') - this.buildroot_rootfs_raw_file = os.path.join(this.buildroot_images_dir, 'rootfs.ext2') - this.buildroot_qcow2_file = this.buildroot_rootfs_raw_file + '.qcow2' - this.staging_dir = os.path.join(this.out_dir, 'staging', args.arch) - this.buildroot_staging_dir = os.path.join(this.buildroot_build_dir, 'staging') - this.target_dir = os.path.join(this.buildroot_build_dir, 'target') - this.run_dir_base = os.path.join(this.out_dir, 'run') - this.gem5_run_dir = os.path.join(this.run_dir_base, 'gem5', args.arch, str(args.run_id)) - this.m5out_dir = os.path.join(this.gem5_run_dir, 'm5out') - this.stats_file = os.path.join(this.m5out_dir, 'stats.txt') - this.trace_txt_file = os.path.join(this.m5out_dir, 'trace.txt') - this.gem5_readfile = os.path.join(this.gem5_run_dir, 'readfile') - this.gem5_termout_file = os.path.join(this.gem5_run_dir, 'termout.txt') - this.qemu_run_dir = os.path.join(this.run_dir_base, 'qemu', args.arch, str(args.run_id)) - this.qemu_trace_basename = 'trace.bin' - this.qemu_trace_file = os.path.join(this.qemu_run_dir, 'trace.bin') - this.qemu_trace_txt_file = os.path.join(this.qemu_run_dir, 'trace.txt') - this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt') - this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile') - this.gem5_out_dir = os.path.join(this.out_dir, 'gem5') + if this_module.machine is None: + this_module.machine = 'pc' + this_module.buildroot_out_dir = os.path.join(this_module.out_dir, 'buildroot') + this_module.buildroot_build_dir = os.path.join(this_module.buildroot_out_dir, 'build', args.buildroot_build_id, args.arch) + this_module.buildroot_download_dir = os.path.join(this_module.buildroot_out_dir, 'download') + this_module.buildroot_config_file = os.path.join(this_module.buildroot_build_dir, '.config') + this_module.build_dir = os.path.join(this_module.buildroot_build_dir, 'build') + this_module.qemu_build_dir = os.path.join(this_module.out_dir, 'qemu', args.qemu_build_id) + this_module.qemu_executable_basename = 'qemu-system-{}'.format(args.arch) + this_module.qemu_executable = os.path.join(this_module.qemu_build_dir, '{}-softmmu'.format(args.arch), this_module.qemu_executable_basename) + this_module.qemu_img_basename = 'qemu-img' + this_module.qemu_img_executable = os.path.join(this_module.qemu_build_dir, this_module.qemu_img_basename) + this_module.qemu_guest_build_dir = os.path.join(this_module.build_dir, 'qemu-custom') + this_module.host_dir = os.path.join(this_module.buildroot_build_dir, 'host') + this_module.host_bin_dir = os.path.join(this_module.host_dir, 'usr', 'bin') + this_module.buildroot_pkg_config = os.path.join(this_module.host_bin_dir, 'pkg-config') + this_module.buildroot_images_dir = os.path.join(this_module.buildroot_build_dir, 'images') + this_module.buildroot_rootfs_raw_file = os.path.join(this_module.buildroot_images_dir, 'rootfs.ext2') + this_module.buildroot_qcow2_file = this_module.buildroot_rootfs_raw_file + '.qcow2' + this_module.staging_dir = os.path.join(this_module.out_dir, 'staging', args.arch) + this_module.buildroot_staging_dir = os.path.join(this_module.buildroot_build_dir, 'staging') + this_module.target_dir = os.path.join(this_module.buildroot_build_dir, 'target') + this_module.run_dir_base = os.path.join(this_module.out_dir, 'run') + this_module.gem5_run_dir = os.path.join(this_module.run_dir_base, 'gem5', args.arch, str(args.run_id)) + this_module.m5out_dir = os.path.join(this_module.gem5_run_dir, 'm5out') + this_module.stats_file = os.path.join(this_module.m5out_dir, 'stats.txt') + this_module.trace_txt_file = os.path.join(this_module.m5out_dir, 'trace.txt') + this_module.gem5_readfile = os.path.join(this_module.gem5_run_dir, 'readfile') + this_module.gem5_termout_file = os.path.join(this_module.gem5_run_dir, 'termout.txt') + this_module.qemu_run_dir = os.path.join(this_module.run_dir_base, 'qemu', args.arch, str(args.run_id)) + this_module.qemu_trace_basename = 'trace.bin' + this_module.qemu_trace_file = os.path.join(this_module.qemu_run_dir, 'trace.bin') + this_module.qemu_trace_txt_file = os.path.join(this_module.qemu_run_dir, 'trace.txt') + this_module.qemu_termout_file = os.path.join(this_module.qemu_run_dir, 'termout.txt') + this_module.qemu_rrfile = os.path.join(this_module.qemu_run_dir, 'rrfile') + this_module.gem5_out_dir = os.path.join(this_module.out_dir, 'gem5') if args.gem5_build_dir is None: - this.gem5_build_dir = os.path.join(this.gem5_out_dir, args.gem5_build_id, args.gem5_build_type) + this_module.gem5_build_dir = os.path.join(this_module.gem5_out_dir, args.gem5_build_id, args.gem5_build_type) else: - this.gem5_build_dir = args.gem5_build_dir - this.gem5_fake_iso = os.path.join(this.gem5_out_dir, 'fake.iso') - this.gem5_m5term = os.path.join(this.gem5_build_dir, 'm5term') - this.gem5_build_build_dir = os.path.join(this.gem5_build_dir, 'build') - this.gem5_executable = os.path.join(this.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type)) - this.gem5_system_dir = os.path.join(this.gem5_build_dir, 'system') - this.crosstool_ng_out_dir = os.path.join(this.out_dir, 'crosstool-ng') - this.crosstool_ng_buildid_dir = os.path.join(this.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id) - this.crosstool_ng_install_dir = os.path.join(this.crosstool_ng_buildid_dir, 'install', args.arch) - this.crosstool_ng_bin_dir = os.path.join(this.crosstool_ng_install_dir, 'bin') - this.crosstool_ng_util_dir = os.path.join(this.crosstool_ng_buildid_dir, 'util') - this.crosstool_ng_config = os.path.join(this.crosstool_ng_util_dir, '.config') - this.crosstool_ng_defconfig = os.path.join(this.crosstool_ng_util_dir, 'defconfig') - this.crosstool_ng_executable = os.path.join(this.crosstool_ng_util_dir, 'ct-ng') - this.crosstool_ng_build_dir = os.path.join(this.crosstool_ng_buildid_dir, 'build') - this.crosstool_ng_download_dir = os.path.join(this.crosstool_ng_out_dir, 'download') - this.gem5_default_src_dir = os.path.join(submodules_dir, 'gem5') + this_module.gem5_build_dir = args.gem5_build_dir + this_module.gem5_fake_iso = os.path.join(this_module.gem5_out_dir, 'fake.iso') + this_module.gem5_m5term = os.path.join(this_module.gem5_build_dir, 'm5term') + this_module.gem5_build_build_dir = os.path.join(this_module.gem5_build_dir, 'build') + this_module.gem5_executable = os.path.join(this_module.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type)) + this_module.gem5_system_dir = os.path.join(this_module.gem5_build_dir, 'system') + this_module.crosstool_ng_out_dir = os.path.join(this_module.out_dir, 'crosstool-ng') + this_module.crosstool_ng_buildid_dir = os.path.join(this_module.crosstool_ng_out_dir, 'build', args.crosstool_ng_build_id) + this_module.crosstool_ng_install_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'install', args.arch) + this_module.crosstool_ng_bin_dir = os.path.join(this_module.crosstool_ng_install_dir, 'bin') + this_module.crosstool_ng_util_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'util') + this_module.crosstool_ng_config = os.path.join(this_module.crosstool_ng_util_dir, '.config') + this_module.crosstool_ng_defconfig = os.path.join(this_module.crosstool_ng_util_dir, 'defconfig') + this_module.crosstool_ng_executable = os.path.join(this_module.crosstool_ng_util_dir, 'ct-ng') + this_module.crosstool_ng_build_dir = os.path.join(this_module.crosstool_ng_buildid_dir, 'build') + this_module.crosstool_ng_download_dir = os.path.join(this_module.crosstool_ng_out_dir, 'download') + this_module.gem5_default_src_dir = os.path.join(submodules_dir, 'gem5') if args.gem5_source_dir is not None: - this.gem5_src_dir = args.gem5_source_dir + this_module.gem5_src_dir = args.gem5_source_dir assert(os.path.exists(args.gem5_source_dir)) else: if args.gem5_worktree is not None: - this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree) + this_module.gem5_src_dir = os.path.join(this_module.gem5_non_default_src_root_dir, args.gem5_worktree) else: - this.gem5_src_dir = this.gem5_default_src_dir - this.gem5_m5_src_dir = os.path.join(this.gem5_src_dir, 'util', 'm5') - this.gem5_m5_build_dir = os.path.join(this.out_dir, 'util', 'm5') + this_module.gem5_src_dir = this_module.gem5_default_src_dir + this_module.gem5_m5_src_dir = os.path.join(this_module.gem5_src_dir, 'util', 'm5') + this_module.gem5_m5_build_dir = os.path.join(this_module.out_dir, 'util', 'm5') if args.gem5: - this.executable = this.gem5_executable - this.run_dir = this.gem5_run_dir - this.termout_file = this.gem5_termout_file + this_module.executable = this_module.gem5_executable + this_module.run_dir = this_module.gem5_run_dir + this_module.termout_file = this_module.gem5_termout_file else: - this.executable = this.qemu_executable - this.run_dir = this.qemu_run_dir - this.termout_file = this.qemu_termout_file - this.gem5_config_dir = os.path.join(this.gem5_src_dir, 'configs') - this.gem5_se_file = os.path.join(this.gem5_config_dir, 'example', 'se.py') - this.gem5_fs_file = os.path.join(this.gem5_config_dir, 'example', 'fs.py') - this.run_cmd_file = os.path.join(this.run_dir, 'run.sh') + this_module.executable = this_module.qemu_executable + this_module.run_dir = this_module.qemu_run_dir + this_module.termout_file = this_module.qemu_termout_file + this_module.gem5_config_dir = os.path.join(this_module.gem5_src_dir, 'configs') + this_module.gem5_se_file = os.path.join(this_module.gem5_config_dir, 'example', 'se.py') + this_module.gem5_fs_file = os.path.join(this_module.gem5_config_dir, 'example', 'fs.py') + this_module.run_cmd_file = os.path.join(this_module.run_dir, 'run.sh') # Linux - this.linux_buildroot_build_dir = os.path.join(this.build_dir, 'linux-custom') - this.linux_build_dir = os.path.join(this.out_dir, 'linux', args.linux_build_id, args.arch) - this.vmlinux = os.path.join(this.linux_build_dir, "vmlinux") + this_module.linux_buildroot_build_dir = os.path.join(this_module.build_dir, 'linux-custom') + this_module.linux_build_dir = os.path.join(this_module.out_dir, 'linux', args.linux_build_id, args.arch) + this_module.vmlinux = os.path.join(this_module.linux_build_dir, "vmlinux") if args.arch == 'arm': - this.linux_arch = 'arm' - this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'zImage') + this_module.linux_arch = 'arm' + this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'zImage') elif args.arch == 'aarch64': - this.linux_arch = 'arm64' - this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'Image') + this_module.linux_arch = 'arm64' + this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'Image') elif args.arch == 'x86_64': - this.linux_arch = 'x86' - this.linux_image = os.path.join('arch', this.linux_arch, 'boot', 'bzImage') - this.linux_image = os.path.join(this.linux_build_dir, linux_image) + this_module.linux_arch = 'x86' + this_module.linux_image = os.path.join('arch', this_module.linux_arch, 'boot', 'bzImage') + this_module.linux_image = os.path.join(this_module.linux_build_dir, linux_image) # Kernel modules. - this.kernel_modules_build_base_dir = os.path.join(this.out_dir, 'kernel_modules') - this.kernel_modules_build_dir = os.path.join(this.kernel_modules_build_base_dir, args.arch) - this.kernel_modules_build_host_dir = os.path.join(this.kernel_modules_build_base_dir, 'host') - this.userland_build_dir = os.path.join(this.out_dir, 'userland', args.arch) - this.out_rootfs_overlay_dir = os.path.join(this.out_dir, 'rootfs_overlay', args.arch) - this.out_rootfs_overlay_bin_dir = os.path.join(this.out_rootfs_overlay_dir, 'bin') + this_module.kernel_modules_build_base_dir = os.path.join(this_module.out_dir, 'kernel_modules') + this_module.kernel_modules_build_dir = os.path.join(this_module.kernel_modules_build_base_dir, args.arch) + this_module.kernel_modules_build_host_dir = os.path.join(this_module.kernel_modules_build_base_dir, 'host') + this_module.userland_build_dir = os.path.join(this_module.out_dir, 'userland', args.arch) + this_module.out_rootfs_overlay_dir = os.path.join(this_module.out_dir, 'rootfs_overlay', args.arch) + this_module.out_rootfs_overlay_bin_dir = os.path.join(this_module.out_rootfs_overlay_dir, 'bin') # Ports if args.port_offset is None: @@ -701,69 +741,69 @@ def setup(parser): except ValueError: args.port_offset = 0 if args.gem5: - this.gem5_telnet_port = 3456 + args.port_offset - this.gdb_port = 7000 + args.port_offset + this_module.gem5_telnet_port = 3456 + args.port_offset + this_module.gdb_port = 7000 + args.port_offset else: - this.qemu_base_port = 45454 + 10 * args.port_offset - this.qemu_monitor_port = this.qemu_base_port + 0 - this.qemu_hostfwd_generic_port = this.qemu_base_port + 1 - this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2 - this.qemu_gdb_port = this.qemu_base_port + 3 - this.gdb_port = this.qemu_gdb_port + this_module.qemu_base_port = 45454 + 10 * args.port_offset + this_module.qemu_monitor_port = this_module.qemu_base_port + 0 + this_module.qemu_hostfwd_generic_port = this_module.qemu_base_port + 1 + this_module.qemu_hostfwd_ssh_port = this_module.qemu_base_port + 2 + this_module.qemu_gdb_port = this_module.qemu_base_port + 3 + this_module.gdb_port = this_module.qemu_gdb_port # Baremetal. - this.baremetal = args.baremetal - this.baremetal_lib_basename = 'lib' - this.baremetal_src_dir = os.path.join(this.root_dir, 'baremetal') - this.baremetal_src_lib_dir = os.path.join(this.baremetal_src_dir, this.baremetal_lib_basename) + this_module.baremetal = args.baremetal + this_module.baremetal_lib_basename = 'lib' + this_module.baremetal_src_dir = os.path.join(this_module.root_dir, 'baremetal') + this_module.baremetal_src_lib_dir = os.path.join(this_module.baremetal_src_dir, this_module.baremetal_lib_basename) if args.gem5: - this.simulator_name = 'gem5' + this_module.simulator_name = 'gem5' else: - this.simulator_name = 'qemu' - this.baremetal_out_dir = os.path.join(out_dir, 'baremetal', args.arch, this.simulator_name, this.machine) - this.baremetal_out_lib_dir = os.path.join(this.baremetal_out_dir, this.baremetal_lib_basename) - this.baremetal_out_ext = '.elf' + this_module.simulator_name = 'qemu' + this_module.baremetal_out_dir = os.path.join(out_dir, 'baremetal', args.arch, this_module.simulator_name, this_module.machine) + this_module.baremetal_out_lib_dir = os.path.join(this_module.baremetal_out_dir, this_module.baremetal_lib_basename) + this_module.baremetal_out_ext = '.elf' # Docker - this.docker_build_dir = os.path.join(this.out_dir, 'docker', args.arch) - this.docker_tar_dir = os.path.join(this.docker_build_dir, 'export') - this.docker_tar_file = os.path.join(this.docker_build_dir, 'export.tar') - this.docker_rootfs_raw_file = os.path.join(this.docker_build_dir, 'export.ext2') - this.docker_qcow2_file = os.path.join(this.docker_rootfs_raw_file + '.qcow2') + this_module.docker_build_dir = os.path.join(this_module.out_dir, 'docker', args.arch) + this_module.docker_tar_dir = os.path.join(this_module.docker_build_dir, 'export') + this_module.docker_tar_file = os.path.join(this_module.docker_build_dir, 'export.tar') + this_module.docker_rootfs_raw_file = os.path.join(this_module.docker_build_dir, 'export.ext2') + this_module.docker_qcow2_file = os.path.join(this_module.docker_rootfs_raw_file + '.qcow2') if args.docker: - this.rootfs_raw_file = this.docker_rootfs_raw_file - this.qcow2_file = this.docker_qcow2_file + this_module.rootfs_raw_file = this_module.docker_rootfs_raw_file + this_module.qcow2_file = this_module.docker_qcow2_file else: - this.rootfs_raw_file = this.buildroot_rootfs_raw_file - this.qcow2_file = this.buildroot_qcow2_file + this_module.rootfs_raw_file = this_module.buildroot_rootfs_raw_file + this_module.qcow2_file = this_module.buildroot_qcow2_file # Image. if args.baremetal is None: if args.gem5: - this.image = this.vmlinux - this.disk_image = this.rootfs_raw_file + this_module.image = this_module.vmlinux + this_module.disk_image = this_module.rootfs_raw_file else: - this.image = this.linux_image - this.disk_image = this.qcow2_file + this_module.image = this_module.linux_image + this_module.disk_image = this_module.qcow2_file else: - this.disk_image = this.gem5_fake_iso + this_module.disk_image = this_module.gem5_fake_iso paths = [ - os.path.join(this.baremetal_out_dir, this.baremetal), + os.path.join(this_module.baremetal_out_dir, this_module.baremetal), os.path.join( - this.baremetal_out_dir, - os.path.relpath(this.baremetal, this.baremetal_src_dir), + this_module.baremetal_out_dir, + os.path.relpath(this_module.baremetal, this_module.baremetal_src_dir), ) ] - paths[:] = [os.path.splitext(path)[0] + this.baremetal_out_ext for path in paths] + paths[:] = [os.path.splitext(path)[0] + this_module.baremetal_out_ext for path in paths] found = False for path in paths: if os.path.exists(path): found = True break - if not found and this.baremetal != 'all': + if not found and this_module.baremetal != 'all': raise Exception('Baremetal ELF file not found. Tried:\n' + '\n'.join(paths)) - this.image = path + this_module.image = path return args def write_configs(config_path, configs, config_fragments=None):