mirror of
https://github.com/cirosantilli/linux-kernel-module-cheat.git
synced 2026-01-13 20:12:26 +00:00
795 lines
32 KiB
Python
795 lines
32 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import base64
|
|
import collections
|
|
import copy
|
|
import datetime
|
|
import glob
|
|
import imp
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib
|
|
import urllib.request
|
|
|
|
import cli_function
|
|
import shell_helpers
|
|
|
|
common = sys.modules[__name__]
|
|
|
|
# Fixed parameters that don't depend on CLI arguments.
|
|
consts = {}
|
|
consts['repo_short_id'] = 'lkmc'
|
|
# https://stackoverflow.com/questions/20010199/how-to-determine-if-a-process-runs-inside-lxc-docker
|
|
consts['in_docker'] = os.path.exists('/.dockerenv')
|
|
consts['root_dir'] = os.path.dirname(os.path.abspath(__file__))
|
|
consts['data_dir'] = os.path.join(consts['root_dir'], 'data')
|
|
consts['p9_dir'] = os.path.join(consts['data_dir'], '9p')
|
|
consts['gem5_non_default_src_root_dir'] = os.path.join(consts['data_dir'], 'gem5')
|
|
if consts['in_docker']:
|
|
consts['out_dir'] = os.path.join(consts['root_dir'], 'out.docker')
|
|
else:
|
|
consts['out_dir'] = os.path.join(consts['root_dir'], 'out')
|
|
consts['gem5_out_dir'] = os.path.join(consts['out_dir'], 'gem5')
|
|
consts['kernel_modules_build_base_dir'] = os.path.join(consts['out_dir'], 'kernel_modules')
|
|
consts['buildroot_out_dir'] = os.path.join(consts['out_dir'], 'buildroot')
|
|
consts['gem5_m5_build_dir'] = os.path.join(consts['out_dir'], 'util', 'm5')
|
|
consts['run_dir_base'] = os.path.join(consts['out_dir'], 'run')
|
|
consts['crosstool_ng_out_dir'] = os.path.join(consts['out_dir'], 'crosstool-ng')
|
|
consts['bench_boot'] = os.path.join(consts['out_dir'], 'bench-boot.txt')
|
|
consts['packages_dir'] = os.path.join(consts['root_dir'], 'buildroot_packages')
|
|
consts['kernel_modules_subdir'] = 'kernel_modules'
|
|
consts['kernel_modules_src_dir'] = os.path.join(consts['root_dir'], consts['kernel_modules_subdir'])
|
|
consts['userland_subdir'] = 'userland'
|
|
consts['userland_src_dir'] = os.path.join(consts['root_dir'], consts['userland_subdir'])
|
|
consts['userland_build_ext'] = '.out'
|
|
consts['include_subdir'] = 'include'
|
|
consts['include_src_dir'] = os.path.join(consts['root_dir'], consts['include_subdir'])
|
|
consts['submodules_dir'] = os.path.join(consts['root_dir'], 'submodules')
|
|
consts['buildroot_src_dir'] = os.path.join(consts['submodules_dir'], 'buildroot')
|
|
consts['crosstool_ng_src_dir'] = os.path.join(consts['submodules_dir'], 'crosstool-ng')
|
|
consts['crosstool_ng_supported_archs'] = set(['arm', 'aarch64'])
|
|
consts['linux_src_dir'] = os.path.join(consts['submodules_dir'], 'linux')
|
|
consts['linux_config_dir'] = os.path.join(consts['root_dir'], 'linux_config')
|
|
consts['gem5_default_src_dir'] = os.path.join(consts['submodules_dir'], 'gem5')
|
|
consts['rootfs_overlay_dir'] = os.path.join(consts['root_dir'], 'rootfs_overlay')
|
|
consts['extract_vmlinux'] = os.path.join(consts['linux_src_dir'], 'scripts', 'extract-vmlinux')
|
|
consts['qemu_src_dir'] = os.path.join(consts['submodules_dir'], 'qemu')
|
|
consts['parsec_benchmark_src_dir'] = os.path.join(consts['submodules_dir'], 'parsec-benchmark')
|
|
consts['ccache_dir'] = os.path.join('/usr', 'lib', 'ccache')
|
|
consts['default_build_id'] = 'default'
|
|
consts['arch_short_to_long_dict'] = collections.OrderedDict([
|
|
('x', 'x86_64'),
|
|
('a', 'arm'),
|
|
('A', 'aarch64'),
|
|
])
|
|
consts['all_archs'] = [consts['arch_short_to_long_dict'][k] for k in consts['arch_short_to_long_dict']]
|
|
consts['arch_choices'] = []
|
|
for key in consts['arch_short_to_long_dict']:
|
|
consts['arch_choices'].append(key)
|
|
consts['arch_choices'].append(consts['arch_short_to_long_dict'][key])
|
|
consts['default_arch'] = 'x86_64'
|
|
consts['gem5_cpt_prefix'] = '^cpt\.'
|
|
consts['sha'] = subprocess.check_output(['git', '-C', consts['root_dir'], 'log', '-1', '--format=%H']).decode().rstrip()
|
|
consts['release_dir'] = os.path.join(consts['out_dir'], 'release')
|
|
consts['release_zip_file'] = os.path.join(consts['release_dir'], 'lkmc-{}.zip'.format(consts['sha']))
|
|
consts['github_repo_id'] = 'cirosantilli/linux-kernel-module-cheat'
|
|
consts['asm_ext'] = '.S'
|
|
consts['c_ext'] = '.c'
|
|
consts['header_ext'] = '.h'
|
|
consts['kernel_module_ext'] = '.ko'
|
|
consts['obj_ext'] = '.o'
|
|
consts['config_file'] = os.path.join(consts['data_dir'], 'config.py')
|
|
consts['magic_fail_string'] = b'lkmc_test_fail'
|
|
consts['baremetal_lib_basename'] = 'lib'
|
|
|
|
class LkmcCliFunction(cli_function.CliFunction):
|
|
'''
|
|
Common functionality shared across our CLI functions:
|
|
|
|
* command timing
|
|
* some common flags, e.g.: --arch, --dry-run, --verbose
|
|
'''
|
|
def __init__(self, *args, **kwargs):
|
|
kwargs['config_file'] = consts['config_file']
|
|
super().__init__(*args, **kwargs)
|
|
|
|
# Args for all scripts.
|
|
self.add_argument(
|
|
'-a', '--arch', choices=consts['arch_choices'], default=consts['default_arch'],
|
|
help='CPU architecture.'
|
|
)
|
|
self.add_argument(
|
|
'--dry-run',
|
|
default=False,
|
|
help='''\
|
|
Print the commands that would be run, but don't run them.
|
|
|
|
We aim display every command that modifies the filesystem state, and generate
|
|
Bash equivalents even for actions taken directly in Python without shelling out.
|
|
|
|
mkdir are generally omitted since those are obvious
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'-v', '--verbose', default=False,
|
|
help='Show full compilation commands when they are not shown by default.'
|
|
)
|
|
|
|
# Gem5 args.
|
|
self.add_argument(
|
|
'--gem5-build-dir',
|
|
help='''\
|
|
Use the given directory as the gem5 build directory.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'-M', '--gem5-build-id', default='default',
|
|
help='''\
|
|
gem5 build ID. Allows you to keep multiple separate gem5 builds.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--gem5-build-type', default='opt',
|
|
help='gem5 build type, most often used for "debug" builds.'
|
|
)
|
|
self.add_argument(
|
|
'--gem5-source-dir',
|
|
help='''\
|
|
Use the given directory as the gem5 source tree. Ignore `--gem5-worktree`.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'-N', '--gem5-worktree',
|
|
help='''\
|
|
Create and use a git worktree of the gem5 submodule.
|
|
See: https://github.com/cirosantilli/linux-kernel-module-cheat#gem5-worktree
|
|
'''
|
|
)
|
|
|
|
# Linux kernel.
|
|
self.add_argument(
|
|
'-L', '--linux-build-id', default=consts['default_build_id'],
|
|
help='''\
|
|
Linux build ID. Allows you to keep multiple separate Linux builds.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--initramfs', default=False,
|
|
)
|
|
self.add_argument(
|
|
'--initrd', default=False,
|
|
)
|
|
|
|
# Baremetal.
|
|
self.add_argument(
|
|
'-b', '--baremetal',
|
|
help='''\
|
|
Use the given baremetal executable instead of the Linux kernel.
|
|
|
|
If the path is absolute, it is used as is.
|
|
|
|
If the path is relative, we assume that it points to a source code
|
|
inside baremetal/ and then try to use corresponding executable.
|
|
'''
|
|
)
|
|
|
|
# Buildroot.
|
|
self.add_argument(
|
|
'--buildroot-build-id',
|
|
default=consts['default_build_id'],
|
|
help='Buildroot build ID. Allows you to keep multiple separate gem5 builds.'
|
|
)
|
|
self.add_argument(
|
|
'--buildroot-linux', default=False,
|
|
help='Boot with the Buildroot Linux kernel instead of our custom built one. Mostly for sanity checks.'
|
|
)
|
|
|
|
# crosstool-ng
|
|
self.add_argument(
|
|
'--crosstool-ng-build-id', default=consts['default_build_id'],
|
|
help='Crosstool-NG build ID. Allows you to keep multiple separate crosstool-NG builds.'
|
|
)
|
|
self.add_argument(
|
|
'--docker', default=False,
|
|
help='''\
|
|
Use the docker download Ubuntu root filesystem instead of the default Buildroot one.
|
|
'''
|
|
)
|
|
|
|
self.add_argument(
|
|
'--machine',
|
|
help='''Machine type.
|
|
QEMU default: virt
|
|
gem5 default: VExpress_GEM5_V1
|
|
See the documentation for other values known to work.
|
|
'''
|
|
)
|
|
|
|
# QEMU.
|
|
self.add_argument(
|
|
'-Q', '--qemu-build-id', default=consts['default_build_id'],
|
|
help='QEMU build ID. Allows you to keep multiple separate QEMU builds.'
|
|
)
|
|
|
|
# Userland.
|
|
self.add_argument(
|
|
'--userland-build-id', default=None
|
|
)
|
|
|
|
# Run.
|
|
self.add_argument(
|
|
'-n', '--run-id', default='0',
|
|
help='''\
|
|
ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
|
|
and then inspect separate outputs later in different output directories.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'-P', '--prebuilt', default=False,
|
|
help='''\
|
|
Use prebuilt packaged host utilities as much as possible instead
|
|
of the ones we built ourselves. Saves build time, but decreases
|
|
the likelihood of incompatibilities.
|
|
'''
|
|
)
|
|
self.add_argument(
|
|
'--port-offset', type=int,
|
|
help='''\
|
|
Increase the ports to be used such as for GDB by an offset to run multiple
|
|
instances in parallel. Default: the run ID (-n) if that is an integer, otherwise 0.
|
|
'''
|
|
)
|
|
|
|
# Misc.
|
|
self.add_argument(
|
|
'-g', '--gem5', default=False,
|
|
help='Use gem5 instead of QEMU.'
|
|
)
|
|
self.add_argument(
|
|
'--qemu', default=False,
|
|
help='''\
|
|
Use QEMU as the emulator. This option exists in addition to --gem5
|
|
to allow overriding configs from the CLI.
|
|
'''
|
|
)
|
|
|
|
def _init_env(self, env):
|
|
'''
|
|
Update the kwargs from the command line with derived arguments.
|
|
'''
|
|
def join(*paths):
|
|
return os.path.join(*paths)
|
|
if env['qemu'] or not env['gem5']:
|
|
env['emulator'] = 'qemu'
|
|
else:
|
|
env['emulator'] = 'gem5'
|
|
if env['arch'] in env['arch_short_to_long_dict']:
|
|
env['arch'] = env['arch_short_to_long_dict'][env['arch']]
|
|
if env['userland_build_id'] is None:
|
|
env['userland_build_id'] = env['default_build_id']
|
|
env['userland_build_id_given'] = False
|
|
else:
|
|
env['userland_build_id_given'] = True
|
|
if env['gem5_worktree'] is not None and env['gem5_build_id'] is None:
|
|
env['gem5_build_id'] = env['gem5_worktree']
|
|
env['is_arm'] = False
|
|
if env['arch'] == 'arm':
|
|
env['armv'] = 7
|
|
env['gem5_arch'] = 'ARM'
|
|
env['mcpu'] = 'cortex-a15'
|
|
env['buildroot_toolchain_prefix'] = 'arm-buildroot-linux-uclibcgnueabihf'
|
|
env['crosstool_ng_toolchain_prefix'] = 'arm-unknown-eabi'
|
|
env['ubuntu_toolchain_prefix'] = 'arm-linux-gnueabihf'
|
|
if env['emulator'] == 'gem5':
|
|
if env['machine'] is None:
|
|
env['machine'] = 'VExpress_GEM5_V1'
|
|
else:
|
|
if env['machine'] is None:
|
|
env['machine'] = 'virt'
|
|
env['is_arm'] = True
|
|
elif env['arch'] == 'aarch64':
|
|
env['armv'] = 8
|
|
env['gem5_arch'] = 'ARM'
|
|
env['mcpu'] = 'cortex-a57'
|
|
env['buildroot_toolchain_prefix'] = 'aarch64-buildroot-linux-uclibc'
|
|
env['crosstool_ng_toolchain_prefix'] = 'aarch64-unknown-elf'
|
|
env['ubuntu_toolchain_prefix'] = 'aarch64-linux-gnu'
|
|
if env['emulator'] == 'gem5':
|
|
if env['machine'] is None:
|
|
env['machine'] = 'VExpress_GEM5_V1'
|
|
else:
|
|
if env['machine'] is None:
|
|
env['machine'] = 'virt'
|
|
env['is_arm'] = True
|
|
elif env['arch'] == 'x86_64':
|
|
env['crosstool_ng_toolchain_prefix'] = 'x86_64-unknown-elf'
|
|
env['gem5_arch'] = 'X86'
|
|
env['buildroot_toolchain_prefix'] = 'x86_64-buildroot-linux-uclibc'
|
|
env['ubuntu_toolchain_prefix'] = 'x86_64-linux-gnu'
|
|
if env['emulator'] == 'gem5':
|
|
if env['machine'] is None:
|
|
env['machine'] = 'TODO'
|
|
else:
|
|
if env['machine'] is None:
|
|
env['machine'] = 'pc'
|
|
|
|
# Buildroot
|
|
env['buildroot_build_dir'] = join(env['buildroot_out_dir'], 'build', env['buildroot_build_id'], env['arch'])
|
|
env['buildroot_download_dir'] = join(env['buildroot_out_dir'], 'download')
|
|
env['buildroot_config_file'] = join(env['buildroot_build_dir'], '.config')
|
|
env['buildroot_build_build_dir'] = join(env['buildroot_build_dir'], 'build')
|
|
env['buildroot_linux_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
|
|
env['buildroot_vmlinux'] = join(env['buildroot_linux_build_dir'], "vmlinux")
|
|
env['host_dir'] = join(env['buildroot_build_dir'], 'host')
|
|
env['host_bin_dir'] = join(env['host_dir'], 'usr', 'bin')
|
|
env['buildroot_pkg_config'] = join(env['host_bin_dir'], 'pkg-config')
|
|
env['buildroot_images_dir'] = join(env['buildroot_build_dir'], 'images')
|
|
env['buildroot_rootfs_raw_file'] = join(env['buildroot_images_dir'], 'rootfs.ext2')
|
|
env['buildroot_qcow2_file'] = env['buildroot_rootfs_raw_file'] + '.qcow2'
|
|
env['staging_dir'] = join(env['out_dir'], 'staging', env['arch'])
|
|
env['buildroot_staging_dir'] = join(env['buildroot_build_dir'], 'staging')
|
|
env['target_dir'] = join(env['buildroot_build_dir'], 'target')
|
|
env['linux_buildroot_build_dir'] = join(env['buildroot_build_build_dir'], 'linux-custom')
|
|
|
|
# QEMU
|
|
env['qemu_build_dir'] = join(env['out_dir'], 'qemu', env['qemu_build_id'])
|
|
env['qemu_executable_basename'] = 'qemu-system-{}'.format(env['arch'])
|
|
env['qemu_executable'] = join(env['qemu_build_dir'], '{}-softmmu'.format(env['arch']), env['qemu_executable_basename'])
|
|
env['qemu_img_basename'] = 'qemu-img'
|
|
env['qemu_img_executable'] = join(env['qemu_build_dir'], env['qemu_img_basename'])
|
|
|
|
# gem5
|
|
if env['gem5_build_dir'] is None:
|
|
env['gem5_build_dir'] = join(env['gem5_out_dir'], env['gem5_build_id'], env['gem5_build_type'])
|
|
env['gem5_fake_iso'] = join(env['gem5_out_dir'], 'fake.iso')
|
|
env['gem5_m5term'] = join(env['gem5_build_dir'], 'm5term')
|
|
env['gem5_build_build_dir'] = join(env['gem5_build_dir'], 'build')
|
|
env['gem5_executable'] = join(env['gem5_build_build_dir'], env['gem5_arch'], 'gem5.{}'.format(env['gem5_build_type']))
|
|
env['gem5_system_dir'] = join(env['gem5_build_dir'], 'system')
|
|
|
|
# gem5 source
|
|
if env['gem5_source_dir'] is not None:
|
|
assert os.path.exists(env['gem5_source_dir'])
|
|
else:
|
|
if env['gem5_worktree'] is not None:
|
|
env['gem5_source_dir'] = join(env['gem5_non_default_src_root_dir'], env['gem5_worktree'])
|
|
else:
|
|
env['gem5_source_dir'] = env['gem5_default_src_dir']
|
|
env['gem5_m5_source_dir'] = join(env['gem5_source_dir'], 'util', 'm5')
|
|
env['gem5_config_dir'] = join(env['gem5_source_dir'], 'configs')
|
|
env['gem5_se_file'] = join(env['gem5_config_dir'], 'example', 'se.py')
|
|
env['gem5_fs_file'] = join(env['gem5_config_dir'], 'example', 'fs.py')
|
|
|
|
# crosstool-ng
|
|
env['crosstool_ng_buildid_dir'] = join(env['crosstool_ng_out_dir'], 'build', env['crosstool_ng_build_id'])
|
|
env['crosstool_ng_install_dir'] = join(env['crosstool_ng_buildid_dir'], 'install', env['arch'])
|
|
env['crosstool_ng_bin_dir'] = join(env['crosstool_ng_install_dir'], 'bin')
|
|
env['crosstool_ng_util_dir'] = join(env['crosstool_ng_buildid_dir'], 'util')
|
|
env['crosstool_ng_config'] = join(env['crosstool_ng_util_dir'], '.config')
|
|
env['crosstool_ng_defconfig'] = join(env['crosstool_ng_util_dir'], 'defconfig')
|
|
env['crosstool_ng_executable'] = join(env['crosstool_ng_util_dir'], 'ct-ng')
|
|
env['crosstool_ng_build_dir'] = join(env['crosstool_ng_buildid_dir'], 'build')
|
|
env['crosstool_ng_download_dir'] = join(env['crosstool_ng_out_dir'], 'download')
|
|
|
|
# run
|
|
env['gem5_run_dir'] = join(env['run_dir_base'], 'gem5', env['arch'], str(env['run_id']))
|
|
env['m5out_dir'] = join(env['gem5_run_dir'], 'm5out')
|
|
env['stats_file'] = join(env['m5out_dir'], 'stats.txt')
|
|
env['gem5_trace_txt_file'] = join(env['m5out_dir'], 'trace.txt')
|
|
env['gem5_guest_terminal_file'] = join(env['m5out_dir'], 'system.terminal')
|
|
env['gem5_readfile'] = join(env['gem5_run_dir'], 'readfile')
|
|
env['gem5_termout_file'] = join(env['gem5_run_dir'], 'termout.txt')
|
|
env['qemu_run_dir'] = join(env['run_dir_base'], 'qemu', env['arch'], str(env['run_id']))
|
|
env['qemu_termout_file'] = join(env['qemu_run_dir'], 'termout.txt')
|
|
env['qemu_trace_basename'] = 'trace.bin'
|
|
env['qemu_trace_file'] = join(env['qemu_run_dir'], 'trace.bin')
|
|
env['qemu_trace_txt_file'] = join(env['qemu_run_dir'], 'trace.txt')
|
|
env['qemu_rrfile'] = join(env['qemu_run_dir'], 'rrfile')
|
|
env['gem5_out_dir'] = join(env['out_dir'], 'gem5')
|
|
|
|
# Ports
|
|
if env['port_offset'] is None:
|
|
try:
|
|
env['port_offset'] = int(env['run_id'])
|
|
except ValueError:
|
|
env['port_offset'] = 0
|
|
if env['emulator'] == 'gem5':
|
|
env['gem5_telnet_port'] = 3456 + env['port_offset']
|
|
env['gdb_port'] = 7000 + env['port_offset']
|
|
else:
|
|
env['qemu_base_port'] = 45454 + 10 * env['port_offset']
|
|
env['qemu_monitor_port'] = env['qemu_base_port'] + 0
|
|
env['qemu_hostfwd_generic_port'] = env['qemu_base_port'] + 1
|
|
env['qemu_hostfwd_ssh_port'] = env['qemu_base_port'] + 2
|
|
env['qemu_gdb_port'] = env['qemu_base_port'] + 3
|
|
env['extra_serial_port'] = env['qemu_base_port'] + 4
|
|
env['gdb_port'] = env['qemu_gdb_port']
|
|
env['qemu_background_serial_file'] = join(env['qemu_run_dir'], 'background.log')
|
|
|
|
# gem5 QEMU polymorphism.
|
|
if env['emulator'] == 'gem5':
|
|
env['executable'] = env['gem5_executable']
|
|
env['run_dir'] = env['gem5_run_dir']
|
|
env['termout_file'] = env['gem5_termout_file']
|
|
env['guest_terminal_file'] = env['gem5_guest_terminal_file']
|
|
env['trace_txt_file'] = env['gem5_trace_txt_file']
|
|
else:
|
|
env['executable'] = env['qemu_executable']
|
|
env['run_dir'] = env['qemu_run_dir']
|
|
env['termout_file'] = env['qemu_termout_file']
|
|
env['guest_terminal_file'] = env['qemu_termout_file']
|
|
env['trace_txt_file'] = env['qemu_trace_txt_file']
|
|
env['run_cmd_file'] = join(env['run_dir'], 'run.sh')
|
|
|
|
# Linux kernl.
|
|
if 'linux_build_id' in env:
|
|
env['linux_build_dir'] = join(env['out_dir'], 'linux', env['linux_build_id'], env['arch'])
|
|
env['lkmc_vmlinux'] = join(env['linux_build_dir'], "vmlinux")
|
|
if env['arch'] == 'arm':
|
|
env['linux_arch'] = 'arm'
|
|
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'zImage')
|
|
elif env['arch'] == 'aarch64':
|
|
env['linux_arch'] = 'arm64'
|
|
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'Image')
|
|
elif env['arch'] == 'x86_64':
|
|
env['linux_arch'] = 'x86'
|
|
env['linux_image_prefix'] = join('arch', env['linux_arch'], 'boot', 'bzImage')
|
|
env['lkmc_linux_image'] = join(env['linux_build_dir'], env['linux_image_prefix'])
|
|
env['buildroot_linux_image'] = join(env['buildroot_linux_build_dir'], env['linux_image_prefix'])
|
|
if env['buildroot_linux']:
|
|
env['vmlinux'] = env['buildroot_vmlinux']
|
|
env['linux_image'] = env['buildroot_linux_image']
|
|
else:
|
|
env['vmlinux'] = env['lkmc_vmlinux']
|
|
env['linux_image'] = env['lkmc_linux_image']
|
|
|
|
# Kernel modules.
|
|
env['kernel_modules_build_dir'] = join(env['kernel_modules_build_base_dir'], env['arch'])
|
|
env['kernel_modules_build_subdir'] = join(env['kernel_modules_build_dir'], env['kernel_modules_subdir'])
|
|
env['kernel_modules_build_host_dir'] = join(env['kernel_modules_build_base_dir'], 'host')
|
|
env['kernel_modules_build_host_subdir'] = join(env['kernel_modules_build_host_dir'], env['kernel_modules_subdir'])
|
|
env['userland_build_dir'] = join(env['out_dir'], 'userland', env['userland_build_id'], env['arch'])
|
|
env['out_rootfs_overlay_dir'] = join(env['out_dir'], 'rootfs_overlay', env['arch'])
|
|
env['out_rootfs_overlay_bin_dir'] = join(env['out_rootfs_overlay_dir'], 'bin')
|
|
|
|
# Baremetal.
|
|
env['baremetal_src_dir'] = join(env['root_dir'], 'baremetal')
|
|
env['baremetal_src_lib_dir'] = join(env['baremetal_src_dir'], env['baremetal_lib_basename'])
|
|
if env['emulator'] == 'gem5':
|
|
env['simulator_name'] = 'gem5'
|
|
else:
|
|
env['simulator_name'] = 'qemu'
|
|
env['baremetal_build_dir'] = join(env['out_dir'], 'baremetal', env['arch'], env['simulator_name'], env['machine'])
|
|
env['baremetal_build_lib_dir'] = join(env['baremetal_build_dir'], env['baremetal_lib_basename'])
|
|
env['baremetal_build_ext'] = '.elf'
|
|
|
|
# Docker
|
|
env['docker_build_dir'] = join(env['out_dir'], 'docker', env['arch'])
|
|
env['docker_tar_dir'] = join(env['docker_build_dir'], 'export')
|
|
env['docker_tar_file'] = join(env['docker_build_dir'], 'export.tar')
|
|
env['docker_rootfs_raw_file'] = join(env['docker_build_dir'], 'export.ext2')
|
|
env['docker_qcow2_file'] = join(env['docker_rootfs_raw_file'] + '.qcow2')
|
|
if env['docker']:
|
|
env['rootfs_raw_file'] = env['docker_rootfs_raw_file']
|
|
env['qcow2_file'] = env['docker_qcow2_file']
|
|
else:
|
|
env['rootfs_raw_file'] = env['buildroot_rootfs_raw_file']
|
|
env['qcow2_file'] = env['buildroot_qcow2_file']
|
|
|
|
# Image.
|
|
if env['baremetal'] is None:
|
|
if env['emulator'] == 'gem5':
|
|
env['image'] = env['vmlinux']
|
|
env['disk_image'] = env['rootfs_raw_file']
|
|
else:
|
|
env['image'] = env['linux_image']
|
|
env['disk_image'] = env['qcow2_file']
|
|
else:
|
|
env['disk_image'] = env['gem5_fake_iso']
|
|
if env['baremetal'] == 'all':
|
|
path = env['baremetal']
|
|
else:
|
|
path = self.resolve_executable(
|
|
env['baremetal'],
|
|
env['baremetal_src_dir'],
|
|
env['baremetal_build_dir'],
|
|
env['baremetal_build_ext'],
|
|
)
|
|
source_path_noext = os.path.splitext(join(
|
|
env['baremetal_src_dir'],
|
|
os.path.relpath(path, env['baremetal_build_dir'])
|
|
))[0]
|
|
for ext in [c_ext, asm_ext]:
|
|
source_path = source_path_noext + ext
|
|
if os.path.exists(source_path):
|
|
env['source_path'] = source_path
|
|
break
|
|
env['image'] = path
|
|
self.env = env
|
|
|
|
def assert_crosstool_ng_supports_arch(self, arch):
|
|
if arch not in self.env['crosstool_ng_supported_archs']:
|
|
raise Exception('arch not yet supported: ' + arch)
|
|
|
|
@staticmethod
|
|
def base64_encode(string):
|
|
return base64.b64encode(string.encode()).decode()
|
|
|
|
def gem_list_checkpoint_dirs(self):
|
|
'''
|
|
List checkpoint directory, oldest first.
|
|
'''
|
|
prefix_re = re.compile(self.env['gem5_cpt_prefix'])
|
|
files = list(filter(lambda x: os.path.isdir(os.path.join(self.env['m5out_dir'], x)) and prefix_re.search(x), os.listdir(self.env['m5out_dir'])))
|
|
files.sort(key=lambda x: os.path.getmtime(os.path.join(self.env['m5out_dir'], x)))
|
|
return files
|
|
|
|
def get_elf_entry(self, elf_file_path):
|
|
readelf_header = subprocess.check_output([
|
|
self.get_toolchain_tool('readelf'),
|
|
'-h',
|
|
elf_file_path
|
|
])
|
|
for line in readelf_header.decode().split('\n'):
|
|
split = line.split()
|
|
if line.startswith(' Entry point address:'):
|
|
addr = line.split()[-1]
|
|
break
|
|
return int(addr, 0)
|
|
|
|
def get_stats(self, stat_re=None, stats_file=None):
|
|
if stat_re is None:
|
|
stat_re = '^system.cpu[0-9]*.numCycles$'
|
|
if stats_file is None:
|
|
stats_file = self.env['stats_file']
|
|
stat_re = re.compile(stat_re)
|
|
ret = []
|
|
with open(stats_file, 'r') as statfile:
|
|
for line in statfile:
|
|
if line[0] != '-':
|
|
cols = line.split()
|
|
if len(cols) > 1 and stat_re.search(cols[0]):
|
|
ret.append(cols[1])
|
|
return ret
|
|
|
|
def get_toolchain_prefix(self, tool, allowed_toolchains=None):
|
|
buildroot_full_prefix = os.path.join(self.env['host_bin_dir'], self.env['buildroot_toolchain_prefix'])
|
|
buildroot_exists = os.path.exists('{}-{}'.format(buildroot_full_prefix, tool))
|
|
crosstool_ng_full_prefix = os.path.join(self.env['crosstool_ng_bin_dir'], self.env['crosstool_ng_toolchain_prefix'])
|
|
crosstool_ng_exists = os.path.exists('{}-{}'.format(crosstool_ng_full_prefix, tool))
|
|
host_tool = '{}-{}'.format(self.env['ubuntu_toolchain_prefix'], tool)
|
|
host_path = shutil.which(host_tool)
|
|
if host_path is not None:
|
|
host_exists = True
|
|
host_full_prefix = host_path[:-(len(tool)+1)]
|
|
else:
|
|
host_exists = False
|
|
host_full_prefix = None
|
|
known_toolchains = {
|
|
'crosstool-ng': (crosstool_ng_exists, crosstool_ng_full_prefix),
|
|
'buildroot': (buildroot_exists, buildroot_full_prefix),
|
|
'host': (host_exists, host_full_prefix),
|
|
}
|
|
if allowed_toolchains is None:
|
|
if self.env['baremetal'] is None:
|
|
allowed_toolchains = ['buildroot', 'crosstool-ng', 'host']
|
|
else:
|
|
allowed_toolchains = ['crosstool-ng', 'buildroot', 'host']
|
|
tried = []
|
|
for toolchain in allowed_toolchains:
|
|
exists, prefix = known_toolchains[toolchain]
|
|
tried.append('{}-{}'.format(prefix, tool))
|
|
if exists:
|
|
return prefix
|
|
raise Exception('Tool not found. Tried:\n' + '\n'.join(tried))
|
|
|
|
def get_toolchain_tool(self, tool, allowed_toolchains=None):
|
|
return '{}-{}'.format(self.get_toolchain_prefix(tool, allowed_toolchains), tool)
|
|
|
|
@staticmethod
|
|
def github_make_request(
|
|
authenticate=False,
|
|
data=None,
|
|
extra_headers=None,
|
|
path='',
|
|
subdomain='api',
|
|
url_params=None,
|
|
**extra_request_args
|
|
):
|
|
if extra_headers is None:
|
|
extra_headers = {}
|
|
headers = {'Accept': 'application/vnd.github.v3+json'}
|
|
headers.update(extra_headers)
|
|
if authenticate:
|
|
headers['Authorization'] = 'token ' + os.environ['LKMC_GITHUB_TOKEN']
|
|
if url_params is not None:
|
|
path += '?' + urllib.parse.urlencode(url_params)
|
|
request = urllib.request.Request(
|
|
'https://' + subdomain + '.github.com/repos/' + github_repo_id + path,
|
|
headers=headers,
|
|
data=data,
|
|
**extra_request_args
|
|
)
|
|
response_body = urllib.request.urlopen(request).read().decode()
|
|
if response_body:
|
|
_json = json.loads(response_body)
|
|
else:
|
|
_json = {}
|
|
return _json
|
|
|
|
@staticmethod
|
|
def log_error(msg):
|
|
print('error: {}'.format(msg), file=sys.stderr)
|
|
|
|
def main(self, **kwargs):
|
|
'''
|
|
Time the main of the derived class.
|
|
'''
|
|
if not kwargs['dry_run']:
|
|
start_time = time.time()
|
|
kwargs.update(consts)
|
|
self._init_env(kwargs)
|
|
self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
|
|
ret = self.timed_main()
|
|
if not kwargs['dry_run']:
|
|
end_time = time.time()
|
|
self.print_time(end_time - start_time)
|
|
return ret
|
|
|
|
def make_build_dirs(self):
|
|
os.makedirs(self.env['buildroot_build_build_dir'], exist_ok=True)
|
|
os.makedirs(self.env['gem5_build_dir'], exist_ok=True)
|
|
os.makedirs(self.env['out_rootfs_overlay_dir'], exist_ok=True)
|
|
|
|
def make_run_dirs(self):
|
|
'''
|
|
Make directories required for the run.
|
|
The user could nuke those anytime between runs to try and clean things up.
|
|
'''
|
|
os.makedirs(self.env['gem5_run_dir'], exist_ok=True)
|
|
os.makedirs(self.env['p9_dir'], exist_ok=True)
|
|
os.makedirs(self.env['qemu_run_dir'], exist_ok=True)
|
|
|
|
@staticmethod
|
|
def need_rebuild(srcs, dst):
|
|
if not os.path.exists(dst):
|
|
return True
|
|
for src in srcs:
|
|
if os.path.getmtime(src) > os.path.getmtime(dst):
|
|
return True
|
|
return False
|
|
|
|
@staticmethod
|
|
def print_time(ellapsed_seconds):
|
|
hours, rem = divmod(ellapsed_seconds, 3600)
|
|
minutes, seconds = divmod(rem, 60)
|
|
print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds)))
|
|
|
|
def raw_to_qcow2(eslf, prebuilt=False, reverse=False):
|
|
if prebuilt or not os.path.exists(self.env['qemu_img_executable']):
|
|
disable_trace = []
|
|
qemu_img_executable = self.env['qemu_img_basename']
|
|
else:
|
|
# Prevent qemu-img from generating trace files like QEMU. Disgusting.
|
|
disable_trace = ['-T', 'pr_manager_run,file=/dev/null', LF,]
|
|
qemu_img_executable = self.env['qemu_img_executable']
|
|
infmt = 'raw'
|
|
outfmt = 'qcow2'
|
|
infile = self.env['rootfs_raw_file']
|
|
outfile = self.env['qcow2_file']
|
|
if reverse:
|
|
tmp = infmt
|
|
infmt = outfmt
|
|
outfmt = tmp
|
|
tmp = infile
|
|
infile = outfile
|
|
outfile = tmp
|
|
self.sh.run_cmd(
|
|
[
|
|
qemu_img_executable, LF,
|
|
] +
|
|
disable_trace +
|
|
[
|
|
'convert', LF,
|
|
'-f', infmt, LF,
|
|
'-O', outfmt, LF,
|
|
infile, LF,
|
|
outfile, LF,
|
|
]
|
|
)
|
|
|
|
@staticmethod
|
|
def resolve_args(defaults, args, extra_args):
|
|
if extra_args is None:
|
|
extra_args = {}
|
|
argcopy = copy.copy(args)
|
|
argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items()))
|
|
return argcopy
|
|
|
|
@staticmethod
|
|
def resolve_executable(in_path, magic_in_dir, magic_out_dir, out_ext):
|
|
if os.path.isabs(in_path):
|
|
return in_path
|
|
else:
|
|
paths = [
|
|
os.path.join(magic_out_dir, in_path),
|
|
os.path.join(
|
|
magic_out_dir,
|
|
os.path.relpath(in_path, magic_in_dir),
|
|
)
|
|
]
|
|
paths[:] = [os.path.splitext(path)[0] + out_ext for path in paths]
|
|
for path in paths:
|
|
if os.path.exists(path):
|
|
return path
|
|
raise Exception('Executable file not found. Tried:\n' + '\n'.join(paths))
|
|
|
|
def resolve_userland(self, path):
|
|
return self.resolve_executable(
|
|
path,
|
|
self.env['userland_src_dir'],
|
|
self.env['userland_build_dir'],
|
|
self.env['userland_build_ext'],
|
|
)
|
|
|
|
def timed_main(self):
|
|
'''
|
|
Main action of the derived class.
|
|
'''
|
|
raise NotImplementedError()
|
|
|
|
class BuildCliFunction(LkmcCliFunction):
|
|
'''
|
|
A CLI function with common facilities to build stuff, e.g.:
|
|
|
|
* `--clean` to clean the build directory
|
|
* `--nproc` to set he number of build threads
|
|
'''
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.add_argument(
|
|
'--clean',
|
|
default=False,
|
|
help='Clean the build instead of building.',
|
|
),
|
|
self.add_argument(
|
|
'-j',
|
|
'--nproc',
|
|
default=multiprocessing.cpu_count(),
|
|
type=int,
|
|
help='Number of processors to use for the build.',
|
|
)
|
|
|
|
def clean(self):
|
|
build_dir = self.get_build_dir()
|
|
if build_dir is not None:
|
|
self.sh.rmrf(build_dir)
|
|
|
|
def build(self):
|
|
'''
|
|
Do the actual main build work.
|
|
'''
|
|
raise NotImplementedError()
|
|
|
|
def get_build_dir(self):
|
|
return None
|
|
|
|
def timed_main(self):
|
|
'''
|
|
Parse CLI, and to the build based on it.
|
|
|
|
The actual build work is done by do_build in implementing classes.
|
|
'''
|
|
if self.env['clean']:
|
|
return self.clean()
|
|
else:
|
|
return self.build()
|