linux-kernel-module-cheat/build-buildroot

#!/usr/bin/env python3

import os
import pathlib
import shutil
import subprocess
import sys
import time
import re

import common

class BuildrootComponent(self.Component):
    def add_parser_arguments(self, parser):
        parser.add_argument(
            '--build-linux', default=self._defaults['build_linux'],
            help='''\
Enable building the Linux kernel with Buildroot. This is done mostly
to extract Buildroot's default kernel configurations when updating Buildroot.
This kernel will not be use by our other scripts. Configuring this kernel is
not currently supported, juse use ./build-linux script if you want to do that.
'''
        )
        parser.add_argument(
            '--baseline', default=self._defaults['baseline'],
            help='''Do a default-ish Buildroot defconfig build, without any of our extra options.
Mostly to track how much slower we are than a basic build.
'''
        )
        parser.add_argument(
            '--config', default=self._defaults['config'], action='append',
            help='''Add a single Buildroot config to the current build.
Example value: 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'.
Can be used multiple times to add multiple configs.
Takes precedence over any Buildroot config files.
'''
        )
        parser.add_argument(
            '--config-fragment', default=self._defaults['config_fragment'], action='append',
            help='''Also use the given Buildroot configuration fragment file.
Pass multiple times to use multiple fragment files.
'''
        )
        parser.add_argument(
            '--no-all', default=self._defaults['no_all'],
            help='''\
Don't build the all target which normally gets build by default.
That target builds the root filesystem and all its dependencies.
'''
        )
        parser.add_argument(
            '--no-overlay', default=self._defaults['no_all'],
            help='''\
Don't add our overlay which contains all files we build without going through Buildroot.
This prevents us from overwriting certain Buildroot files. Remember however that you must
still rebuild the Buildroot package that provides those files to actually put the Buildroot
files on the root filesystem.
'''
        )
        parser.add_argument(
            'extra_make_args', default=self._defaults['extra_make_args'], metavar='extra-make-args', nargs='*',
            help='''\
Extra arguments to be passed to the Buildroot make,
usually extra Buildroot targets.
'''
        )

    def do_build(self, args):
        build_dir = self.get_build_dir(args)
        os.makedirs(kwargs['out_dir'], exist_ok=True)
        extra_make_args = self.sh.add_newlines(kwargs['extra_make_args'])
        if kwargs['build_linux']:
            extra_make_args.extend(['linux-reconfigure', LF])
        if kwargs['emulator'] == 'gem5':
            extra_make_args.extend(['gem5-reconfigure', LF])
        if kwargs['arch'] == 'x86_64':
            defconfig = 'qemu_x86_64_defconfig'
        elif kwargs['arch'] == 'arm':
            defconfig = 'qemu_arm_vexpress_defconfig'
        elif kwargs['arch'] == 'aarch64':
            defconfig = 'qemu_aarch64_virt_defconfig'
        br2_external_dirs = []
        for package_dir in os.listdir(kwargs['packages_dir']):
            package_dir_abs = os.path.join(kwargs['packages_dir'], package_dir)
            if os.path.isdir(package_dir_abs):
                br2_external_dirs.append(self._path_relative_to_buildroot(package_dir_abs))
        br2_external_str = ':'.join(br2_external_dirs)
        self.sh.run_cmd(
            [
                'make', LF,
                'O={}'.format(kwargs['buildroot_build_dir']), LF,
                'BR2_EXTERNAL={}'.format(br2_external_str), LF,
                defconfig, LF,
            ],
            cwd=kwargs['buildroot_src_dir'],
        )
        configs = kwargs['config']
        configs.extend([
            'BR2_JLEVEL={}'.format(kwargs['nproc']),
            'BR2_DL_DIR="{}"'.format(kwargs['buildroot_download_dir']),
        ])
        if not kwargs['build_linux']:
            configs.extend([
                '# BR2_LINUX_KERNEL is not set',
            ])
        config_fragments = []
        if not kwargs['baseline']:
            configs.extend([
                'BR2_GLOBAL_PATCH_DIR="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'patches', 'global'))
                ),
                'BR2_PACKAGE_BUSYBOX_CONFIG_FRAGMENT_FILES="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'busybox_config_fragment'))
                ),
                'BR2_PACKAGE_OVERRIDE_FILE="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'buildroot_override'))
                ),
                'BR2_ROOTFS_POST_BUILD_SCRIPT="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'rootfs-post-build-script'))
                ),
                'BR2_ROOTFS_USERS_TABLES="{}"'.format(
                    self._path_relative_to_buildroot(os.path.join(kwargs['root_dir'], 'user_table'))
                ),
            ])
            if not kwargs['no_overlay']:
                configs.append('BR2_ROOTFS_OVERLAY="{}"'.format(
                    self._path_relative_to_buildroot(kwargs['out_rootfs_overlay_dir'])
                ))
            config_fragments = [
                os.path.join(kwargs['root_dir'], 'buildroot_config', 'default')
            ] + kwargs['config_fragment']
        # TODO Can't get rid of these for now with nice fragments on Buildroot:
        # http://stackoverflow.com/questions/44078245/is-it-possible-to-use-config-fragments-with-buildroots-config
        self.sh.write_configs(kwargs['buildroot_config_file'], configs, config_fragments)
        self.sh.run_cmd(
            [
                'make', LF,
                'O={}'.format(kwargs['buildroot_build_dir']), LF,
                'olddefconfig', LF,
            ],
            cwd=kwargs['buildroot_src_dir'],
        )
        self.make_build_dirs()
        if not kwargs['no_all']:
            extra_make_args.extend(['all', LF])
        self.sh.run_cmd(
            [
                'make', LF,
                'LKMC_GEM5_SRCDIR="{}"'.format(kwargs['gem5_source_dir']), LF,
                'LKMC_PARSEC_BENCHMARK_SRCDIR="{}"'.format(kwargs['parsec_benchmark_src_dir']), LF,
                'O={}'.format(kwargs['buildroot_build_dir']), LF,
                'V={}'.format(int(kwargs['verbose'])), LF,
            ] +
            extra_make_args
            ,
            out_file=os.path.join(kwargs['buildroot_build_dir'], 'lkmc.log'),
            delete_env=['LD_LIBRARY_PATH'],
            cwd=kwargs['buildroot_src_dir'],
        )
        # Create the qcow2 from ext2.
        # Skip if qemu is not present, because gem5 does not need the qcow2.
        # so we don't force a QEMU build for gem5.
        if not kwargs['no_all'] and os.path.exists(kwargs['qemu_img_executable']):
            self.raw_to_qcow2()

    def get_argparse_args(self):
        return {
            'description': '''\
Run Linux on an emulator
'''
        }

    def get_build_dir(self, args):
        return kwargs['buildroot_build_dir']

    _defaults = {
        'baseline': False,
        'build_linux': False,
        'config': [],
        'config_fragment': [],
        'extra_make_args': [],
        'no_all': False,
        'skip_configure': False,
    }

    def _path_relative_to_buildroot(self, abspath):
        return os.path.relpath(abspath, kwargs['buildroot_src_dir'])

if __name__ == '__main__':
    BuildrootComponent().build()