mirror of
https://github.com/cirosantilli/linux-kernel-module-cheat.git
synced 2026-01-13 20:12:26 +00:00
484 lines
16 KiB
Python
Executable File
484 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import collections
|
|
import platform
|
|
import re
|
|
import os
|
|
|
|
import common
|
|
|
|
class Component:
|
|
'''
|
|
Yes, we are re-inventing a crappy dependency resolution system.
|
|
I can't believe it.
|
|
|
|
The hard part is that we have optional dependencies as well...
|
|
e.g. buildroot optionally depends on m5 to put m5 in the root filesystem,
|
|
and buildroot optionally depends on qemu to build the qcow2 version
|
|
of the image.
|
|
'''
|
|
def __init__(
|
|
self,
|
|
build_callback=None,
|
|
supported_archs=None,
|
|
dependencies=None,
|
|
apt_get_pkgs=None,
|
|
apt_build_deps=None,
|
|
submodules=None,
|
|
submodules_shallow=None,
|
|
python2_pkgs=None,
|
|
python3_pkgs=None,
|
|
):
|
|
self.build_callback = build_callback
|
|
self.supported_archs = supported_archs
|
|
self.dependencies = dependencies or set()
|
|
self.apt_get_pkgs = apt_get_pkgs or set()
|
|
self.apt_build_deps = apt_build_deps or set()
|
|
self.submodules = submodules or set()
|
|
self.submodules_shallow = submodules_shallow or set()
|
|
self.python2_pkgs = python2_pkgs or set()
|
|
self.python3_pkgs = python3_pkgs or set()
|
|
def build(self, arch):
|
|
if (
|
|
(self.build_callback is not None) and
|
|
(self.supported_archs is None or arch in self.supported_archs)
|
|
):
|
|
self.build_callback(arch)
|
|
|
|
def run_cmd(cmd, arch):
|
|
global args
|
|
cmd_abs = cmd.copy()
|
|
cmd_abs[0] = os.path.join(kwargs['root_dir'], cmd[0])
|
|
cmd_abs.extend(['--arch', arch])
|
|
if kwargs['extra_args']:
|
|
cmd_abs.append(kwargs['extra_args'])
|
|
self.sh.run_cmd(cmd_abs, dry_run=kwargs['dry_run'])
|
|
|
|
buildroot_component = Component(
|
|
lambda arch: run_cmd(['build-buildroot'], arch),
|
|
submodules = {'buildroot'},
|
|
# https://buildroot.org/downloads/manual/manual.html#requirement
|
|
apt_get_pkgs={
|
|
'bash',
|
|
'bc',
|
|
'binutils',
|
|
'build-essential',
|
|
'bzip2',
|
|
'cpio',
|
|
'g++',
|
|
'gcc',
|
|
'graphviz',
|
|
'gzip',
|
|
'make',
|
|
'patch',
|
|
'perl',
|
|
'python-matplotlib',
|
|
'python3',
|
|
'rsync',
|
|
'sed',
|
|
'tar',
|
|
'unzip',
|
|
},
|
|
)
|
|
|
|
name_to_component_map = {
|
|
# Leaves without dependencies.
|
|
'baremetal-qemu': Component(
|
|
lambda arch: run_cmd(['build-baremetal', '--qemu'], arch),
|
|
supported_archs=kwargs['crosstool_ng_supported_archs'],
|
|
),
|
|
'baremetal-gem5': Component(
|
|
lambda arch: run_cmd(['build-baremetal', '--gem5'], arch),
|
|
supported_archs=kwargs['crosstool_ng_supported_archs'],
|
|
),
|
|
'baremetal-gem5-pbx': Component(
|
|
lambda arch: run_cmd(['build-baremetal', '--gem5', '--machine', 'RealViewPBX'], arch),
|
|
supported_archs=kwargs['crosstool_ng_supported_archs'],
|
|
),
|
|
'buildroot': buildroot_component,
|
|
'buildroot-gcc': buildroot_component,
|
|
'copy-overlay': Component(
|
|
lambda arch: run_cmd(['copy-overlay'], arch),
|
|
),
|
|
'crosstool-ng': Component(
|
|
lambda arch: run_cmd(['build-crosstool-ng'], arch),
|
|
supported_archs=kwargs['crosstool_ng_supported_archs'],
|
|
# http://crosstool-ng.github.io/docs/os-setup/
|
|
apt_get_pkgs={
|
|
'bison',
|
|
'docbook2x',
|
|
'flex',
|
|
'gawk',
|
|
'gcc',
|
|
'gperf',
|
|
'help2man',
|
|
'libncurses5-dev',
|
|
'libtool-bin',
|
|
'make',
|
|
'python-dev',
|
|
'texinfo',
|
|
},
|
|
submodules={'crosstool-ng'},
|
|
),
|
|
'gem5': Component(
|
|
lambda arch: run_cmd(['build-gem5'], arch),
|
|
# TODO test it out on Docker and answer that question properly:
|
|
# https://askubuntu.com/questions/350475/how-can-i-install-gem5
|
|
apt_get_pkgs={
|
|
'device-tree-compiler',
|
|
'diod',
|
|
'libgoogle-perftools-dev',
|
|
'm4',
|
|
'protobuf-compiler',
|
|
'python-dev',
|
|
'python-pip',
|
|
# For prebuilt qcow2 unpack.
|
|
'qemu-utils',
|
|
'scons',
|
|
'zlib1g-dev',
|
|
},
|
|
python2_pkgs={
|
|
# Generate graphs of config.ini under m5out.
|
|
'pydot',
|
|
},
|
|
submodules={'gem5'},
|
|
),
|
|
'gem5-debug': Component(
|
|
lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'debug'], arch),
|
|
),
|
|
'gem5-fast': Component(
|
|
lambda arch: run_cmd(['build-gem5', '--gem5-build-type', 'fast'], arch),
|
|
),
|
|
'linux': Component(
|
|
lambda arch: run_cmd(['build-linux'], arch),
|
|
submodules_shallow={'linux'},
|
|
apt_get_pkgs={
|
|
'bison',
|
|
'flex',
|
|
# Without this started failing in kernel 4.15 with:
|
|
# Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
|
|
'libelf-dev',
|
|
},
|
|
),
|
|
'modules': Component(
|
|
lambda arch: run_cmd(['build-modules'], arch),
|
|
),
|
|
'm5': Component(
|
|
lambda arch: run_cmd(['build-m5'], arch),
|
|
submodules={'gem5'},
|
|
),
|
|
'qemu': Component(
|
|
lambda arch: run_cmd(['build-qemu'], arch),
|
|
apt_build_deps={'qemu'},
|
|
apt_get_pkgs={'libsdl2-dev'},
|
|
submodules={'qemu'},
|
|
),
|
|
'qemu-user': Component(
|
|
lambda arch: run_cmd(['build-qemu', '--userland'], arch),
|
|
apt_build_deps = {'qemu'},
|
|
apt_get_pkgs={'libsdl2-dev'},
|
|
submodules = {'qemu'},
|
|
),
|
|
'parsec-benchmark': Component(
|
|
submodules = {'parsec-benchmark'},
|
|
),
|
|
'userland': Component(
|
|
lambda arch: run_cmd(['build-userland'], arch),
|
|
),
|
|
|
|
# Dependency only nodes.
|
|
'all': Component(dependencies=[
|
|
'all-linux',
|
|
'all-baremetal',
|
|
]),
|
|
'all-baremetal': Component(dependencies=[
|
|
'qemu-baremetal',
|
|
'gem5-baremetal',
|
|
'baremetal-gem5-pbx',
|
|
],
|
|
supported_archs=kwargs['crosstool_ng_supported_archs'],
|
|
),
|
|
'all-linux': Component(dependencies=[
|
|
'qemu-gem5-buildroot',
|
|
'gem5-debug',
|
|
'gem5-fast',
|
|
'qemu-user',
|
|
]),
|
|
'baremetal': Component(dependencies=[
|
|
'baremetal-gem5',
|
|
'baremetal-qemu',
|
|
]),
|
|
'gem5-buildroot': Component(dependencies=[
|
|
'buildroot-gcc',
|
|
'linux',
|
|
'm5',
|
|
'overlay',
|
|
'gem5',
|
|
]),
|
|
'gem5-baremetal': Component(dependencies=[
|
|
'gem5',
|
|
'crosstool-ng',
|
|
'baremetal-gem5',
|
|
]),
|
|
'overlay': Component(dependencies=[
|
|
'copy-overlay',
|
|
'modules',
|
|
'userland',
|
|
'buildroot',
|
|
]),
|
|
'qemu-baremetal': Component(dependencies=[
|
|
'qemu',
|
|
'crosstool-ng',
|
|
'baremetal-qemu',
|
|
]),
|
|
'qemu-buildroot': Component(dependencies=[
|
|
'qemu',
|
|
'buildroot-gcc',
|
|
'overlay',
|
|
'linux',
|
|
]),
|
|
'qemu-gem5-buildroot': Component(dependencies=[
|
|
'qemu',
|
|
'gem5-buildroot',
|
|
]),
|
|
'release': Component(dependencies=[
|
|
'qemu-buildroot',
|
|
]),
|
|
}
|
|
parser = argparse.ArgumentParser(
|
|
description= '''\
|
|
Shallow helper to build everything, or a subset of everything conveniently.
|
|
|
|
Our build-* scripts don't build any dependencies to make iterative
|
|
development fast and more predictable.
|
|
|
|
While modifying a specific component however, you will likely want to just run the
|
|
individual build-* commands which:
|
|
|
|
* build no dependencies, and so are fast and predictable
|
|
* can take multiple options to custumize the build
|
|
|
|
Without any args, build only what is necessary for
|
|
https://github.com/cirosantilli/linux-kernel-module-cheat#qemu-buildroot-setup
|
|
for x86_64:
|
|
|
|
....
|
|
./%(prog)s
|
|
....
|
|
|
|
This is equivalent to:
|
|
|
|
....
|
|
./%(prog)s --arch x86_64 qemu-buildroot
|
|
....
|
|
|
|
If `--arch` is given, build just for the given archs:
|
|
|
|
....
|
|
./%(prog)s --arch arm --arch aarch64
|
|
....
|
|
|
|
This will build `qemu-buildroot` for arm and aarch64 only, but not `x86_64`.
|
|
|
|
Clean all Linux kernel builds:
|
|
|
|
....
|
|
./build --all-archs --extra-args=--clean buildroot
|
|
....
|
|
''',
|
|
formatter_class=argparse.RawTextHelpFormatter,
|
|
)
|
|
parser.add_argument('--all', default=False, action='store_true', help='''\
|
|
Build absolutely everything for all archs.
|
|
''')
|
|
group = parser.add_mutually_exclusive_group(required=False)
|
|
group.add_argument('-A', '--all-archs', default=False, action='store_true', help='''\
|
|
Build the selected components for all archs.
|
|
''')
|
|
group.add_argument('-a', '--arch', choices=kwargs['arch_choices'], default=[], action='append', help='''\
|
|
Build the selected components for this arch. Select multiple archs by
|
|
passing this option multiple times. Default: [{}]
|
|
'''.format(kwargs['default_arch']))
|
|
parser.add_argument('-D', '--download-dependencies', default=False, action='store_true', help='''\
|
|
Also download all dependencies required for a given build: Ubuntu packages,
|
|
Python packages and git submodules.
|
|
''')
|
|
parser.add_argument('--extra-args', default='', help='''\
|
|
Extra args to pass to all scripts.
|
|
'''
|
|
)
|
|
parser.add_argument('--travis', default=False, action='store_true', help='''\
|
|
Extra args to pass to all scripts.
|
|
'''
|
|
)
|
|
parser.add_argument('components', choices=list(name_to_component_map.keys()) + [[]], default=[], nargs='*', help='''\
|
|
Which components to build. Default: qemu-buildroot
|
|
'''.format(kwargs['default_arch']))
|
|
self.add_dry_run_argument(parser)
|
|
args = parser.parse_args()
|
|
self.setup_dry_run_arguments(args)
|
|
|
|
# Decide archs.
|
|
if kwargs['arch'] == []:
|
|
if kwargs['all'] or kwargs['all_archs']:
|
|
archs = kwargs['all_archs'].copy()
|
|
else:
|
|
archs = set([kwargs['default_arch']])
|
|
else:
|
|
archs = set()
|
|
for arch in kwargs['arch']:
|
|
if arch in kwargs['arch_short_to_long_dict']:
|
|
arch = kwargs['arch_short_to_long_dict'][arch]
|
|
archs.add(arch)
|
|
|
|
# Decide components.
|
|
components = kwargs['components']
|
|
if kwargs['all']:
|
|
components = ['all']
|
|
elif components == []:
|
|
components = ['qemu-buildroot']
|
|
selected_components = []
|
|
selected_component_name_set = set()
|
|
for component_name in components:
|
|
todo = [component_name]
|
|
while todo:
|
|
current_name = todo.pop(0)
|
|
if current_name not in selected_component_name_set:
|
|
selected_component_name_set.add(current_name)
|
|
component = name_to_component_map[current_name]
|
|
selected_components.append(component)
|
|
todo.extend(component.dependencies)
|
|
|
|
if kwargs['download_dependencies']:
|
|
apt_get_pkgs = {
|
|
# Core requirements for this repo.
|
|
'git',
|
|
'moreutils', # ts
|
|
'python3-pip',
|
|
'tmux',
|
|
'vinagre',
|
|
'wget',
|
|
}
|
|
# E.e. on an ARM host, the package gcc-arm-linux-gnueabihf
|
|
# is called just gcc.
|
|
processor = platform.processor()
|
|
if processor != 'arm':
|
|
apt_get_pkgs.update({
|
|
'gcc-arm-linux-gnueabihf',
|
|
'g++-arm-linux-gnueabihf',
|
|
})
|
|
if processor != 'aarch64':
|
|
apt_get_pkgs.update({
|
|
'gcc-aarch64-linux-gnu',
|
|
'g++-aarch64-linux-gnu',
|
|
})
|
|
apt_build_deps = set()
|
|
submodules = set()
|
|
submodules_shallow = set()
|
|
python2_pkgs = set()
|
|
python3_pkgs = {
|
|
'pexpect==4.6.0',
|
|
}
|
|
for component in selected_components:
|
|
apt_get_pkgs.update(component.apt_get_pkgs)
|
|
apt_build_deps.update(component.apt_build_deps)
|
|
submodules.update(component.submodules)
|
|
submodules_shallow.update(component.submodules_shallow)
|
|
python2_pkgs.update(component.python2_pkgs)
|
|
python3_pkgs.update(component.python3_pkgs)
|
|
if apt_get_pkgs or apt_build_deps:
|
|
if kwargs['travis']:
|
|
interacive_pkgs = {
|
|
'libsdl2-dev',
|
|
}
|
|
apt_get_pkgs.difference_update(interacive_pkgs)
|
|
if kwargs['in_docker']:
|
|
sudo = []
|
|
# https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
|
|
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
|
|
# https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
|
|
sources_path = os.path.join('/etc', 'apt', 'sources.list')
|
|
with open(sources_path, 'r') as f:
|
|
sources_txt = f.read()
|
|
sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
|
|
with open(sources_path, 'w') as f:
|
|
f.write(sources_txt)
|
|
else:
|
|
sudo = ['sudo']
|
|
if kwargs['in_docker'] or kwargs['travis']:
|
|
y = ['-y']
|
|
else:
|
|
y = []
|
|
self.sh.run_cmd(
|
|
sudo + ['apt-get', 'update', LF]
|
|
)
|
|
if apt_get_pkgs:
|
|
self.sh.run_cmd(
|
|
sudo + ['apt-get', 'install'] + y + [LF] +
|
|
self.sh.add_newlines(sorted(apt_get_pkgs))
|
|
)
|
|
if apt_build_deps:
|
|
self.sh.run_cmd(
|
|
sudo +
|
|
['apt-get', 'build-dep'] + y + [LF] +
|
|
self.sh.add_newlines(sorted(apt_build_deps))
|
|
)
|
|
if python2_pkgs:
|
|
self.sh.run_cmd(
|
|
['python', '-m', 'pip', 'install', '--user', LF] +
|
|
self.sh.add_newlines(sorted(python2_pkgs))
|
|
)
|
|
if python3_pkgs:
|
|
# Not with pip executable directly:
|
|
# https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
|
|
self.sh.run_cmd(
|
|
['python3', '-m', 'pip', 'install', '--user', LF] +
|
|
self.sh.add_newlines(sorted(python3_pkgs))
|
|
)
|
|
git_cmd_common = ['git', 'submodule', 'update', '--init', '--recursive']
|
|
if submodules:
|
|
# == Other nice git options for when distros move to newer Git
|
|
#
|
|
# Currently not on Ubuntu 16.04:
|
|
#
|
|
# `--progress`: added on Git 2.10:
|
|
#
|
|
# * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
|
|
# * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
|
|
#
|
|
# `--jobs"`: https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
|
|
self.sh.run_cmd(
|
|
git_cmd_common + ['--', LF] +
|
|
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules)])
|
|
)
|
|
if submodules_shallow:
|
|
# == Shallow cloning.
|
|
#
|
|
# TODO Ideally we should shallow clone --depth 1 all of them.
|
|
#
|
|
# However, most git servers out there are crap or craply configured
|
|
# and don't allow shallow cloning except for branches.
|
|
#
|
|
# So for now, let's shallow clone only the Linux kernel, which has by far
|
|
# the largest .git repo history, and full clone the others.
|
|
#
|
|
# Then we will maintain a GitHub Linux kernel mirror / fork that always has a
|
|
# lkmc branch, and point to it, so that it will always succeed.
|
|
#
|
|
# See also:
|
|
#
|
|
# * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
|
|
# * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
|
|
# * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
|
|
#
|
|
self.sh.run_cmd(
|
|
git_cmd_common + ['--depth', '1', '--', LF] +
|
|
self.sh.add_newlines([os.path.join(kwargs['submodules_dir'], x) for x in sorted(submodules_shallow)])
|
|
)
|
|
|
|
# Do the build.
|
|
for arch in archs:
|
|
for component in selected_components:
|
|
component.build(arch)
|