From fd55d694c06a3be697b4e7ee256954dba775dfba Mon Sep 17 00:00:00 2001 From: Ciro Santilli Date: Thu, 23 Aug 2018 09:25:21 +0100 Subject: [PATCH] lkmc v2-rc Unsquashed version at v2-rc-unsquashed, but that cannot be merged as it breaks bisects at several points. All bugs will not bisect to this humongous change. It all started with a conversion of the Bash scripts to Python, mainly because I couldn't stand not being able to properly use --options for run which has a million options. Then since that required a full testing, I decided to do all the refactorings that I had in mind at once, and so I did and it became v2-rc. This is the largest patch I have ever done! OMG a few weeks of extra time. I'm never writing a Bash script for anything that starts getting big again. Some of the features are: * separate build-qemu and build-gem5 commands * common: convert scripts to python. Add --option for everything * rename build to build-buildroot now that we are splitting all the build commands, Linux kernel to follow * move all git submodules to submodules/ and all buildroot packages to packages/ * refactor the out/ structure. Keep projects on toplevel, because guest projects separate archs and host ones don't, making a toplevel arch wrong * do-release: rename to just release https://stackoverflow.com/questions/16174992/cant-get-argparse-to-read-quoted-string-with-dashes-in-it * run: add --terminal and explain gem5 pdb * just track the lvimrc * store CLI kernel config fragment inside buildlroot to avoid conflicts * gem5: document m5 initparam * readme: make a bunch of things awesomer * readme: fix broken refs * parsec-benchmark: update to 75d55ac446a43c47efb1044844a108c6c330184c Could not fetch otherwise. * gem5: M5_OVERRIDE_PY_SOURCE --- .gitignore | 4 + .gitmodules | 24 +- .lvimrc | 9 + .travis.yml | 1 + README.adoc | 4299 +++++++++-------- bench-all | 124 +- bench-boot | 84 +- bench-cmd | 17 + bisect-linux-boot-gem5 | 34 + bisect-qemu-linux-boot | 6 + br2/qemu | 6 - bst-vs-heap | 28 +- build | 223 - build-all | 10 +- build-buildroot | 324 ++ build-gem5 | 83 + build-qemu | 41 + {br2 => buildroot_config}/default | 82 +- {br2 => buildroot_config}/gpio | 0 {br2 => buildroot_config}/kmscube | 0 {br2 => buildroot_config}/x11 | 0 buildroot_override | 7 +- common | 204 - common.py | 395 ++ config.example | 7 +- configure | 14 +- eeval | 3 +- gem5-bench-cache | 67 +- gem5-bench-dhrystone | 26 + gem5-shell | 26 +- gem5-stat | 39 +- gem5/Config.in | 7 - gem5/build | 61 - getvar | 37 +- .../Makefile | 0 .../hello.c | 0 .../default | 31 +- .../display | 0 .../gpio | 0 .../leds | 0 {kernel_config_fragment => kernel_config}/min | 2 +- kernel_module/external.desc | 1 - linux-bisect-boot-gem5 | 28 - {gem5 => packages/gem5}/.gitignore | 0 packages/gem5/Config.in | 7 + {gem5 => packages/gem5}/README.adoc | 0 {gem5 => packages/gem5}/external.desc | 0 {gem5 => packages/gem5}/external.mk | 9 +- .../kernel_modules}/Config.in | 4 +- .../kernel_modules}/Makefile | 2 +- .../kernel_modules}/README.adoc | 0 .../kernel_modules}/anonymous_inode.c | 0 .../kernel_modules}/anonymous_inode.h | 0 .../kernel_modules}/character_device.c | 0 .../kernel_modules}/character_device_create.c | 0 .../kernel_modules}/debugfs.c | 0 .../kernel_modules}/dep.c | 0 .../kernel_modules}/dep2.c | 0 .../kernel_modules}/dump_stack.c | 0 packages/kernel_modules/external.desc | 1 + .../kernel_modules}/external.mk | 18 +- .../kernel_modules}/fops.c | 0 .../kernel_modules}/hello.c | 0 .../kernel_modules}/hello2.c | 0 .../kernel_modules}/init_module.c | 0 .../kernel_modules}/ioctl.c | 0 .../kernel_modules}/ioctl.h | 0 .../kernel_modules}/irq.c | 0 .../kernel_modules}/kprobe_example.c | 0 .../kernel_modules}/kstrto.c | 0 .../kernel_modules}/kthread.c | 0 .../kernel_modules}/kthreads.c | 0 .../kernel_modules}/make-host.sh | 4 +- .../kernel_modules}/memcpy_overflow.c | 0 .../kernel_modules}/mmap.c | 0 .../kernel_modules}/module_info.c | 0 .../kernel_modules}/myprintk.c | 0 .../kernel_modules}/netlink.c | 0 .../kernel_modules}/netlink.h | 0 .../kernel_modules}/oops.c | 0 .../kernel_modules}/panic.c | 0 .../kernel_modules}/params.c | 0 .../kernel_modules}/pci_min.c | 0 .../kernel_modules}/pmccntr.c | 0 .../kernel_modules}/poll.c | 0 .../kernel_modules}/procfs.c | 0 .../kernel_modules}/qemu_edu.c | 0 .../kernel_modules}/ring0.c | 0 .../kernel_modules}/ring0.h | 0 .../kernel_modules}/schedule.c | 0 .../kernel_modules}/seq_file.c | 0 .../kernel_modules}/seq_file_single_open.c | 0 .../kernel_modules}/sleep.c | 0 .../kernel_modules}/strlen_overflow.c | 0 .../kernel_modules}/sysfs.c | 0 .../kernel_modules}/timer.c | 0 .../kernel_modules}/user/Makefile | 0 .../kernel_modules}/user/README.adoc | 0 .../kernel_modules}/user/anonymous_inode.c | 0 .../kernel_modules}/user/bst_vs_heap.cpp | 0 .../kernel_modules}/user/common.h | 0 .../kernel_modules}/user/ctrl_alt_del.c | 0 .../kernel_modules}/user/eigen_hello.cpp | 0 .../kernel_modules}/user/hello.c | 0 .../kernel_modules}/user/hello_cpp.cpp | 0 .../kernel_modules/user/init_env_poweroff.c | 26 + .../kernel_modules}/user/ioctl.c | 0 .../kernel_modules}/user/libdrm_modeset.c | 0 .../kernel_modules}/user/m5ops.c | 0 .../kernel_modules}/user/m5ops.h | 0 .../kernel_modules}/user/mmap.c | 0 .../kernel_modules}/user/myinsmod.c | 0 .../kernel_modules}/user/myrmmod.c | 0 .../kernel_modules}/user/netlink.c | 0 .../kernel_modules}/user/openblas.c | 0 .../kernel_modules}/user/openmp.c | 0 .../kernel_modules}/user/pagemap_dump.c | 0 .../kernel_modules}/user/poll.c | 0 .../kernel_modules}/user/poweroff.c | 2 - .../kernel_modules}/user/proc_events.c | 0 .../kernel_modules}/user/rand_check.c | 0 .../kernel_modules}/user/rdtsc.c | 0 .../kernel_modules}/user/ring0.c | 0 .../kernel_modules}/user/sched_getaffinity.c | 0 .../user/sched_getaffinity_threads.c | 0 .../kernel_modules}/user/sleep_forever.c | 0 .../kernel_modules}/user/time_boot.c | 0 .../kernel_modules}/user/uio_read.c | 0 .../kernel_modules}/user/virt_to_phys_test.c | 0 .../kernel_modules}/user/virt_to_phys_user.c | 0 .../kernel_modules}/vermagic.c | 0 .../kernel_modules}/vermagic_fail.c | 0 .../kernel_modules}/virt_to_phys.c | 0 .../kernel_modules}/wait_queue.c | 0 .../kernel_modules}/wait_queue2.c | 0 .../kernel_modules}/warn_on.c | 0 .../kernel_modules}/work_from_work.c | 0 .../kernel_modules}/workqueue_cheat.c | 0 .../parsec_benchmark}/Config.in | 0 .../parsec_benchmark}/external.desc | 0 .../parsec_benchmark}/external.mk | 13 +- parsec-benchmark/parsec-benchmark | 1 - qemu-bisect-boot | 6 - qemu-trace2txt | 41 +- do-release => release | 20 +- rootfs_overlay/gem5.sh | 2 +- rootfs_overlay/init_env_poweroff.sh | 4 - rootfs_post_build_script | 2 - rootfs_post_image_script | 10 - run | 853 ++-- rungdb | 195 +- rungdb-user | 67 +- rungdbserver | 46 +- runtc | 66 +- buildroot => submodules/buildroot | 0 {gem5 => submodules}/gem5 | 0 linux => submodules/linux | 0 submodules/parsec-benchmark | 1 + qemu => submodules/qemu | 0 trace-boot | 88 +- trace2line | 84 +- trace2line.sh | 23 + zip-img | 27 +- 163 files changed, 4520 insertions(+), 3456 deletions(-) create mode 100644 .lvimrc create mode 100755 bench-cmd create mode 100755 bisect-linux-boot-gem5 create mode 100755 bisect-qemu-linux-boot delete mode 100644 br2/qemu delete mode 100755 build create mode 100755 build-buildroot create mode 100755 build-gem5 create mode 100755 build-qemu rename {br2 => buildroot_config}/default (65%) rename {br2 => buildroot_config}/gpio (100%) rename {br2 => buildroot_config}/kmscube (100%) rename {br2 => buildroot_config}/x11 (100%) delete mode 100644 common create mode 100644 common.py create mode 100755 gem5-bench-dhrystone delete mode 100644 gem5/Config.in delete mode 100755 gem5/build rename {hello_host => hello_host_kernel_module}/Makefile (100%) rename {hello_host => hello_host_kernel_module}/hello.c (100%) rename {kernel_config_fragment => kernel_config}/default (99%) rename {kernel_config_fragment => kernel_config}/display (100%) rename {kernel_config_fragment => kernel_config}/gpio (100%) rename {kernel_config_fragment => kernel_config}/leds (100%) rename {kernel_config_fragment => kernel_config}/min (95%) delete mode 100644 kernel_module/external.desc delete mode 100755 linux-bisect-boot-gem5 rename {gem5 => packages/gem5}/.gitignore (100%) create mode 100644 packages/gem5/Config.in rename {gem5 => packages/gem5}/README.adoc (100%) rename {gem5 => packages/gem5}/external.desc (100%) rename {gem5 => packages/gem5}/external.mk (54%) rename {kernel_module => packages/kernel_modules}/Config.in (52%) rename {kernel_module => packages/kernel_modules}/Makefile (84%) rename {kernel_module => packages/kernel_modules}/README.adoc (100%) rename {kernel_module => packages/kernel_modules}/anonymous_inode.c (100%) rename {kernel_module => packages/kernel_modules}/anonymous_inode.h (100%) rename {kernel_module => packages/kernel_modules}/character_device.c (100%) rename {kernel_module => packages/kernel_modules}/character_device_create.c (100%) rename {kernel_module => packages/kernel_modules}/debugfs.c (100%) rename {kernel_module => packages/kernel_modules}/dep.c (100%) rename {kernel_module => packages/kernel_modules}/dep2.c (100%) rename {kernel_module => packages/kernel_modules}/dump_stack.c (100%) create mode 100644 packages/kernel_modules/external.desc rename {kernel_module => packages/kernel_modules}/external.mk (73%) rename {kernel_module => packages/kernel_modules}/fops.c (100%) rename {kernel_module => packages/kernel_modules}/hello.c (100%) rename {kernel_module => packages/kernel_modules}/hello2.c (100%) rename {kernel_module => packages/kernel_modules}/init_module.c (100%) rename {kernel_module => packages/kernel_modules}/ioctl.c (100%) rename {kernel_module => packages/kernel_modules}/ioctl.h (100%) rename {kernel_module => packages/kernel_modules}/irq.c (100%) rename {kernel_module => packages/kernel_modules}/kprobe_example.c (100%) rename {kernel_module => packages/kernel_modules}/kstrto.c (100%) rename {kernel_module => packages/kernel_modules}/kthread.c (100%) rename {kernel_module => packages/kernel_modules}/kthreads.c (100%) rename {kernel_module => packages/kernel_modules}/make-host.sh (81%) rename {kernel_module => packages/kernel_modules}/memcpy_overflow.c (100%) rename {kernel_module => packages/kernel_modules}/mmap.c (100%) rename {kernel_module => packages/kernel_modules}/module_info.c (100%) rename {kernel_module => packages/kernel_modules}/myprintk.c (100%) rename {kernel_module => packages/kernel_modules}/netlink.c (100%) rename {kernel_module => packages/kernel_modules}/netlink.h (100%) rename {kernel_module => packages/kernel_modules}/oops.c (100%) rename {kernel_module => packages/kernel_modules}/panic.c (100%) rename {kernel_module => packages/kernel_modules}/params.c (100%) rename {kernel_module => packages/kernel_modules}/pci_min.c (100%) rename {kernel_module => packages/kernel_modules}/pmccntr.c (100%) rename {kernel_module => packages/kernel_modules}/poll.c (100%) rename {kernel_module => packages/kernel_modules}/procfs.c (100%) rename {kernel_module => packages/kernel_modules}/qemu_edu.c (100%) rename {kernel_module => packages/kernel_modules}/ring0.c (100%) rename {kernel_module => packages/kernel_modules}/ring0.h (100%) rename {kernel_module => packages/kernel_modules}/schedule.c (100%) rename {kernel_module => packages/kernel_modules}/seq_file.c (100%) rename {kernel_module => packages/kernel_modules}/seq_file_single_open.c (100%) rename {kernel_module => packages/kernel_modules}/sleep.c (100%) rename {kernel_module => packages/kernel_modules}/strlen_overflow.c (100%) rename {kernel_module => packages/kernel_modules}/sysfs.c (100%) rename {kernel_module => packages/kernel_modules}/timer.c (100%) rename {kernel_module => packages/kernel_modules}/user/Makefile (100%) rename {kernel_module => packages/kernel_modules}/user/README.adoc (100%) rename {kernel_module => packages/kernel_modules}/user/anonymous_inode.c (100%) rename {kernel_module => packages/kernel_modules}/user/bst_vs_heap.cpp (100%) rename {kernel_module => packages/kernel_modules}/user/common.h (100%) rename {kernel_module => packages/kernel_modules}/user/ctrl_alt_del.c (100%) rename {kernel_module => packages/kernel_modules}/user/eigen_hello.cpp (100%) rename {kernel_module => packages/kernel_modules}/user/hello.c (100%) rename {kernel_module => packages/kernel_modules}/user/hello_cpp.cpp (100%) create mode 100644 packages/kernel_modules/user/init_env_poweroff.c rename {kernel_module => packages/kernel_modules}/user/ioctl.c (100%) rename {kernel_module => packages/kernel_modules}/user/libdrm_modeset.c (100%) rename {kernel_module => packages/kernel_modules}/user/m5ops.c (100%) rename {kernel_module => packages/kernel_modules}/user/m5ops.h (100%) rename {kernel_module => packages/kernel_modules}/user/mmap.c (100%) rename {kernel_module => packages/kernel_modules}/user/myinsmod.c (100%) rename {kernel_module => packages/kernel_modules}/user/myrmmod.c (100%) rename {kernel_module => packages/kernel_modules}/user/netlink.c (100%) rename {kernel_module => packages/kernel_modules}/user/openblas.c (100%) rename {kernel_module => packages/kernel_modules}/user/openmp.c (100%) rename {kernel_module => packages/kernel_modules}/user/pagemap_dump.c (100%) rename {kernel_module => packages/kernel_modules}/user/poll.c (100%) rename {kernel_module => packages/kernel_modules}/user/poweroff.c (84%) rename {kernel_module => packages/kernel_modules}/user/proc_events.c (100%) rename {kernel_module => packages/kernel_modules}/user/rand_check.c (100%) rename {kernel_module => packages/kernel_modules}/user/rdtsc.c (100%) rename {kernel_module => packages/kernel_modules}/user/ring0.c (100%) rename {kernel_module => packages/kernel_modules}/user/sched_getaffinity.c (100%) rename {kernel_module => packages/kernel_modules}/user/sched_getaffinity_threads.c (100%) rename {kernel_module => packages/kernel_modules}/user/sleep_forever.c (100%) rename {kernel_module => packages/kernel_modules}/user/time_boot.c (100%) rename {kernel_module => packages/kernel_modules}/user/uio_read.c (100%) rename {kernel_module => packages/kernel_modules}/user/virt_to_phys_test.c (100%) rename {kernel_module => packages/kernel_modules}/user/virt_to_phys_user.c (100%) rename {kernel_module => packages/kernel_modules}/vermagic.c (100%) rename {kernel_module => packages/kernel_modules}/vermagic_fail.c (100%) rename {kernel_module => packages/kernel_modules}/virt_to_phys.c (100%) rename {kernel_module => packages/kernel_modules}/wait_queue.c (100%) rename {kernel_module => packages/kernel_modules}/wait_queue2.c (100%) rename {kernel_module => packages/kernel_modules}/warn_on.c (100%) rename {kernel_module => packages/kernel_modules}/work_from_work.c (100%) rename {kernel_module => packages/kernel_modules}/workqueue_cheat.c (100%) rename {parsec-benchmark => packages/parsec_benchmark}/Config.in (100%) rename {parsec-benchmark => packages/parsec_benchmark}/external.desc (100%) rename {parsec-benchmark => packages/parsec_benchmark}/external.mk (82%) delete mode 160000 parsec-benchmark/parsec-benchmark delete mode 100755 qemu-bisect-boot rename do-release => release (59%) mode change 100755 => 100644 delete mode 100755 rootfs_overlay/init_env_poweroff.sh delete mode 100755 rootfs_post_image_script rename buildroot => submodules/buildroot (100%) rename {gem5 => submodules}/gem5 (100%) rename linux => submodules/linux (100%) create mode 160000 submodules/parsec-benchmark rename qemu => submodules/qemu (100%) create mode 100755 trace2line.sh diff --git a/.gitignore b/.gitignore index 3e05d6e3..ea666c73 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ gitignore* /images-*.zip /out /out.* + +# Python trash. +*.pyc +__pycache__ diff --git a/.gitmodules b/.gitmodules index e9d1c793..a858b9a9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,16 +1,20 @@ -[submodule "buildroot"] - path = buildroot +[submodule "submodules/buildroot"] + path = submodules/buildroot url = https://github.com/cirosantilli/buildroot ignore = dirty -[submodule "qemu"] - path = qemu +[submodule "submodules/qemu"] + path = submodules/qemu url = https://github.com/cirosantilli/qemu -[submodule "linux"] - path = linux +# The true upstream does not accept git submodule update --init --depth 1 +# git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git +# But git clone --branch --depth 1 worked weirdly: +# https://unix.stackexchange.com/questions/338578/linux-kernel-source-code-size-difference +[submodule "submodules/linux"] + path = submodules/linux url = https://github.com/cirosantilli/linux -[submodule "gem5/gem5"] - path = gem5/gem5 +[submodule "submodules/gem5"] + path = submodules/gem5 url = https://gem5.googlesource.com/public/gem5 -[submodule "parsec-benchmark/parsec-benchmark"] - path = parsec-benchmark/parsec-benchmark +[submodule "submodules/parsec-benchmark"] + path = submodules/parsec-benchmark url = https://github.com/cirosantilli/parsec-benchmark diff --git a/.lvimrc b/.lvimrc new file mode 100644 index 00000000..f5259378 --- /dev/null +++ b/.lvimrc @@ -0,0 +1,9 @@ +" Use this automatically with: +" +" Plugin 'embear/vim-localvimrc' +" let g:localvimrc_ask = 0 +" let g:localvimrc_sandbox = 0 + +if &filetype ==# 'c' || &filetype ==# 'cpp' + setlocal noexpandtab +endif diff --git a/.travis.yml b/.travis.yml index bb24335a..2b979eac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,5 +21,6 @@ script: | # The following packages have unmet dependencies: # libsdl2-dev : Depends: libegl1-mesa-dev # Depends: libgles2-mesa-dev + bash -x ./build-qemu -j 16 -S |& awk 'NR % 1000 == 0' bash -x ./build -j 16 -S |& awk 'NR % 1000 == 0' bash -x ./run -e 'init=/poweroff.out' diff --git a/README.adoc b/README.adoc index 52db887a..7dea07cb 100644 --- a/README.adoc +++ b/README.adoc @@ -9,34 +9,115 @@ :toclevels: 6 :toc-title: -Run one command, get a QEMU or gem5 Buildroot BusyBox virtual machine built from source with several minimal Linux kernel 4.18 module development example tutorials with GDB and KGDB step debugging and minimal educational hardware device models. "Tested" in x86, ARM and MIPS guests, Ubuntu 18.04 host. +Run one command, get a QEMU or gem5 Buildroot BusyBox virtual machine built from source with several minimal Linux kernel 4.18 module development example tutorials with <> and <> step debugging and minimal educational hardware device models. "Tested" in Ubuntu 18.04 host, x86 and ARM guests. + +TL;DR: <> toc::[] +== About + +This project was created to help me understand, modify and test low level system components by using system simulators. + +System simulators are cool compared to real hardware because they are: + +* free +* make experiments highly reproducible +* give full visibility to the system: you can inspect any byte in memory, or the state of any hardware register. The laws of physics sometimes get in the way when doing that for real hardware. + +The current components we focus the most on are: + +* Linux kernel and Linux kernel modules +* full systems emulators, currently <> and <> +* <>. We use and therefore document, a large part of its feature set. + +The following components are not covered, but it shouldn't be hard to do: + +* C standard libraries +* compilers. Project idea: add a new instruction to x86, then hack up GCC to actually use it, and make a C program that generates it. + +The design goals are to provide setups that are: + +* highly automated: "just works" +* thoroughly documented: you know what "just works" means +* can be fully built from source: to give visibility and allow modifications +* can also use prebuilt binaries as much as possible: in case you are lazy or unable to build from source + +=== Fairy tale + +____ +Once upon a time, there was a boy called Linus. + +Linus made a super fun toy, and since he was not very humble, decided to call it Linux. + +Linux was an awesome toy, but it had one big problem: it was very difficult to learn how to play with it! + +As a result, only some weird kids who were very bored ended up playing with Linux, and everyone thought those kids were very cool, in their own weird way. + +One day, a mysterious new kid called Ciro tried to play with Linux, and like many before him, got very frustrated, and gave up. + +A few years later, Ciro had grown up a bit, and by chance came across a very cool toy made by the boy Petazzoni and his gang: it was called Buildroot. + +Ciro noticed that if you used Buildroot together with Linux, and Linux suddenly became very fun to play with! + +So Ciro decided to explain to as many kids as possible how to use Buildroot to play with Linux. + +And so everyone was happy. Except some of the old weird kernel hackers who wanted to keep their mystique, but so be it. + +THE END +____ + == Getting started -=== Setup types +There are several different possible setups to use this repo. -==== Getting started natively +Each child section of this section describes one of those setups, and the trade-offs of each. -This is the best setup if you are on one of the supported systems: Ubuntu 16.04 or Ubuntu 18.04. +If you don't know which one to go for, start with <>. -Everything will likely also work on other Linux distros if you install the analogous required packages for your distro from link:configure[], but this is not currently well tested. Compatibility patches are welcome. You can also try <> if you are on other distros. +The trade-offs are basically a balance between: + +* speed ans size; how long and how much disk space do the build and run take? +* visibility: can you GDB step debug everything and read source code? +* modifiability: can you modify the source code and rebuild a modified version? +* portability: does it work on a Windows host? Could it ever? +* accuracy: how accurate does the simulation represent real hardware? +* compatibility: how likely is is that all the components will work well together: emulator, compiler, kernel, standard library, ... +* guest software availability: how wide is your choice of easily installed guest software packages? + +=== QEMU Buildroot setup + +This is the best setup if you are on Ubuntu. We tend to test this repo the most on the latest Ubuntu, and on the latest Ubuntu LTS. + +Everything will likely also work on other Linux distros if you install the analogous required packages for your distro from link:configure[], but this is not currently well tested. Compatibility patches are welcome. You can also try <> if you are on other Linux distros. Native Windows is unlikely feasible because Buildroot is a huge set of GNU Make scripts + host tools, just use an Ubuntu VM in that case. Reserve 12Gb of disk and run: .... git clone https://github.com/cirosantilli/linux-kernel-module-cheat cd linux-kernel-module-cheat -./configure && ./build && ./run +./configure && \ + ./build-qemu && + ./build-buildroot && \ + ./run && \ +:; .... -It does not work if you just download the .zip from GitHub because we use Git submodules, you must clone this repo. `./configure` then fetches only the required submodules for you. - -It is also trivial to build for different supported <>. - The first configure will take a while (30 minutes to 2 hours) to clone and build, see <> for more details. +It does not work if you just download the `.zip` from GitHub because we use link:.gitmodules[Git submodules], you must clone this repo. `./configure` then fetches only the required submodules for you. + +It is super easy to build for different CPU architectures, just use the `--arch` option: + +.... +./build-qemu --arch arm && \ + ./build-buildroot --arch arm && \ + ./run --arch arm && \ +:; +.... + +See also: <>. + If you don't want to wait, you could also try the following faster but much more limited methods: * <> @@ -64,44 +145,57 @@ hello2 cleanup which are `printk` messages from `init` and `cleanup` methods of those modules. +Sources: + +* link:packages/kernel_modules/hello.c[] +* link:packages/kernel_modules/hello2.c[] + Quit QEMU with: .... Ctrl-A X .... -See also: <>. +See also: <>. -Source: +All available modules can be found in the link:packages/kernel_modules/[`kernel_modules` directory]. -* link:kernel_module/hello.c[] -* link:kernel_module/hello2.c[] +I now urge you to read the following sections which contain widely applicable information: + +* <> +* <> +* <> +* <> +* <> +* Linux kernel +** <> +** <> Once you use <> and <>, your terminal will look a bit like this: .... -[ 1.451857] input: AT Translated Set 2 keyboard as /devices/platform/i8042/s1│loading @0xffffffffc0000000: ../kernel_module-1.0//timer.ko +[ 1.451857] input: AT Translated Set 2 keyboard as /devices/platform/i8042/s1│loading @0xffffffffc0000000: ../kernel_modules-1.0//timer.ko [ 1.454310] ledtrig-cpu: registered to indicate activity on CPUs │(gdb) b lkmc_timer_callback [ 1.455621] usbcore: registered new interface driver usbhid │Breakpoint 1 at 0xffffffffc0000000: file /home/ciro/bak/git/linux-kernel-module -[ 1.455811] usbhid: USB HID core driver │-cheat/out/x86_64/buildroot/build/kernel_module-1.0/./timer.c, line 28. +[ 1.455811] usbhid: USB HID core driver │-cheat/out/x86_64/buildroot/build/kernel_modules-1.0/./timer.c, line 28. [ 1.462044] NET: Registered protocol family 10 │(gdb) c [ 1.467911] Segment Routing with IPv6 │Continuing. [ 1.468407] sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver │ [ 1.470859] NET: Registered protocol family 17 │Breakpoint 1, lkmc_timer_callback (data=0xffffffffc0002000 ) [ 1.472017] 9pnet: Installing 9P2000 support │ at /linux-kernel-module-cheat//out/x86_64/buildroot/build/ -[ 1.475461] sched_clock: Marking stable (1473574872, 0)->(1554017593, -80442)│kernel_module-1.0/./timer.c:28 +[ 1.475461] sched_clock: Marking stable (1473574872, 0)->(1554017593, -80442)│kernel_modules-1.0/./timer.c:28 [ 1.479419] ALSA device list: │28 { [ 1.479567] No soundcards found. │(gdb) c [ 1.619187] ata2.00: ATAPI: QEMU DVD-ROM, 2.5+, max UDMA/100 │Continuing. [ 1.622954] ata2.00: configured for MWDMA2 │ [ 1.644048] scsi 1:0:0:0: CD-ROM QEMU QEMU DVD-ROM 2.5+ P5│Breakpoint 1, lkmc_timer_callback (data=0xffffffffc0002000 ) [ 1.741966] tsc: Refined TSC clocksource calibration: 2904.010 MHz │ at /linux-kernel-module-cheat//out/x86_64/buildroot/build/ -[ 1.742796] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x29dc0f4s│kernel_module-1.0/./timer.c:28 +[ 1.742796] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x29dc0f4s│kernel_modules-1.0/./timer.c:28 [ 1.743648] clocksource: Switched to clocksource tsc │28 { [ 2.072945] input: ImExPS/2 Generic Explorer Mouse as /devices/platform/i8043│(gdb) bt [ 2.078641] EXT4-fs (vda): couldn't mount as ext3 due to feature incompatibis│#0 lkmc_timer_callback (data=0xffffffffc0002000 ) [ 2.080350] EXT4-fs (vda): mounting ext2 file system using the ext4 subsystem│ at /linux-kernel-module-cheat//out/x86_64/buildroot/build/ -[ 2.088978] EXT4-fs (vda): mounted filesystem without journal. Opts: (null) │kernel_module-1.0/./timer.c:28 +[ 2.088978] EXT4-fs (vda): mounted filesystem without journal. Opts: (null) │kernel_modules-1.0/./timer.c:28 [ 2.089872] VFS: Mounted root (ext2 filesystem) readonly on device 254:0. │#1 0xffffffff810ab494 in call_timer_fn (timer=0xffffffffc0002000 , [ 2.097168] devtmpfs: mounted │ fn=0xffffffffc0000000 ) at kernel/time/timer.c:1326 [ 2.126472] Freeing unused kernel memory: 1264K │#2 0xffffffff810ab71f in expire_timers (head=, @@ -121,17 +215,181 @@ hello /root/.profile │(gdb) .... -All available modules can be found in the link:kernel_module/[`kernel_module` directory]. +==== Your first kernel hack -We will try to support the following Ubuntu versions at least: +Modify link:packages/kernel_modules/hello.c[] to contain: -* if the latest release is an LTS, support both the latest LTS and the previous one -* otherwise, support both latest LTS and the latest non-LTS +.... +pr_info("hello init hacked\n"); +.... + +and then rebuild the kernel modules and re-run to see it take effect: + +.... +./build-buildroot --kernel-modules +./run --eval-busybox 'insmod /hello.ko' +.... + +Congratulations, you are now officially a kernel module hacker! + +We use `./build-buildroot` here because kernel modules go into the root filesystem, and it is Buildroot that generates our root filesystem. The kernel modules are inside a Buildroot package. + +`--kernel-modules` is required even if files were modified as explained at: <>. + +The reboot after rebuild is annoying. We don't have a perfect solution for it yet, but there are some ideas cooking at: <>. + +Not satisfied with kernel modules? OK then, let's hack up the <> Linux kernel itself. + +Open the file: + +.... +vim submodules/linux/init/main.c +.... + +and find the `start_kernel` function, then add there a: + +.... +pr_info("I'VE HACKED THE LINUX KERNEL!!!"); +.... + +Then rebuild the Linux kernel and reboot: + +.... +./build-buildroot && ./run +.... + +and, surely enough, your message has appeared at the beginning of the boot. + +So you are now officially a kernel hacker, way to go! + +Not satisfied with mere software? OK then, let's hack up the QEMU x86 CPU identification: + +.... +vim submodules/qemu/target/i386/cpu.c +.... + +and modify: + +.... +.model_id = "QEMU Virtual CPU version " QEMU_HW_VERSION, +.... + +to contain: + +.... +.model_id = "QEMU Virtual CPU version HACKED " QEMU_HW_VERSION, +.... + +then as usual rebuild and re-run: + +..... +./build-qemu && ./run --eval-busybox 'grep "model name" /proc/cpuinfo' +..... + +and once again, there is your message: QEMU communicated it to the Linux kernel, which printed it out. + +You have now gone from newb to hardware hacker in a mere 15 minutes, your rate of progress is truly astounding!!! + +BTW: if you want to be a real hardware hacker, it just can't be done with open source tools as of 2018. The root obstacle is that link:https://en.wikipedia.org/wiki/Semiconductor_fabrication_plant[Silicon fabs] don't publish reveal their link:https://en.wikipedia.org/wiki/Design_rule_checking[design rules], which implies that there are no decent standard cell libraries, which implies that you can't get decent power performance and area estimates, see: https://www.quora.com/Are-there-good-open-source-standard-cell-libraries-to-learn-IC-synthesis-with-EDA-tools/answer/Ciro-Santilli The only thing you can do with open source is purely functional designs with Verilator, but you will never know if it can be actually produced and how efficient it can be. + +==== About the QEMU Buildroot setup + +link:https://en.wikipedia.org/wiki/Buildroot[Buildroot] is a set of `make` scripts that downloads from source and compiles compatible versions of: + +* GCC +* Linux kernel +* glibc +* link:https://en.wikipedia.org/wiki/BusyBox[BusyBox] + +It therefore produces a pristine, blob-less, debuggable setup, where all moving parts are configured to work perfectly together. + +The downsides of Buildroot are: + +* the first build takes a while, but it is well worth it +* the selection of software packages is relatively limited if compared to Debian, e.g. no Java or Python package in guest out of the box. ++ +In theory, any software can be packaged, and the Buildroot side is easy. ++ +The hard part is dealing with crappy third party build systems and huge dependency chains. + +link:https://en.wikipedia.org/wiki/QEMU[QEMU] is a system simulator: it simulates a CPU and devices such as interrupt handlers, timers, UART, screen, keyboard, etc. + +QEMU is the leading cross arch system simulator as of 2018. It is even the default Android simulator that developers get with Android Studio 3 to develop apps without real hardware. + +QEMU is also supported by Buildroot in-tree, see e.g.: https://github.com/buildroot/buildroot/blob/2018.05/configs/qemu_aarch64_virt_defconfig + +All of this makes QEMU the natural choice of system simulator. + +=== gem5 Buildroot setup + +==== About the gem5 Buildroot setup + +This setup is like the <>, but it uses link:http://gem5.org/[gem5] instead of QEMU as a system simulator. + +QEMU tries to run as fast as possible and give correct results at the end, but it does not tell us how many CPU cycles it takes to do something, just the number of instructions it ran, and this cannot be used to estimate system performance. This is known as a functional simulation + +gem5 on the other hand, can simulate the system in more detail than QEMU, including: + +* simplified CPU pipeline +* caches +* DRAM timing + +and can therefore be used to estimate system performance, see: <> for an example. + +The downside of gem5 much slower than QEMU because of the greater simulation detail. + +See <> for a more thorough comparison. + +==== gem5 Buildroot setup getting started + +For the most part, if you just add the `--gem5` option or `-gem5` suffix to all commands and everything should magically work: + +.... +./configure --gem5 && \ + ./build-gem5 --arch aarch64 && \ + ./build-buildroot --arch aarch64 --gem5 && \ + ./run --arch aarch64 --gem5 &&\ +:; +.... + +If you have already built <> previously, don't be afraid, gem5 and QEMU use almost the same root filesystem and kernel, so `./build-buildroot --gem` will be fast. It is currently only needed for the <> tool. + +To get a terminal, either open a new shell and run: + +.... +./gem5-shell +.... + +or if you are inside tmux, which I highly recommend, just run gem5 with: + +.... +./run --tmux +.... + +This will open up a split terminal by default so that you can see both the gem5 stdout and the terminal. See also: <> + +If you forgot to open the shell and gem5 exit, you can inspect the terminal output post-mortem at: + +.... +less "$(./getvar --gem5 termout_file)" +.... + +TODO: `arm` boot broken on kernel 4.18 with: + +.... +Internal error: Oops - undefined instruction +.... + +Workaround by checking out kernel 4.17 as explained at <>. Bisected down to kernel 1d4238c56f9816ce0f9c8dbe42d7f2ad81cb6613, gem5 is not implementing the `CSDB` instruction. + +More information at: <> [[docker]] -==== Getting started with Docker +=== QEMU Buildroot setup inside Docker -This is a good option if you are on a Linux host, but the <> failed due to your weird host distribution. +This is the same as the native <>, but run from inside a Docker on the host. + +This is a good option if you are on a Linux host, but the native setup failed due to your weird host distribution, and you have better things to do with your life than to debug it. Before anything, you must get rid of any host build files on `out/` if you have any. A simple way to do this it to: @@ -199,10 +457,10 @@ If you mistakenly run `./rundocker` twice, it opens two mirrored terminals. To q Ctrl-P Ctrl-Q .... -To use <> from Docker: +To use <> from Docker: .... -./run -Vx +./run --graphic --vnc .... and then on host: @@ -234,34 +492,22 @@ After this, to start using Docker again will you need another: .... [[prebuilt]] -==== Getting started with prebuilts +=== Prebuilt setup + +==== About the prebuilt setup + +This setup uses prebuilt binaries of the <> that we upload to GitHub from time to time. We don't currently provide a full prebuilt because it would be too big to host freely, notably because of the cross toolchain. -However, we do provide a prebuilt filesystem and kernel, which allows you to quickly try out running our kernel modules: +Our prebuilts currently include: -. Download QEMU and this repo: -+ -.... -sudo apt-get install qemu-system-x86 -git clone https://github.com/cirosantilli/linux-kernel-module-cheat -cd linux-kernel-module-cheat -.... -. go to the latest release link:https://github.com/cirosantilli/linux-kernel-module-cheat/releases[], download the `lkmc-*.zip` file and extract it into the repository: -+ -.... -unzip lkmc-*.zip -.... -+ -It is link:https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downloading-a-file-in-the-latest-release-of-a-repo/50540591#50540591[not possible to automate this step without the API], and I'm not venturing there at this time, pull requests welcome. -. checkout to the prebuilt repo version so that the scripts and documentation will be compatible with it, and run with the `-P` option: -+ -.... -git checkout -./run -P -.... +* Linux kernel +* root filesystem -Limitations of this method: +Advantages: saves time and disk space. + +Limitations: * can't GDB step debug the kernel, since the source and cross toolchain with GDB are not available. Buildroot cannot easily use a host toolchain: <>. + @@ -271,8 +517,45 @@ Maybe we could work around this by just downloading the kernel source somehow, a * you won't get the latest version of this repository. Our <> attempt to automate builds failed, and storing a release for every commit would likely make GitHub mad at us. * <> is not currently supported, although it should not be too hard to do. One annoyance is that there is no Debian package for it, so you have to compile your own, so you might as well just build the image itself. +==== Prebuilt setup getting started + +Download this repo: + +.... +git clone https://github.com/cirosantilli/linux-kernel-module-cheat +cd linux-kernel-module-cheat +.... + +Go to the latest release link:https://github.com/cirosantilli/linux-kernel-module-cheat/releases[], download the `lkmc-*.zip` file and extract it into the repository: + +.... +unzip lkmc-*.zip +.... + +It is link:https://stackoverflow.com/questions/24987542/is-there-a-link-to-github-for-downloading-a-file-in-the-latest-release-of-a-repo/50540591#50540591[not possible to automate this step without the API], and I'm not venturing there at this time, pull requests welcome. + +Checkout to the prebuilt repo version so that the scripts and documentation will be compatible with it, and run with the `--prebuilt` option: + +.... +git checkout +git submodule update --init "$(./getvar qemu_src_dir)" +./build-qemu +./run +.... + +and you are done. + +Alternatively, you can also try to use the host QEMU: + +.... +sudo apt-get install qemu-system-x86 +./run --prebuilt +.... + +but QEMU builds are pretty quick, and this further increases the probability of incompatibilities, are you really that lazy? + [[host]] -==== Getting started on host +=== Host kernel module setup This method runs the kernel modules directly on your host computer without a VM, and saves you the compilation time and disk usage of the virtual machine method. @@ -289,7 +572,7 @@ It has however severe limitations, and you will soon see that the compilation ti Still interested? .... -cd kernel_module +cd packages/kernel_modules ./make-host.sh .... @@ -309,7 +592,7 @@ sudo insmod hello.ko sudo lsmod | grep hello # Last message should be: hello init -dmest -T +dmesg -T sudo rmmod hello @@ -323,7 +606,7 @@ sudo lsmod | grep hello Once you are done with this method, you must clean up the in-tree build objects before you decide to do the right thing and move on to the superior `./build` Buildroot method: .... -cd "kernel_module" +cd packages/kernel_modules ./make-host.sh clean .... @@ -334,7 +617,7 @@ otherwise they will cause problems. Minimal host build system example: .... -cd hello_host +cd hello_host_kernel_module make insmod hello.ko dmesg @@ -342,868 +625,6 @@ rmmod hello.ko dmesg .... -=== Text mode - -By default, we show the serial console directly on the current terminal, without opening a QEMU window. - -Quit QEMU immediately: - -.... -Ctrl-A X -.... - -https://superuser.com/questions/1087859/how-to-quit-the-qemu-monitor-when-not-using-a-gui - -Alternative methods: - -* `quit` command on the <> -* `pkill qemu` - -TODO: if you hit `Ctrl-C` several times while `arm` or `aarch64` are booting, after boot the userland shell does not show any updates when you type, this seems to be a bug on the Linux kernel v4.16: http://lists.nongnu.org/archive/html/qemu-discuss/2018-04/msg00027.html - -=== Graphic mode - -Enable graphic mode: - -.... -./run -x -.... - -Text mode is the default due to the following considerable advantages: - -* copy and paste commands and stdout output to / from host -* get full panic traces when you start making the kernel crash :-) See also: https://unix.stackexchange.com/questions/208260/how-to-scroll-up-after-a-kernel-panic -* have a large scroll buffer, and be able to search it, e.g. by using tmux on host -* one less window floating around to think about in addition to your shell :-) -* graphics mode has only been properly tested on `x86_64`. - -Text mode has the following limitations over graphics mode: - -* you can't see graphics such as those produced by <> -* very early kernel messages such as `early console in extract_kernel` only show on the GUI, since at such early stages, not even the serial has been setup. - -`x86_64` has a VGA device enabled by default, as can be seen as: - -.... -./qemumonitor info qtree -.... - -and the Linux kernel picks it up through the link:https://en.wikipedia.org/wiki/Linux_framebuffer[fbdev] graphics system as can be seen from: - -.... -cat /dev/urandom > /dev/fb0 -.... - -flooding the screen with colors. See also: https://superuser.com/questions/223094/how-do-i-know-if-i-have-kms-enabled - -==== Graphic mode arm - -===== Graphic mode arm terminal - -TODO: on arm, we see the penguin and some boot messages, but don't get a shell at then end: - -.... -./run -a aarch64 -x -.... - -I think it does not work because the graphic window is <> only, i.e.: - -.... -cat /dev/urandom > /dev/fb0 -.... - -fails with: - -.... -cat: write error: No space left on device -.... - -and has no effect, and the Linux kernel does not appear to have a built-in DRM console as it does for fbdev with <>. - -There is however one out-of-tree implementation: <>. - -===== Graphic mode arm terminal implementation - -`arm` and `aarch64` rely on the QEMU CLI option: - -.... --device virtio-gpu-pci -.... - -and the kernel config options: - -.... -CONFIG_DRM=y -CONFIG_DRM_VIRTIO_GPU=y -.... - -Unlike x86, `arm` and `aarch64` don't have a display device attached by default, thus the need for `virtio-gpu-pci`. - -See also https://wiki.qemu.org/Documentation/Platforms/ARM (recently edited and corrected by yours truly... :-)). - -===== Graphic mode arm VGA - -TODO: how to use VGA on ARM? https://stackoverflow.com/questions/20811203/how-can-i-output-to-vga-through-qemu-arm Tried: - -.... --device VGA -.... - -But https://github.com/qemu/qemu/blob/v2.12.0/docs/config/mach-virt-graphical.cfg#L264 says: - -.... -# We use virtio-gpu because the legacy VGA framebuffer is -# very troublesome on aarch64, and virtio-gpu is the only -# video device that doesn't implement it. -.... - -so maybe it is not possible? - -==== Graphic mode gem5 - -TODO could not get it working on `x86_64`, only ARM. - -Overview: https://stackoverflow.com/questions/50364863/how-to-get-graphical-gui-output-and-user-touch-keyboard-mouse-input-in-a-ful/50364864#50364864 - -More concretely: - -.... -git -C linux checkout gem5/v4.15 -./build -gl -aa -K linux/arch/arm/configs/gem5_defconfig -L gem5-v4.15 -git -C linux checkout - -./run -aa -g -L gem5-v4.15 -.... - -and then on another shell: - -.... -vinagre localhost:5900 -.... - -The <> penguin only appears after several seconds, together with kernel messages of type: - -.... -[ 0.152755] [drm] found ARM HDLCD version r0p0 -[ 0.152790] hdlcd 2b000000.hdlcd: bound virt-encoder (ops 0x80935f94) -[ 0.152795] [drm] Supports vblank timestamp caching Rev 2 (21.10.2013). -[ 0.152799] [drm] No driver support for vblank timestamp query. -[ 0.215179] Console: switching to colour frame buffer device 240x67 -[ 0.230389] hdlcd 2b000000.hdlcd: fb0: frame buffer device -[ 0.230509] [drm] Initialized hdlcd 1.0.0 20151021 for 2b000000.hdlcd on minor 0 -.... - -The port `5900` is incremented by one if you already have something running on that port, `gem5` stdout tells us the right port on stdout as: - -.... -system.vncserver: Listening for connections on port 5900 -.... - -and when we connect it shows a message: - -.... -info: VNC client attached -.... - -Alternatively, you can also view the frames with `--frame-capture`: - -.... -./run -aa -g -L gem5-v4.15 -- --frame-capture -.... - -This option dumps one compressed PNG whenever the screen image changes inside `m5out`, indexed by the cycle ID. This allows for more controlled experiments. - -It is fun to see how we get one new frame whenever the white underscore cursor appears and reappears under the penguin. - -TODO <> failed on `aarch64` with: - -.... -kmscube[706]: unhandled level 2 translation fault (11) at 0x00000000, esr 0x92000006, in libgbm.so.1.0.0[7fbf6a6000+e000] -.... - -Tested on: link:http://github.com/cirosantilli/linux-kernel-module-cheat/commit/38fd6153d965ba20145f53dc1bb3ba34b336bde9[38fd6153d965ba20145f53dc1bb3ba34b336bde9] - -==== Graphic mode gem5 aarch64 - -For `aarch64` we also need `-c kernel_config_fragment/display`: - -.... -git -C linux checkout gem5/v4.15 -./build -gl -a A \ - -c kernel_config_fragment/display \ - -K linux/arch/arm64/configs/gem5_defconfig \ - -L gem5-v4.15 \ -; -git -C linux checkout - -./run -a A -gu -L gem5-v4.15 -.... - -This is because the gem5 `aarch64` defconfig does not enable HDLCD like the 32 bit one `arm` one for some reason. - -===== Graphic mode gem5 internals - -We cannot use mainline Linux because the <> are required at least to provide the `CONFIG_DRM_VIRT_ENCODER` option. - -gem5 emulates the link:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0541c/CHDBAIDI.html[HDLCD] ARM Holdings hardware for `arm` and `aarch64`. - -The kernel uses HDLCD to implement the <> interface, the required kernel config options are present at: link:kernel_config_fragment/display[]. - -TODO: minimize out the `-K`. If we just remove it on `arm`: it does not work with a failing dmesg: - -.... -[ 0.066208] [drm] found ARM HDLCD version r0p0 -[ 0.066241] hdlcd 2b000000.hdlcd: bound virt-encoder (ops drm_vencoder_ops) -[ 0.066247] [drm] Supports vblank timestamp caching Rev 2 (21.10.2013). -[ 0.066252] [drm] No driver support for vblank timestamp query. -[ 0.066276] hdlcd 2b000000.hdlcd: Cannot do DMA to address 0x0000000000000000 -[ 0.066281] swiotlb: coherent allocation failed for device 2b000000.hdlcd size=8294400 -[ 0.066288] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.15.0 #1 -[ 0.066293] Hardware name: V2P-AARCH64 (DT) -[ 0.066296] Call trace: -[ 0.066301] dump_backtrace+0x0/0x1b0 -[ 0.066306] show_stack+0x24/0x30 -[ 0.066311] dump_stack+0xb8/0xf0 -[ 0.066316] swiotlb_alloc_coherent+0x17c/0x190 -[ 0.066321] __dma_alloc+0x68/0x160 -[ 0.066325] drm_gem_cma_create+0x98/0x120 -[ 0.066330] drm_fbdev_cma_create+0x74/0x2e0 -[ 0.066335] __drm_fb_helper_initial_config_and_unlock+0x1d8/0x3a0 -[ 0.066341] drm_fb_helper_initial_config+0x4c/0x58 -[ 0.066347] drm_fbdev_cma_init_with_funcs+0x98/0x148 -[ 0.066352] drm_fbdev_cma_init+0x40/0x50 -[ 0.066357] hdlcd_drm_bind+0x220/0x428 -[ 0.066362] try_to_bring_up_master+0x21c/0x2b8 -[ 0.066367] component_master_add_with_match+0xa8/0xf0 -[ 0.066372] hdlcd_probe+0x60/0x78 -[ 0.066377] platform_drv_probe+0x60/0xc8 -[ 0.066382] driver_probe_device+0x30c/0x478 -[ 0.066388] __driver_attach+0x10c/0x128 -[ 0.066393] bus_for_each_dev+0x70/0xb0 -[ 0.066398] driver_attach+0x30/0x40 -[ 0.066402] bus_add_driver+0x1d0/0x298 -[ 0.066408] driver_register+0x68/0x100 -[ 0.066413] __platform_driver_register+0x54/0x60 -[ 0.066418] hdlcd_platform_driver_init+0x20/0x28 -[ 0.066424] do_one_initcall+0x44/0x130 -[ 0.066428] kernel_init_freeable+0x13c/0x1d8 -[ 0.066433] kernel_init+0x18/0x108 -[ 0.066438] ret_from_fork+0x10/0x1c -[ 0.066444] hdlcd 2b000000.hdlcd: Failed to set initial hw configuration. -[ 0.066470] hdlcd 2b000000.hdlcd: master bind failed: -12 -[ 0.066477] hdlcd: probe of 2b000000.hdlcd failed with error -12 -[ -.... - -So what other options are missing from `gem5_defconfig`? It would be cool to minimize it out to better understand the options. - -=== Automatic startup commands - -When debugging a module, it becomes tedious to wait for build and re-type: - -.... -/modulename.sh -.... - -every time. - -To automate that, use the methods described at: <> - -=== printk - -We use `printk` a lot, and it shows on the terminal by default, along with stdout and what you type. - -Hide all `printk` messages: - -.... -dmesg -n 1 -.... - -or equivalently: - -.... -echo 1 > /proc/sys/kernel/printk -.... - -See also: https://superuser.com/questions/351387/how-to-stop-kernel-messages-from-flooding-my-console - -Do it with a <> to affect the boot itself: - -.... -./run -e 'loglevel=5' -.... - -and now only boot warning messages or worse show, which is useful to identify problems. - -Our default `printk` format is: - -.... -[TIMESTAMP] MESSAGE -.... - -e.g.: - -.... -<6>[ 2.979121] Freeing unused kernel memory: 2024K -.... - -where: - -* `LEVEL`: higher means less serious -* `TIMESTAMP`: seconds since boot - -This format is selected by the following boot options: - -* `console_msg_format=syslog`: add the `` part. Added in v4.16. -* `printk.time=y`: add the `[TIMESTAMP]` part - -Scroll up in <>: - -.... -Shift-PgUp -.... - -but I never managed to increase that buffer: - -* https://askubuntu.com/questions/709697/how-to-increase-scrollback-lines-in-ubuntu14-04-2-server-edition -* https://unix.stackexchange.com/questions/346018/how-to-increase-the-scrollback-buffer-size-for-tty - -The superior alternative is to use text mode and GNU screen or tmux. - -==== pr_debug - -https://stackoverflow.com/questions/28936199/why-is-pr-debug-of-the-linux-kernel-not-giving-any-output/49835405#49835405 - -Debug messages are not printable by default without recompiling. - -But the awesome `CONFIG_DYNAMIC_DEBUG=y` option which we enable by default allows us to do: - -.... -echo 8 > /proc/sys/kernel/printk -echo 'file kernel/module.c +p' > /sys/kernel/debug/dynamic_debug/control -/myinsmod.out /hello.ko -.... - -and we have a shortcut at: - -.... -/pr_debug.sh -.... - -Source: link:rootfs_overlay/pr_debug.sh[]. - -Syntax: https://www.kernel.org/doc/html/v4.11/admin-guide/dynamic-debug-howto.html - -Wildcards are also accepted, e.g. enable all messages from all files: - -.... -echo 'file * +p' > /sys/kernel/debug/dynamic_debug/control -.... - -TODO: why is this not working: - -.... -echo 'func sys_init_module +p' > /sys/kernel/debug/dynamic_debug/control -.... - -Enable messages in specific modules: - -.... -echo 8 > /proc/sys/kernel/printk -echo 'module myprintk +p' > /sys/kernel/debug/dynamic_debug/control -insmod /myprintk.ko -.... - -Source: link:kernel_module/myprintk.c[] - -This outputs the `pr_debug` message: - -.... -printk debug -.... - -but TODO: it also shows debug messages even without enabling them explicitly: - -.... -echo 8 > /proc/sys/kernel/printk -insmod /myprintk.ko -.... - -and it shows as enabled: - -.... -# grep myprintk /sys/kernel/debug/dynamic_debug/control -/linux-kernel-module-cheat/out/x86_64/buildroot/build/kernel_module-1.0/./myprintk.c:12 [myprintk]myinit =p "pr_debug\012" -.... - -Enable `pr_debug` for boot messages as well, before we can reach userland and write to `/proc`: - -.... -./run -e 'dyndbg="file * +p" loglevel=8' -.... - -Get ready for the noisiest boot ever, I think it overflows the `printk` buffer and funny things happen. - -===== pr_debug != printk(KERN_DEBUG - -When `CONFIG_DYNAMIC_DEBUG` is set, `printk(KERN_DEBUG` is not the exact same as `pr_debug(` since `printk(KERN_DEBUG` messages are visible with: - -.... -./run -e 'initcall_debug logleve=8' -.... - -which outputs lines of type: - -.... -<7>[ 1.756680] calling clk_disable_unused+0x0/0x130 @ 1 -<7>[ 1.757003] initcall clk_disable_unused+0x0/0x130 returned 0 after 111 usecs -.... - -which are `printk(KERN_DEBUG` inside `init/main.c` in v4.16. - -Mentioned at: https://stackoverflow.com/questions/37272109/how-to-get-details-of-all-modules-drivers-got-initialized-probed-during-kernel-b - -This likely comes from the ifdef split at `init/main.c`: - -.... -/* If you are writing a driver, please use dev_dbg instead */ -#if defined(CONFIG_DYNAMIC_DEBUG) -#include - -/* dynamic_pr_debug() uses pr_fmt() internally so we don't need it here */ -#define pr_debug(fmt, ...) \ - dynamic_pr_debug(fmt, ##__VA_ARGS__) -#elif defined(DEBUG) -#define pr_debug(fmt, ...) \ - printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) -#else -#define pr_debug(fmt, ...) \ - no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) -#endif -.... - -==== ignore_loglevel - -.... -./run -e 'ignore_loglevel' -.... - -enables all log levels, and is basically the same as: - -.... -./run -e 'loglevel=8' -.... - -except that you don't need to know what is the maximum level. - -=== Rebuild - -After making changes to a package, you must explicitly request it to be rebuilt. - -For example, you you modify the kernel modules, you must rebuild with: - -.... -./build -k -.... - -which is just an alias for: - -.... -./build -- kernel_module-reconfigure -.... - -where `kernel_module` is the name of out Buildroot package that contains the kernel modules. - -Other important targets are: - -.... -./build -l -q -g -.... - -which rebuild the Linux kernel, and QEMU and gem5 respectively. They are essentially aliases for: - -.... -./build -- linux-reconfigure host-qemu-reconfigure gem5-reconfigure -.... - -However, some of our aliases such as `-l` also have some magic convenience properties. So generally just use the aliases instead. - -We don't rebuild by default because, even with `make` incremental rebuilds, the timestamp check takes a few annoying seconds. - -Not all packages have an alias, when they don't, just use the form: - -.... -./build -- -reconfigure -.... - -==== Rebuild a package with different build options - -For example, if you decide to <> after an initial build is finished, you must first clean the build before rebuilding: - -.... -./build -B 'BR2_OPTIMIZE_3=y' kernel_module-dirclean kernel_module-reconfigure -.... - -as explained at: https://buildroot.org/downloads/manual/manual.html#rebuild-pkg - -The clean is necessary because the source files didn't change, so `make` would just check the timestamps and not build anything. - -[[retype]] -=== Don't retype arguments all the time - -It gets annoying to retype `-a aarch64` for every single command, or to remember `./build -B` setups. - -So simplify that, do: - -.... -cp config.example data/config -.... - -and then edit the `data/config` file to your needs. - -=== Clean the build - -You did something crazy, and nothing seems to work anymore? - -All builds are stored under `buildroot/`, - -The most coarse thing you can do is: - -.... -cd buildroot -git checkout -- . -git clean -xdf . -.... - -To only nuke one architecture, do: - -.... -rm -rf "$(./getvar buildroot_out_dir)" -.... - -Only nuke one one package: - -.... -rm -rf "$(./getvar buildroot_out_dir)/build/host-qemu-custom" -./build -.... - -This is sometimes necessary when changing the version of the submodules, and then builds fail. We should try to understand why and report bugs. - -=== Filesystem persistency - -We disable filesystem persistency for both QEMU and gem5 by default, to prevent the emulator from putting the image in an unknown state. - -For QEMU, this is done by passing the `snapshot` option to `-drive`, and for gem5 it is the default behaviour. - -If you hack up our link:run[] script to remove that option, then: - -.... -./run -F 'date >f;poweroff' - -.... - -followed by: - -.... -./run -F 'cat f' -.... - -gives the date, because `poweroff` without `-n` syncs before shutdown. - -The `sync` command also saves the disk: - -.... -sync -.... - -When you do: - -.... -./build -.... - -the disk image gets overwritten by a fresh filesystem and you lose all changes. - -Remember that if you forcibly turn QEMU off without `sync` or `poweroff` from inside the VM, e.g. by closing the QEMU window, disk changes may not be saved. - -Persistency is also turned off when booting from <> with a CPIO instead of with a disk. - -Disk persistency is useful to re-run shell commands from the history of a previous session with `Ctrl-R`, but we felt that the loss of determinism was not worth it. - -==== gem5 disk persistency - -TODO how to make gem5 disk writes persistent? - -As of cadb92f2df916dbb47f428fd1ec4932a2e1f0f48 there are some `read_only` entries in the `config.ini` under cow sections, but hacking them to true did not work: - -.... -diff --git a/configs/common/FSConfig.py b/configs/common/FSConfig.py -index 17498c42b..76b8b351d 100644 ---- a/configs/common/FSConfig.py -+++ b/configs/common/FSConfig.py -@@ -60,7 +60,7 @@ os_types = { 'alpha' : [ 'linux' ], - } - - class CowIdeDisk(IdeDisk): -- image = CowDiskImage(child=RawDiskImage(read_only=True), -+ image = CowDiskImage(child=RawDiskImage(read_only=False), - read_only=False) - - def childImage(self, ci): -.... - -The directory of interest is `src/dev/storage`. - -qcow2 does not appear supported, there are not hits in the source tree, and there is a mention on Nate's 2009 wishlist: http://gem5.org/Nate%27s_Wish_List - -=== Kernel command line parameters - -Bootloaders can pass a string as input to the Linux kernel when it is booting to control its behaviour, much like the `execve` system call does to userland processes. - -This allows us to control the behaviour of the kernel without rebuilding anything. - -With QEMU, QEMU itself acts as the bootloader, and provides the `-append` option and we expose it through `./run -e`, e.g.: - -.... -./run -e 'foo bar' -.... - -Then inside the host, you can check which options were given with: - -.... -cat /proc/cmdline -.... - -They are also printed at the beginning of the boot message: - -.... -dmesg | grep "Command line" -.... - -See also: - -* https://unix.stackexchange.com/questions/48601/how-to-display-the-linux-kernel-command-line-parameters-given-for-the-current-bo -* https://askubuntu.com/questions/32654/how-do-i-find-the-boot-parameters-used-by-the-running-kernel - -The arguments are documented in the kernel documentation: https://www.kernel.org/doc/html/v4.14/admin-guide/kernel-parameters.html - -When dealing with real boards, extra command line options are provided on some magic bootloader configuration file, e.g.: - -* GRUB configuration files: https://askubuntu.com/questions/19486/how-do-i-add-a-kernel-boot-parameter -* Raspberry pi `/boot/cmdline.txt` on a magic partition: https://raspberrypi.stackexchange.com/questions/14839/how-to-change-the-kernel-commandline-for-archlinuxarm-on-raspberry-pi-effectly - -==== Kernel command line parameters escaping - -Double quotes can be used to escape spaces as in `opt="a b"`, but double quotes themselves cannot be escaped, e.g. `opt"a\"b"` - -This even lead us to use base64 encoding with `-E`! - -==== Kernel command line parameters definition points - -There are two methods: - -* `__setup` as in: -+ -.... -__setup("console=", console_setup); -.... -* `core_param` as in: -+ -.... -core_param(panic, panic_timeout, int, 0644); -.... - -`core_param` suggests how they are different: - -.... -/** - * core_param - define a historical core kernel parameter. - -... - - * core_param is just like module_param(), but cannot be modular and - * doesn't add a prefix (such as "printk."). This is for compatibility - * with __setup(), and it makes sense as truly core parameters aren't - * tied to the particular file they're in. - */ -.... - -==== norandmaps - -Disable userland address space randomization. Test it out by running <> twice: - -.... -./run -F '/rand_check.out;/poweroff.out' -./run -F '/rand_check.out;/poweroff.out' -.... - -If we remove it from our link:run[] script by hacking it up, the addresses shown by `rand_check.out` vary across boots. - -Equivalent to: - -.... -echo 0 > /proc/sys/kernel/randomize_va_space -.... - -=== insmod alternatives - -==== modprobe - -If you are feeling fancy, you can also insert modules with: - -.... -modprobe hello -.... - -which insmods link:kernel_module/hello.c[]. - -`modprobe` searches for modules under: - -.... -ls /lib/modules/*/extra/ -.... - -Kernel modules built from the Linux mainline tree with `CONFIG_SOME_MOD=m`, are automatically available with `modprobe`, e.g.: - -.... -modprobe dummy-irq irq=1 -.... - -==== myinsmod - -If you are feeling raw, you can insert and remove modules with our own minimal module inserter and remover! - -.... -# init_module -/myinsmod.out /hello.ko -# finit_module -/myinsmod.out /hello.ko "" 1 -/myrmmod.out hello -.... - -which teaches you how it is done from C code. - -Source: - -* link:kernel_module/user/myinsmod.c[] -* link:kernel_module/user/myrmmod.c[] - -The Linux kernel offers two system calls for module insertion: - -* `init_module` -* `finit_module` - -and: - -.... -man init_module -.... - -documents that: - -____ -The finit_module() system call is like init_module(), but reads the module to be loaded from the file descriptor fd. It is useful when the authenticity of a kernel module can be determined from its location in the filesystem; in cases where that is possible, the overhead of using cryptographically signed modules to determine the authenticity of a module can be avoided. The param_values argument is as for init_module(). -____ - -`finit` is newer and was added only in v3.8. More rationale: https://lwn.net/Articles/519010/ - -Bibliography: https://stackoverflow.com/questions/5947286/how-to-load-linux-kernel-modules-from-c-code - -=== Simultaneous runs - -When doing long simulations sweeping across multiple system parameters, it becomes fundamental to do multiple simulations in parallel. - -This is specially true for gem5, which runs much slower than QEMU, and cannot use multiple host cores to speed up the simulation: https://github.com/cirosantilli-work/gem5-issues/issues/15 - -This also has a good synergy with <>. - -First shell: - -.... -./run -.... - -Another shell: - -.... -./run -n 1 -.... - -The default run id is `0`. - -This method also allows us to keep run outputs in separate directories for later inspection, e.g.: - -.... -./run -a A -g -n 0 &>/dev/null & -./run -a A -g -n 1 &>/dev/null & -.... - -produces two separate `m5out` directories: - -.... -ls "$(./getvar -a A -g -n 0 m5out_dir)" -ls "$(./getvar -a A -g -n 1 m5out_dir)" -.... - -and the gem5 host executable stdout and stderr can be found at: - -.... -less "$(./getvar -a A -g -n 0 termout_file)" -less "$(./getvar -a A -g -n 1 termout_file)" -.... - -Each line is prepended with the timestamp in seconds since the start of the program when it appeared. - -You can also add a prefix to the build ID before a period: - -.... -./run -a A -g -n some-experiment.1 -.... - -and makes it easier to remember afterwards which directory contains what. - -However this still takes up the same ports as: - -.... -./run -a A -g -n 1 -.... - -so you cannot run both at the same time. - -Like <>, you will need to pass the `-n` option to anything that needs to know runtime information, e.g. <>: - -.... -./run -n 1 -./rungdb -n 1 -.... - -To run multiple gem5 checkouts, see: <>. - -Implementation note: we create multiple namespaces for two things: - -* run output directory -* ports -** QEMU allows setting all ports explicitly. -+ -If a port is not free, it just crashes. -+ -We assign a contiguous port range for each run ID. -** gem5 automatically increments ports until it finds a free one. -+ -gem5 60600f09c25255b3c8f72da7fb49100e2682093a does not seem to expose a way to set the terminal and VNC ports from `fs.py`, so we just let gem5 assign the ports itself, and use `-n` only to match what it assigned. Those ports both appear on `config.ini`. -+ -The GDB port can be assigned on `gem5.opt --remote-gdb-port`, but it does not appear on `config.ini`. - -=== Build the documentation - -You don't need to depend on GitHub: - -.... -./build-doc -xdg-open out/README.html -.... - -Source: link:build-doc[] - [[gdb]] == GDB step debug @@ -1212,7 +633,7 @@ Source: link:build-doc[] `-d` makes QEMU wait for a GDB connection, otherwise we could accidentally go past the point we want to break at: .... -./run -d +./run --debug-guest .... Say you want to break at `start_kernel`. So on another shell: @@ -1304,7 +725,7 @@ tmux Now that you are inside a shell inside tmux, run: .... -./run -du +./run --debug-guest --tmux .... Gives splits the terminal into two panes: @@ -1322,7 +743,7 @@ Now you can navigate with the usual tmux shortcuts: To start again, switch back to the QEMU pane, kill the emulator, and re-run: .... -./run -du +./run --debug-guest --tmux .... This automatically clears the GDB pane, and starts a new one. @@ -1330,7 +751,7 @@ This automatically clears the GDB pane, and starts a new one. Pass extra GDB arguments with: .... -./run -du -U start_kernel +./run --debug-guest --tmux --tmux-args start_kernel .... See the tmux manual for further details: @@ -1344,7 +765,7 @@ man tmux If you are using gem5 instead of QEMU, `-u` has a different effect: it opens the gem5 terminal instead of the debugger: .... -./run -gu +./run --gem5 --tmux .... If you also want to use the debugger with gem5, you will need to create new terminals as usual. @@ -1354,7 +775,7 @@ From inside tmux, you can do that with `Ctrl-B C` or `Ctrl-B %`. To see the debugger by default instead of the terminal, run: .... -./tmu ./rungdb;./run -dg +./tmu ./rungdb;./run --debug-guest --gem5 .... === GDB step debug kernel module @@ -1379,7 +800,7 @@ Wait for the boot to end and run: insmod /timer.ko .... -Source: link:kernel_module/timer.c[]. +Source: link:packages/kernel_modules/timer.c[]. This prints a message to dmesg every second. @@ -1393,7 +814,7 @@ In GDB, hit `Ctrl-C`, and note how it says: .... scanning for modules in /linux-kernel-module-cheat//out/x86_64/buildroot/build/linux-custom -loading @0xffffffffc0000000: ../kernel_module-1.0//timer.ko +loading @0xffffffffc0000000: ../kernel_modules-1.0//timer.ko .... That's `lx-symbols` working! Now simply: @@ -1430,18 +851,18 @@ Error occurred in Python command: Cannot access memory at address 0xbf0000cc Can't reproduce on `x86_64` and `aarch64` are fine. -It is kind of random: if you just `insmod` manually and then immediately `./rungdb -a arm`, then it usually works. +It is kind of random: if you just `insmod` manually and then immediately `./rungdb --arch arm`, then it usually works. But this fails most of the time: shell 1: .... -./run -a arm -F 'insmod /hello.ko' +./run --arch arm --eval-busybox 'insmod /hello.ko' .... shell 2: .... -./rungdb -a arm +./rungdb --arch arm .... then hit `Ctrl-C` on shell 2, and voila. @@ -1463,7 +884,7 @@ so it is close to the failing `0xbf0000cc`. `readelf`: .... -./runtc readelf -s "$(./getvar build_dir)/kernel_module-1.0/hello.ko" +./runtc readelf -- -s "$(./getvar build_dir)/kernel_modules-1.0/hello.ko" .... does not give any interesting hits at `cc`, no symbol was placed that far. @@ -1522,7 +943,7 @@ So once we find the address the first time, we can just reuse it afterwards, as Do a fresh boot and get the module: .... -./run -F '/pr_debug.sh;insmod /fops.ko;/poweroff.out' +./run --eval-busybox '/pr_debug.sh;insmod /fops.ko;/poweroff.out' .... The boot must be fresh, because the load address changes every time we insert, even after removing previous modules. @@ -1536,8 +957,8 @@ The base address shows on terminal: Now let's find the offset of `myinit`: .... -./runtc readelf \ - -s "$(./getvar build_dir)/kernel_module-1.0/fops.ko" | \ +./runtc readelf -- \ + -s "$(./getvar build_dir)/kernel_modules-1.0/fops.ko" | \ grep myinit .... @@ -1556,7 +977,7 @@ so the offset address is `0x240` and we deduce that the function will be placed Now we can just do a fresh boot on shell 1: .... -./run -E 'insmod /fops.ko;/poweroff.out' -d +./run --eval 'insmod /fops.ko;/poweroff.out' --debug-guest .... and on shell 2: @@ -1630,7 +1051,7 @@ directly gives an <> as I'd expect. Useless, but a good way to show how hardcore you are. Disable `lx-symbols` with: .... -./rungdb -L +./rungdb --no-lxsymbols .... From inside guest: @@ -1655,7 +1076,7 @@ And then tell GDB where the module was loaded with: .... Ctrl-C -add-symbol-file ../kernel_module-1.0/fops.ko 0xfffffffa00000000 +add-symbol-file ../kernel_modules-1.0/fops.ko 0xfffffffa00000000 .... Alternatively, if the module panics before you can read `/proc/modules`, there is a <> which shows the load address: @@ -1677,7 +1098,7 @@ And then search for a line of type: Break at the very first instruction executed by QEMU: .... -./rungdb -C +./rungdb --no-continue .... TODO why can't we break at early startup stuff such as: @@ -1698,20 +1119,20 @@ See also: https://stackoverflow.com/questions/2589845/what-are-the-first-operati One possibility is to run: .... -./trace-boot -a arm +./trace-boot --arch arm .... and then find the second address (the first one does not work, already too late maybe): .... -less "$(./getvar -a arm trace_txt_file)" +less "$(./getvar --arch arm trace_txt_file)" .... and break there: .... -./run -a arm -d -./rungdb -a arm '*0x1000' +./run --arch arm --debug-guest +./rungdb --arch arm '*0x1000' .... but TODO: it does not show the source assembly under `arch/arm`: https://stackoverflow.com/questions/11423784/qemu-arm-linux-kernel-boot-debug-no-source-code @@ -1762,14 +1183,16 @@ since GDB does not know that libc is loaded. * Shell 1: + .... -./run -d -e 'init=/sleep_forever.out' +./run --debug-guest --kernel-cli 'init=/sleep_forever.out' .... * Shell 2: + .... -./rungdb-user kernel_module-1.0/user/sleep_forever.out main +./rungdb-user kernel_modules-1.0/user/sleep_forever.out main .... +TODO not working as of f8c0502bb2680f2dbe7c1f3d7958f60265347005, does not break. Bisect on recent QEMU and kernel. Debug by creating an executable that prints the address of `main`. + ==== GDB step debug userland BusyBox init BusyBox custom init process: @@ -1777,7 +1200,7 @@ BusyBox custom init process: * Shell 1: + .... -./run -d -e 'init=/bin/ls' +./run --debug-guest --kernel-cli 'init=/bin/ls' .... * Shell 2: + @@ -1792,7 +1215,7 @@ BusyBox default init process: * Shell 1: + .... -./run -d +./run --debug-guest .... * Shell 2: + @@ -1813,12 +1236,12 @@ Non-init process: * Shell 1: + .... -./run -d +./run --debug-guest .... * Shell 2: + .... -./rungdb-user kernel_module-1.0/user/myinsmod.out main +./rungdb-user kernel_modules-1.0/user/myinsmod.out main .... * Shell 1 after the boot finishes: + @@ -1828,19 +1251,19 @@ Non-init process: This is the least reliable setup as there might be other processes that use the given virtual address. -===== GDB step debug userland non-init without -d +===== GDB step debug userland non-init without --debug-guest TODO: on QEMU bfba11afddae2f7b2c1335b4e23133e9cd3c9126, it works on `x86_64` and `aarch64` but fails on arm as follows: * Shell 1: + .... -./run -a arm +./run --arch arm .... * Shell 2: wait for boot to finish, and run: + .... -./rungdb-user -a arm kernel_module-1.0/user/hello.out main +./rungdb-user --arch arm kernel_modules-1.0/user/hello.out main .... * Shell 1: + @@ -1857,8 +1280,8 @@ Cannot access memory at address 0x10604 We have also double checked the address with: .... -./runtc -a arm readelf \ - -s "$(./getvar -a arm build_dir)/kernel_module-1.0/fops.ko" | \ +./runtc --arch arm readelf -- \ + -s "$(./getvar --arch arm build_dir)/kernel_modules-1.0/fops.ko" | \ grep main .... @@ -1945,10 +1368,10 @@ Not possible as of QEMU 3.0.0 it seems: https://stackoverflow.com/questions/4641 We can set and get which cores the Linux kernel allows a program to run on with `sched_getaffinity` and `sched_setaffinity`: .... -./run -c2 -F '/sched_getaffinity.out' +./run --cpus 2 --eval-busybox '/sched_getaffinity.out' .... -Source: link:kernel_module/user/sched_getaffinity.c[] +Source: link:packages/kernel_modules/user/sched_getaffinity.c[] Sample output: @@ -1990,13 +1413,17 @@ Let's do a QEMU observation to justify this example being in the repository with We will run our `/sched_getaffinity.out` infinitely many time, on core 0 and core 1 alternatively: .... -./run -c2 -d -F 'i=0; while true; do taskset -c $i,$i /sched_getaffinity.out; i=$((! $i)); done' +./run \ + --cpus 2 \ + --debug-guest \ + --eval-busybox 'i=0; while true; do taskset -c $i,$i /sched_getaffinity.out; i=$((! $i)); done' \ +; .... on another shell: .... -./rungdb-user kernel_module-1.0/user/sched_getaffinity.out main +./rungdb-user kernel_modules-1.0/user/sched_getaffinity.out main .... Then, inside GDB: @@ -2021,13 +1448,13 @@ We should also try it out with kernel modules: https://stackoverflow.com/questio TODO we then tried: .... -./run -c2 -F '/sched_getaffinity_threads.out' +./run --cpus 2 --eval-busybox '/sched_getaffinity_threads.out' .... and: .... -./rungdb-user kernel_module-1.0/user/sched_getaffinity_threads.out +./rungdb-user kernel_modules-1.0/user/sched_getaffinity_threads.out .... to switch between two simultaneous live threads with different affinities, it just didn't break on our threads: @@ -2137,7 +1564,7 @@ Bibliography: == KGDB -TODO: only working with <>. Without it, nothing shows on the terminal. So likely something linked to the option `console=ttyS0`. +TODO: only working with <>. Without it, nothing shows on the terminal. So likely something linked to the option `console=ttyS0`. KGDB is kernel dark magic that allows you to GDB the kernel on real hardware without any extra hardware support. @@ -2148,8 +1575,8 @@ Cheaper than JTAG (free) and easier to setup (all you need is serial), but with Usage: .... -./run -k -./rungdb -k +./run --kgdb +./rungdb --kgdb .... In GDB: @@ -2177,7 +1604,7 @@ c And now you can count from GDB! -If you do: `b __x64_sys_write` immediately after `./rungdb -k`, it fails with `KGDB: BP remove failed:
`. I think this is because it would break too early on the boot sequence, and KGDB is not yet ready. +If you do: `b __x64_sys_write` immediately after `./rungdb --kgdb`, it fails with `KGDB: BP remove failed:
`. I think this is because it would break too early on the boot sequence, and KGDB is not yet ready. See also: @@ -2219,7 +1646,7 @@ Source: link:rootfs_overlay/kgdb-mod.sh[]. In GDB: .... -lx-symbols ../kernel_module-1.0/ +lx-symbols ../kernel_modules-1.0/ b fop_write c c @@ -2283,7 +1710,7 @@ Source: link:rootfs_overlay/gdbserver.sh[]. Host: .... -./rungdbserver kernel_module-1.0/user/myinsmod.out +./rungdbserver kernel_modules-1.0/user/myinsmod.out .... You can find the executable with: @@ -2318,7 +1745,7 @@ An implementation overview can be found at: https://reverseengineering.stackexch As usual, different archs work with: .... -./rungdbserver -a arm kernel_module-1.0/user/myinsmod.out +./rungdbserver --arch arm kernel_modules-1.0/user/myinsmod.out .... === gdbserver BusyBox @@ -2353,7 +1780,7 @@ s This is made possible by the GDB command: .... -set sysroot ${common_buildroot_out_dir}/staging +set sysroot ${common_buildroot_build_dir}/staging .... which automatically finds unstripped shared libraries on the host for us. @@ -2373,27 +1800,25 @@ The portability of the kernel and toolchains is amazing: change an option and mo To use `arm` instead of x86 for example: .... -./build -a arm -./run -a arm +./build-buildroot --arch arm +./run --arch arm .... Debug: .... -./run -a arm -d +./run --arch arm --debug-guest # On another terminal. -./rungdb -a arm +./rungdb --arch arm .... -We also have one letter shorthand names for the architectures: +We also have one letter shorthand names for the architectures and `--arch` option: .... # aarch64 ./run -a A # arm ./run -a a -# mips64 -./run -a m # x86_64 ./run -a x .... @@ -2440,9 +1865,9 @@ traps: ring0.out[55] general protection ip:40054c sp:7fffffffec20 error:0 in rin Sources: -* link:kernel_module/ring0.c[] -* link:kernel_module/ring0.h[] -* link:kernel_module/user/ring0.c[] +* link:packages/kernel_modules/ring0.c[] +* link:packages/kernel_modules/ring0.h[] +* link:packages/kernel_modules/user/ring0.c[] In both cases, we attempt to run the exact same code which is shared on the `ring0.h` header file. @@ -2460,8 +1885,8 @@ TODO Can you run arm executables in the aarch64 guest? https://stackoverflow.com I've tried: .... -./runtc -a aarch64 gcc|cg -static ~/test/hello_world.c -o data/9p/a.out -./run -a A -F '/mnt/9p/a.out' +./runtc --arch aarch64 gcc -- -static ~/test/hello_world.c -o data/9p/a.out +./run --arch aarch64 --eval-busybox '/mnt/9p/a.out' .... but it fails with: @@ -2470,76 +1895,73 @@ but it fails with: a.out: line 1: syntax error: unexpected word (expecting ")") .... -=== mips64 +=== MIPS -Keep in mind that MIPS has the worst support compared to our other architectures due to the smaller community. Patches welcome as usual. +We used to "support" it until f8c0502bb2680f2dbe7c1f3d7958f60265347005 (it booted) but dropped since one was testing it often. -TODOs: - -* networking is not working. See also: -** https://stackoverflow.com/questions/21496449/networking-is-not-working-on-qemu-guest-malta-mips -** https://unix.stackexchange.com/questions/208266/setting-up-qemu-and-mipsel-networking-trouble -** https://unix.stackexchange.com/questions/354127/qemu-mips-and-debian -* <> does not work properly, does not find `start_kernel` - -==== mips64 X11 - -Haven't tried it, doubt it will work out of the box! :-) - -Maybe: https://stackoverflow.com/questions/47857023/booting-a-graphical-mips-qemu-machine - -==== mips64 gem5 - -Haven't tried. +If you want to revive and maintain it, send a pull request. == init -When the Linux kernel finishes booting, it runs an executable as the first and only userland process. +When the Linux kernel finishes booting, it runs an executable as the first and only userland process. This executable is called the `init` program. -This init process is then responsible for setting up the entire userland (or destroying everything when you want to have fun). +The init process is then responsible for setting up the entire userland (or destroying everything when you want to have fun). -This typically means reading some configuration files (e.g. `/etc/initrc`) and forking a bunch of userland executables based on those files. +This typically means reading some configuration files (e.g. `/etc/initrc`) and forking a bunch of userland executables based on those files, including the very interactive shell that we end up on. systemd provides a "popular" init implementation for desktop distros as of 2017. BusyBox provides its own minimalistic init implementation which Buildroot, and therefore this repo, uses by default. +The `init` program can be either an executable shell text file, or a compiled ELF file. It becomes easy to accept this once you see that the `exec` system call handles both cases equally: https://unix.stackexchange.com/questions/174062/can-the-init-process-be-a-shell-script-in-linux/395375#395375 + +The `init` executable is searched for in a list of paths in the root filesystem, including `/init`, `/sbin/init` and a few others. For more details see: <> + === Replace init To have more control over the system, you can replace BusyBox's init with your own. -The `-E` option replaces init and evals a command from the <>: +The most direct way to replace `init` with our own is to just use the `init=` <> directly: .... -./run -E 'echo "asdf qwer";insmod /hello.ko;/poweroff.out' +./run --kernel-cli 'init=/count.sh' .... +This just counts every second forever and does not give you a shell. + +This method is not very flexible however, as it is hard to reliably pass multiple commands and command line arguments to the init with it, as explained at: <>. + +For this reason, we have created a more robust helper method with the `--eval` option: + +.... +./run --eval 'echo "asdf qwer";insmod /hello.ko;/poweroff.out' +.... + +The `--eval` option replaces init with a shell script that just evals the given command. + It is basically a shortcut for: .... -./run -e 'init=/eval.sh - lkmc_eval="insmod /hello.ko;/poweroff.out"' +./run --kernel-cli 'init=/eval_base64.sh - lkmc_eval="insmod /hello.ko;/poweroff.out"' .... -Source: link:rootfs_overlay/eval.sh[]. +Source: link:rootfs_overlay/eval_base64.sh[]. -However, `-E` is smarter: +This allows quoting and newlines by base64 encoding on host, and decoding on guest, see: <>. -* allows quoting and newlines by using base64 encoding, see: <> -* automatically chooses between `init=` and `rcinit=` for you, see: <> +It also automatically chooses between `init=` and `rcinit=` for you, see: <> -so you should almost always use it, unless you are really counting each cycle ;-) - -This method replaces BusyBox' init completely, which makes things more minimal, but also has has the following consequences: +`--eval` replaces BusyBox' init completely, which makes things more minimal, but also has has the following consequences: * `/etc/fstab` mounts are not done, notably `/proc` and `/sys`, test it out with: + .... -./run -E 'echo asdf;ls /proc;ls /sys;echo qwer' +./run --eval 'echo asdf;ls /proc;ls /sys;echo qwer' .... * no shell is launched at the end of boot for you to interact with the system. You could explicitly add a `sh` at the end of your commands however: + .... -./run -E 'echo hello;sh' +./run --eval 'echo hello;sh' .... The best way to overcome those limitations is to use: <> @@ -2551,7 +1973,7 @@ echo ' insmod /hello.ko /poweroff.out ' > gitignore.sh -./run -E "$(cat gitignore.sh)" +./run --eval "$(cat gitignore.sh)" .... or add it to a file to the root filesystem guest and rebuild: @@ -2562,8 +1984,8 @@ insmod /hello.ko /poweroff.out ' > rootfs_overlay/gitignore.sh chmod +x rootfs_overlay/gitignore.sh -./build -./run -e 'init=/gitignore.sh' +./build-buildroot +./run --kernel-cli 'init=/gitignore.sh' .... Remember that if your init returns, the kernel will panic, there are just two non-panic possibilities: @@ -2576,7 +1998,7 @@ Remember that if your init returns, the kernel will panic, there are just two no Just using BusyBox' `poweroff` at the end of the `init` does not work and the kernel panics: .... -./run -E poweroff +./run --eval poweroff .... because BusyBox' `poweroff` tries to do some fancy stuff like killing init, likely to allow userland to shutdown nicely. @@ -2586,16 +2008,16 @@ But this fails when we are `init` itself! `poweroff` works more brutally and effectively if you add `-f`: .... -./run -E 'poweroff -f' +./run --eval 'poweroff -f' .... but why not just use our minimal `/poweroff.out` and be done with it? .... -./run -E '/poweroff.out' +./run --eval '/poweroff.out' .... -Source: link:kernel_module/user/poweroff.c[] +Source: link:packages/kernel_modules/user/poweroff.c[] This also illustrates how to shutdown the computer from C: https://stackoverflow.com/questions/28812514/how-to-shutdown-linux-using-c-or-qt-without-call-to-system @@ -2604,10 +2026,10 @@ This also illustrates how to shutdown the computer from C: https://stackoverflow I dare you to guess what this does: .... -./run -E '/sleep_forever.out' +./run --eval '/sleep_forever.out' .... -Source: link:kernel_module/user/sleep_forever.c[] +Source: link:packages/kernel_modules/user/sleep_forever.c[] This executable is a convenient simple init that does not panic and sleeps instead. @@ -2616,7 +2038,7 @@ This executable is a convenient simple init that does not panic and sleeps inste Get a reasonable answer to "how long does boot take?": .... -./run -F '/time_boot.out' +./run --eval-busybox '/time_boot.out' .... Dmesg contains a message of type: @@ -2632,10 +2054,10 @@ Bibliography: https://stackoverflow.com/questions/12683169/measure-time-taken-fo [[init-busybox]] === Run command at the end of BusyBox init -Use the `-F` option is for you rely on something that BusyBox' init set up for you like `/etc/fstab`: +Use the `--eval-busybox` option is for you rely on something that BusyBox' init set up for you like `/etc/fstab`: .... -./run -F 'echo asdf;ls /proc;ls /sys;echo qwer' +./run --eval-busybox 'echo asdf;ls /proc;ls /sys;echo qwer' .... After the commands run, you are left on an interactive shell. @@ -2643,25 +2065,25 @@ After the commands run, you are left on an interactive shell. The above command is basically equivalent to: .... -./run -f 'lkmc_eval="insmod /hello.ko;poweroff.out;"' +./run --kernel-cli-after-dash 'lkmc_eval="insmod /hello.ko;poweroff.out;"' .... -where the `lkmc_eval` option gets evaled by our default `S98` startup script if present. +where the `lkmc_eval` option gets evaled by our default link:rootfs_overlay/etc/init.d/S98[S98] startup script. -However, `-F` is smarter and uses `base64` encoding, much like `-E` vs `-e`, so you will just use `-F` most of the time. +Except that `--eval-busybox` is smarter and uses `base64` encoding. -Alternatively, add them to a new `init.d` entry to run at the end o the BusyBox init: +Alternatively, you can also add the comamdns to run to a new `init.d` entry to run at the end o the BusyBox init: .... cp rootfs_overlay/etc/init.d/S98 rootfs_overlay/etc/init.d/S99.gitignore vim rootfs_overlay/etc/init.d/S99.gitignore -./build +./build-buildroot ./run .... and they will be run automatically before the login prompt. -Scripts under `/etc/init.d` are run by `/etc/init.d/rcS`, which gets called by the line `::sysinit:/etc/init.d/rcS` in `/etc/inittab`. +Scripts under `/etc/init.d` are run by `/etc/init.d/rcS`, which gets called by the line `::sysinit:/etc/init.d/rcS` in link:rootfs_overlay/etc/inittab[`/etc/inittab`]. === Path to init @@ -2683,17 +2105,68 @@ ____ And you can try it out with: .... -./run -e 'init=/init_env_poweroff.sh - asdf=qwer zxcv' +./run --kernel-cli 'init=/init_env_poweroff.out - asdf=qwer zxcv' .... -Source: link:rootfs_overlay/init_env_poweroff.sh[]. - -Also note how the annoying dash `-` also gets passed as a parameter to `init`, which makes it impossible to use this method for most executables. - -Finally, the docs are lying, arguments with dots that come after `-` are still treated specially (of the form `subsystem.somevalue`) and disappear: +Output: .... -./run -e 'init=/init_env_poweroff.sh - /poweroff.out' +args: +/init_env_poweroff.out +- +zxcv + +env: +HOME=/ +TERM=linux +asdf=qwer +.... + +Source: link:packages/kernel_modules/user/init_env_poweroff.c[]. + +==== init environment args + +The annoying dash `-` gets passed as a parameter to `init`, which makes it impossible to use this method for most non custom executables. + +Arguments with dots that come after `-` are still treated specially (of the form `subsystem.somevalue`) and disappear, from args, e.g.: + +.... +./run --kernel-cli 'init=/init_env_poweroff.out - /poweroff.out' +.... + +outputs: + +.... +args +/init_env_poweroff.out +- +ab +.... + +so see how `a.b` is gone. + +==== init environment env + +Wait, where do `HOME` and `TERM` come from? (greps the kernel). Ah, OK, the kernel sets those by default: https://github.com/torvalds/linux/blob/94710cac0ef4ee177a63b5227664b38c95bbf703/init/main.c#L173 + +.... +const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, }; +.... + +Furthermore, if you run something inside a shell: + +.... +./run --eval '/usr/bin/env' +.... + +BusyBox also defines `SHLVL` and `PWD=`: + +.... +SHLVL=1 +HOME=/ +TERM=linux +lkmc_eval=L3Vzci9iaW4vZW52 +PWD=/ .... === Networking @@ -2726,14 +2199,265 @@ ping google.com TODO why: https://unix.stackexchange.com/questions/124283/busybox-ping-ip-works-but-hostname-nslookup-fails-with-bad-address -To enable networking by default, use the methods documented at <> +To enable networking by default, use the methods documented at <>. + +== initrd + +The kernel can boot from an CPIO file, which is a directory serialization format much like tar: https://superuser.com/questions/343915/tar-vs-cpio-what-is-the-difference + +The bootloader, which for us is QEMU itself, is then configured to put that CPIO into memory, and tell the kernel that it is there. + +With this setup, you don't even need to give a root filesystem to the kernel, it just does everything in memory in a ramfs. + +To enable initrd instead of the default ext2 disk image, do: + +.... +./build-buildroot --initrd +./run --initrd +.... + +Notice how it boots fine, even though this leads to not giving QEMU the `-drive` option, as can be verified with: + +.... +cat "$(./getvar run_dir)/run.sh" +.... + +Also as expected, there is no filesystem persistency, since we are doing everything in memory: + +.... +date >f +poweroff +cat f +# can't open 'f': No such file or directory +.... + +which can be good for automated tests, as it ensures that you are using a pristine unmodified system image every time. + +One downside of this method is that it has to put the entire filesystem into memory, and could lead to a panic: + +.... +end Kernel panic - not syncing: Out of memory and no killable processes... +.... + +This can be solved by increasing the memory with: + +.... +./run --initrd --memory 256M +.... + +The main ingredients to get initrd working are: + +* `BR2_TARGET_ROOTFS_CPIO=y`: make Buildroot generate `images/rootfs.cpio` in addition to the other images. ++ +It is also possible to compress that image with other options. +* `qemu -initrd`: make QEMU put the image into memory and tell the kernel about it. +* `CONFIG_BLK_DEV_INITRD=y`: Compile the kernel with initrd support, see also: https://unix.stackexchange.com/questions/67462/linux-kernel-is-not-finding-the-initrd-correctly/424496#424496 ++ +Buildroot forces that option when `BR2_TARGET_ROOTFS_CPIO=y` is given + +https://unix.stackexchange.com/questions/89923/how-does-linux-load-the-initrd-image asks how the mechanism works in more detail. + +=== initrd in desktop distros + +Most modern desktop distributions have an initrd in their root disk to do early setup. + +The rationale for this is described at: https://en.wikipedia.org/wiki/Initial_ramdisk + +One obvious use case is having an encrypted root filesystem: you keep the initrd in an unencrypted partition, and then setup decryption from there. + +I think GRUB then knows read common disk formats, and then loads that initrd to memory with a `/boot/grub/grub.cfg` directive of type: + +.... +initrd /initrd.img-4.4.0-108-generic +.... + +Related: https://stackoverflow.com/questions/6405083/initrd-and-booting-the-linux-kernel + +=== initramfs + +initramfs is just like <>, but you also glue the image directly to the kernel image itself. + +So the only argument that QEMU needs is the `-kernel`, no `-drive` not even `-initrd`! Pretty cool. + +Try it out with: + +.... +./build-buildroot --initramfs -l && ./run --initramfs +.... + +The `-l` (ell) should only be used the first time you move to / from a different root filesystem method (ext2 or cpio) to initramfs to overcome: https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-custom-config-file-and-run-make-linux-reconfi + +.... +./build-buildroot --initramfs && ./run --initramfs +.... + +It is interesting to see how this increases the size of the kernel image if you do a: + +.... +ls -lh "$(./getvar linux_image)" +.... + +before and after using initramfs, since the `.cpio` is now glued to the kernel image. + +In the background, it uses `BR2_TARGET_ROOTFS_INITRAMFS`, and this makes the kernel config option `CONFIG_INITRAMFS_SOURCE` point to the CPIO that will be embedded in the kernel image. + +http://nairobi-embedded.org/initramfs_tutorial.html shows a full manual setup. + +=== gem5 initrd + +TODO we were not able to get it working yet: https://stackoverflow.com/questions/49261801/how-to-boot-the-linux-kernel-with-initrd-or-initramfs-with-gem5 + +== Device tree + +The device tree is a Linux kernel defined data structure that serves to inform the kernel how the hardware is setup. + +<> contains a minimal runnable example of device tree manipulation. + +Device trees serve to reduce the need for hardware vendors to patch the kernel: they just provide a device tree file instead, which is much simpler. + +x86 does not use it device trees, but many other archs to, notably ARM. + +This is notably because ARM boards: + +* typically don't have discoverable hardware extensions like PCI, but rather just put everything on an SoC with magic register addresses +* are made by a wide variety of vendors due to ARM's licensing business model, which increases variability + +The Linux kernel itself has several device trees under `./arch//boot/dts`, see also: https://stackoverflow.com/questions/21670967/how-to-compile-dts-linux-device-tree-source-files-to-dtb/42839737#42839737 + +=== DTB files + +Files that contain device trees have the `.dtb` extension when compiled, and `.dts` when in text form. + +You can convert between those formats with: + +.... +sudo apt-get install device-tree-compiler +dtc -I dtb -O dts -o a.dts a.dtb +dtc -I dts -O dtb -o a.dtb a.dts +.... + +See also: https://stackoverflow.com/questions/14000736/tool-to-visualize-the-device-tree-file-dtb-used-by-the-linux-kernel/39931834#39931834 + +Device tree files are provided to the emulator just like the root filesystem and the Linux kernel image. + +In real hardware, those components are also often provided separately. For example, on the Raspberry Pi 2, the SD card must contain two partitions: + +* the first contains all magic files, including the Linux kernel and the device tree +* the second contains the root filesystem + +See also: https://stackoverflow.com/questions/29837892/how-to-run-a-c-program-with-no-os-on-the-raspberry-pi/40063032#40063032 + +=== Device tree syntax + +Good format descriptions: + +* https://www.raspberrypi.org/documentation/configuration/device-tree.md + +Minimal example + +.... +/dts-v1/; + +/ { + a; +}; +.... + +Check correctness with: + +.... +dtc a.dts +.... + +Separate nodes are simply merged by node path, e.g.: + +.... +/dts-v1/; + +/ { + a; +}; + +/ { + b; +}; +.... + +then `dtc a.dts` gives: + +.... +/dts-v1/; + +/ { + a; + b; +}; +.... + +=== Get device tree from running kernel + +https://unix.stackexchange.com/questions/265890/is-it-possible-to-get-the-information-for-a-device-tree-using-sys-of-a-running/330926#330926 + +This is specially interesting because QEMU and gem5 are capable of generating DTBs that match the selected machine depending on dynamic command line parameters for some types of machines. + +QEMU's `-M virt` for example, which we use by default for `aarch64`, boots just fine without the `-dtb` option: + +.... +./run --arch aarch64 +.... + +Then, from inside the guest: + +.... +dtc -I fs -O dts /sys/firmware/devicetree/base +.... + +contains: + +.... + cpus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + cpu@0 { + compatible = "arm,cortex-a57"; + device_type = "cpu"; + reg = <0x0>; + }; + }; +.... + +=== Device tree emulator generation + +Since emulators know everything about the hardware, they can automatically generate device trees for us, which is very convenient. + +This is the case for both QEMU and gem5. + +For example, if we increase the <> to 2: + +.... +./run --arch aarch64 --cpus 2 +.... + +QEMU automatically adds a second CPU to the DTB! + +.... + cpu@0 { + cpu@1 { +.... + +The action seems to be happening at: `hw/arm/virt.c`. + +<> 2a9573f5942b5416fb0570cf5cb6cdecba733392 can also generate its own DTB. + +gem5 can generate DTBs on ARM with `--generate-dtb`, but we don't use that feature as of f8c0502bb2680f2dbe7c1f3d7958f60265347005 because it was buggy. == KVM You can make QEMU or gem5 <> by passing enabling KVM with: .... -./run -K +./run --kvm .... but it was broken in gem5 with pending patches: https://www.mail-archive.com/gem5-users@gem5.org/msg15046.html It fails immediately on: @@ -2758,13 +2482,464 @@ The main use case for `-enable-kvm` in this repository is to test if something t For example, when porting a benchmark to Buildroot, you can first use QEMU's KVM to test that benchmarks is producing the correct results, before analysing them more deeply in gem5, which runs much slower. -== X11 +== Kernel module utilities + +=== insmod + +link:https://git.busybox.net/busybox/tree/modutils/insmod.c?h=1_29_3[Provided by BusyBox]: + +.... +./run --eval-busybox 'insmod /hello.ko' +.... + +=== modprobe + +If you are feeling fancy, you can also insert modules with: + +.... +modprobe hello +.... + +which insmods link:packages/kernel_modules/hello.c[]. + +`modprobe` searches for modules under: + +.... +ls /lib/modules/*/extra/ +.... + +Kernel modules built from the Linux mainline tree with `CONFIG_SOME_MOD=m`, are automatically available with `modprobe`, e.g.: + +.... +modprobe dummy-irq irq=1 +.... + +=== myinsmod + +If you are feeling raw, you can insert and remove modules with our own minimal module inserter and remover! + +.... +# init_module +/myinsmod.out /hello.ko +# finit_module +/myinsmod.out /hello.ko "" 1 +/myrmmod.out hello +.... + +which teaches you how it is done from C code. + +Source: + +* link:packages/kernel_modules/user/myinsmod.c[] +* link:packages/kernel_modules/user/myrmmod.c[] + +The Linux kernel offers two system calls for module insertion: + +* `init_module` +* `finit_module` + +and: + +.... +man init_module +.... + +documents that: + +____ +The finit_module() system call is like init_module(), but reads the module to be loaded from the file descriptor fd. It is useful when the authenticity of a kernel module can be determined from its location in the filesystem; in cases where that is possible, the overhead of using cryptographically signed modules to determine the authenticity of a module can be avoided. The param_values argument is as for init_module(). +____ + +`finit` is newer and was added only in v3.8. More rationale: https://lwn.net/Articles/519010/ + +Bibliography: https://stackoverflow.com/questions/5947286/how-to-load-linux-kernel-modules-from-c-code + +=== kmod + +https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git + +Multi-call executable that implements: `lsmod`, `insmod`, `rmmod`, and other tools on desktop distros such as Ubuntu 16.04, where e.g.: + +.... +ls -l /bin/lsmod +.... + +gives: + +.... +lrwxrwxrwx 1 root root 4 Jul 25 15:35 /bin/lsmod -> kmod +.... + +and: + +.... +dpkg -l | grep -Ei +.... + +contains: + +.... +ii kmod 22-1ubuntu5 amd64 tools for managing Linux kernel modules +.... + +BusyBox also implements its own version of those executables. There are some differences. + +Buildroot also has a kmod package, but we are not using it since BusyBox' version is good enough so far. + +This page will only describe features that differ from kmod to the BusyBox implementation. + +==== module-init-tools + +Name of a predecessor set of tools. + +==== kmod modprobe + +kmod's `modprobe` can also load modules under different names to avoid conflicts, e.g.: + +.... +sudo modprobe vmhgfs -o vm_hgfs +.... + +== Graphics + +Both QEMU and gem5 are capable of outputting graphics to the screen, and taking mouse and keyboard input. + +https://unix.stackexchange.com/questions/307390/what-is-the-difference-between-ttys0-ttyusb0-and-ttyama0-in-linux + +=== QEMU text mode + +Text mode is the default mode for QEMU. + +The opposite of text mode is <> + +In text mode, we just show the serial console directly on the current terminal, without opening a QEMU GUI window. + +You cannot see any graphics from text mode, but text operations in this mode, including: + +* scrolling up: <> +* copy paste to and from the terminal + +making this a good default, unless you really need to use with graphics. + +Text mode works by sending the terminal character by character to a serial device. + +This is different from a display screen, where each character is a bunch of pixels, and it would be much harder to convert that into actual terminal text. + +For more details, see: + +* https://unix.stackexchange.com/questions/307390/what-is-the-difference-between-ttys0-ttyusb0-and-ttyama0-in-linux +* <> + +Note that you can still see an image even in text mode with the VNC: + +.... +./run --vnc +.... + +and on another terminal: + +.... +./vnc +.... + +but there is not terminal on the VNC window, just the <> penguin. + +==== Quit QEMU from text mode + +https://superuser.com/questions/1087859/how-to-quit-the-qemu-monitor-when-not-using-a-gui + +However, our QEMU setup captures Ctrl + C and other common signals and sends them to the guest, which makes it hard to quit QEMU for the first time since there is no GUI either. + +The simplest way to quit QEMU, is to do: + +.... +Ctrl-A X +.... + +Alternative methods include: + +* `quit` command on the <> +* `pkill qemu` + +=== QEMU graphic mode + +Enable graphic mode with: + +.... +./run --graphic +.... + +Outcome: you see a penguin due to <>. + +For a more exciting GUI experience, see: <> + +Text mode is the default due to the following considerable advantages: + +* copy and paste commands and stdout output to / from host +* get full panic traces when you start making the kernel crash :-) See also: https://unix.stackexchange.com/questions/208260/how-to-scroll-up-after-a-kernel-panic +* have a large scroll buffer, and be able to search it, e.g. by using tmux on host +* one less window floating around to think about in addition to your shell :-) +* graphics mode has only been properly tested on `x86_64`. + +Text mode has the following limitations over graphics mode: + +* you can't see graphics such as those produced by <> +* very early kernel messages such as `early console in extract_kernel` only show on the GUI, since at such early stages, not even the serial has been setup. + +`x86_64` has a VGA device enabled by default, as can be seen as: + +.... +./qemumonitor info qtree +.... + +and the Linux kernel picks it up through the link:https://en.wikipedia.org/wiki/Linux_framebuffer[fbdev] graphics system as can be seen from: + +.... +cat /dev/urandom > /dev/fb0 +.... + +flooding the screen with colors. See also: https://superuser.com/questions/223094/how-do-i-know-if-i-have-kms-enabled + +==== Scroll up in graphic mode + +Scroll up in <>: + +.... +Shift-PgUp +.... + +but I never managed to increase that buffer: + +* https://askubuntu.com/questions/709697/how-to-increase-scrollback-lines-in-ubuntu14-04-2-server-edition +* https://unix.stackexchange.com/questions/346018/how-to-increase-the-scrollback-buffer-size-for-tty + +The superior alternative is to use text mode and GNU screen or <>. + +==== QEMU Graphic mode arm + +===== QEMU graphic mode arm terminal + +TODO: on arm, we see the penguin and some boot messages, but don't get a shell at then end: + +.... +./run --arch aarch64 --graphic +.... + +I think it does not work because the graphic window is <> only, i.e.: + +.... +cat /dev/urandom > /dev/fb0 +.... + +fails with: + +.... +cat: write error: No space left on device +.... + +and has no effect, and the Linux kernel does not appear to have a built-in DRM console as it does for fbdev with <>. + +There is however one out-of-tree implementation: <>. + +===== QEMU graphic mode arm terminal implementation + +`arm` and `aarch64` rely on the QEMU CLI option: + +.... +-device virtio-gpu-pci +.... + +and the kernel config options: + +.... +CONFIG_DRM=y +CONFIG_DRM_VIRTIO_GPU=y +.... + +Unlike x86, `arm` and `aarch64` don't have a display device attached by default, thus the need for `virtio-gpu-pci`. + +See also https://wiki.qemu.org/Documentation/Platforms/ARM (recently edited and corrected by yours truly... :-)). + +===== QEMU graphic mode arm VGA + +TODO: how to use VGA on ARM? https://stackoverflow.com/questions/20811203/how-can-i-output-to-vga-through-qemu-arm Tried: + +.... +-device VGA +.... + +But https://github.com/qemu/qemu/blob/v2.12.0/docs/config/mach-virt-graphical.cfg#L264 says: + +.... +# We use virtio-gpu because the legacy VGA framebuffer is +# very troublesome on aarch64, and virtio-gpu is the only +# video device that doesn't implement it. +.... + +so maybe it is not possible? + +=== gem5 Graphic mode + +gem5 does not have a "text mode", since it cannot redirect the Linux terminal to same host terminal where the executable is running: you are always forced to connect to the terminal with `gem-shell`. + +TODO could not get it working on `x86_64`, only ARM. + +Overview: https://stackoverflow.com/questions/50364863/how-to-get-graphical-gui-output-and-user-touch-keyboard-mouse-input-in-a-ful/50364864#50364864 + +More concretely: + +.... +git -C "$(./getvar linux_src_dir)" checkout gem5/v4.15 +./build-buildroot \ + --arch arm \ + --gem5 \ + -l \ + -K linux/arch/arm/configs/gem5_defconfig \ + --linux-build-id gem5-v4.15 \ +; +git -C "$(./getvar linux_src_dir)" checkout - +./run --arch arm --gem5 --linux-build-id gem5-v4.15 +.... + +and then on another shell: + +.... +vinagre localhost:5900 +.... + +The <> penguin only appears after several seconds, together with kernel messages of type: + +.... +[ 0.152755] [drm] found ARM HDLCD version r0p0 +[ 0.152790] hdlcd 2b000000.hdlcd: bound virt-encoder (ops 0x80935f94) +[ 0.152795] [drm] Supports vblank timestamp caching Rev 2 (21.10.2013). +[ 0.152799] [drm] No driver support for vblank timestamp query. +[ 0.215179] Console: switching to colour frame buffer device 240x67 +[ 0.230389] hdlcd 2b000000.hdlcd: fb0: frame buffer device +[ 0.230509] [drm] Initialized hdlcd 1.0.0 20151021 for 2b000000.hdlcd on minor 0 +.... + +The port `5900` is incremented by one if you already have something running on that port, `gem5` stdout tells us the right port on stdout as: + +.... +system.vncserver: Listening for connections on port 5900 +.... + +and when we connect it shows a message: + +.... +info: VNC client attached +.... + +Alternatively, you can also view the frames with `--frame-capture`: + +.... +./run \ + --arch arm \ + --gem5 \ + --linux-build-id gem5-v4.15 \ + -- --frame-capture \ +; +.... + +This option dumps one compressed PNG whenever the screen image changes inside `m5out`, indexed by the cycle ID. This allows for more controlled experiments. + +It is fun to see how we get one new frame whenever the white underscore cursor appears and reappears under the penguin. + +TODO <> failed on `aarch64` with: + +.... +kmscube[706]: unhandled level 2 translation fault (11) at 0x00000000, esr 0x92000006, in libgbm.so.1.0.0[7fbf6a6000+e000] +.... + +Tested on: link:http://github.com/cirosantilli/linux-kernel-module-cheat/commit/38fd6153d965ba20145f53dc1bb3ba34b336bde9[38fd6153d965ba20145f53dc1bb3ba34b336bde9] + +==== Graphic mode gem5 aarch64 + +For `aarch64` we also need `-c kernel_config_fragment/display`: + +.... +git -C "$(./getvar linux_src_dir)" checkout gem5/v4.15 +./build-buildroot \ + --arch aarch64 \ + -c kernel_config_fragment/display \ + --gem5 \ + -K linux/arch/arm64/configs/gem5_defconfig \ + -l \ + --linux-build-id gem5-v4.15 \ +; +git -C "$(./getvar linux_src_dir)" checkout - +./run --arch aarch64 --gem5 --linux-build-id gem5-v4.15 +.... + +This is because the gem5 `aarch64` defconfig does not enable HDLCD like the 32 bit one `arm` one for some reason. + +==== Graphic mode gem5 internals + +We cannot use mainline Linux because the <> are required at least to provide the `CONFIG_DRM_VIRT_ENCODER` option. + +gem5 emulates the link:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0541c/CHDBAIDI.html[HDLCD] ARM Holdings hardware for `arm` and `aarch64`. + +The kernel uses HDLCD to implement the <> interface, the required kernel config options are present at: link:kernel_config_fragment/display[]. + +TODO: minimize out the `-K`. If we just remove it on `arm`: it does not work with a failing dmesg: + +.... +[ 0.066208] [drm] found ARM HDLCD version r0p0 +[ 0.066241] hdlcd 2b000000.hdlcd: bound virt-encoder (ops drm_vencoder_ops) +[ 0.066247] [drm] Supports vblank timestamp caching Rev 2 (21.10.2013). +[ 0.066252] [drm] No driver support for vblank timestamp query. +[ 0.066276] hdlcd 2b000000.hdlcd: Cannot do DMA to address 0x0000000000000000 +[ 0.066281] swiotlb: coherent allocation failed for device 2b000000.hdlcd size=8294400 +[ 0.066288] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.15.0 #1 +[ 0.066293] Hardware name: V2P-AARCH64 (DT) +[ 0.066296] Call trace: +[ 0.066301] dump_backtrace+0x0/0x1b0 +[ 0.066306] show_stack+0x24/0x30 +[ 0.066311] dump_stack+0xb8/0xf0 +[ 0.066316] swiotlb_alloc_coherent+0x17c/0x190 +[ 0.066321] __dma_alloc+0x68/0x160 +[ 0.066325] drm_gem_cma_create+0x98/0x120 +[ 0.066330] drm_fbdev_cma_create+0x74/0x2e0 +[ 0.066335] __drm_fb_helper_initial_config_and_unlock+0x1d8/0x3a0 +[ 0.066341] drm_fb_helper_initial_config+0x4c/0x58 +[ 0.066347] drm_fbdev_cma_init_with_funcs+0x98/0x148 +[ 0.066352] drm_fbdev_cma_init+0x40/0x50 +[ 0.066357] hdlcd_drm_bind+0x220/0x428 +[ 0.066362] try_to_bring_up_master+0x21c/0x2b8 +[ 0.066367] component_master_add_with_match+0xa8/0xf0 +[ 0.066372] hdlcd_probe+0x60/0x78 +[ 0.066377] platform_drv_probe+0x60/0xc8 +[ 0.066382] driver_probe_device+0x30c/0x478 +[ 0.066388] __driver_attach+0x10c/0x128 +[ 0.066393] bus_for_each_dev+0x70/0xb0 +[ 0.066398] driver_attach+0x30/0x40 +[ 0.066402] bus_add_driver+0x1d0/0x298 +[ 0.066408] driver_register+0x68/0x100 +[ 0.066413] __platform_driver_register+0x54/0x60 +[ 0.066418] hdlcd_platform_driver_init+0x20/0x28 +[ 0.066424] do_one_initcall+0x44/0x130 +[ 0.066428] kernel_init_freeable+0x13c/0x1d8 +[ 0.066433] kernel_init+0x18/0x108 +[ 0.066438] ret_from_fork+0x10/0x1c +[ 0.066444] hdlcd 2b000000.hdlcd: Failed to set initial hw configuration. +[ 0.066470] hdlcd 2b000000.hdlcd: master bind failed: -12 +[ 0.066477] hdlcd: probe of 2b000000.hdlcd failed with error -12 +[ +.... + +So what other options are missing from `gem5_defconfig`? It would be cool to minimize it out to better understand the options. + +[[x11]] +=== X11 Buildroot + +Once you've seen the `CONFIG_LOGO` penguin as a sanity check, you can try to go for a cooler X11 Buildroot setup. Build and run: .... -./build -b br2/x11 -./run -x +./build-buildroot --buildroot-config-fragment br2/x11 +./run --graphic .... Inside QEMU: @@ -2780,6 +2955,8 @@ xcalc xeyes .... +Outcome: + image:x11.png[image] We don't build X11 by default because it takes a considerable amount of time (about 20%), and is not expected to be used by most users: you need to pass the `-x` flag to enable it. @@ -2791,7 +2968,7 @@ Not sure how well that graphics stack represents real systems, but if it does it To x11 packages have an `xserver` prefix as in: .... -./build -b br2/x11 -- xserver_xorg-server-reconfigure +./build-buildroot --buildroot-config-fragment br2/x11 -- xserver_xorg-server-reconfigure .... the easiest way to find them out is to just list `"$(./getvar build_dir)/x*`. @@ -2802,7 +2979,7 @@ TODO as of: c2696c978d6ca88e8b8599c92b1beeda80eb62b2 I noticed that `startx` lea [ 2.809104] WARNING: CPU: 0 PID: 51 at drivers/gpu/drm/ttm/ttm_bo_vm.c:304 ttm_bo_vm_open+0x37/0x40 .... -=== X11 mouse not moving +==== X11 Buildroot mouse not moving TODO 9076c1d9bcc13b6efdb8ef502274f846d8d4e6a1 I'm 100% sure that it was working before, but I didn't run it forever, and it stopped working at some point. Needs bisection, on whatever commit last touched x11 stuff. @@ -2844,7 +3021,7 @@ Note that our current link:kernel_confi_fragment sets: for gem5, so you might want to remove those lines to debug this. -=== X11 ARM +==== X11 Buildroot ARM On ARM, `startx` hangs at a message: @@ -2869,112 +3046,6 @@ A friend told me this but I haven't tried it yet: * `xf86-video-modesetting` is likely the missing ingredient, but it does not seem possible to activate it from Buildroot currently without patching things. * `xf86-video-fbdev` should work as well, but we need to make sure fbdev is enabled, and maybe add some line to the `Xorg.conf` -== initrd - -The kernel can boot from an CPIO file, which is a directory serialization format much like tar: https://superuser.com/questions/343915/tar-vs-cpio-what-is-the-difference - -The bootloader, which for us is QEMU itself, is then configured to put that CPIO into memory, and tell the kernel that it is there. - -With this setup, you don't even need to give a root filesystem to the kernel, it just does everything in memory in a ramfs. - -To enable initrd instead of the default ext2 disk image, do: - -.... -./build -i -./run -i -.... - -Notice how it boots fine, even though this leads to not giving QEMU the `-drive` option, as can be verified with: - -.... -cat "$(./getvar run_dir)/run.sh" -.... - -Also as expected, there is no filesystem persistency, since we are doing everything in memory: - -.... -date >f -poweroff -cat f -# can't open 'f': No such file or directory -.... - -which can be good for automated tests, as it ensures that you are using a pristine unmodified system image every time. - -One downside of this method is that it has to put the entire filesystem into memory, and could lead to a panic: - -.... -end Kernel panic - not syncing: Out of memory and no killable processes... -.... - -This can be solved by increasing the memory with: - -.... -./run -im 256M -.... - -The main ingredients to get initrd working are: - -* `BR2_TARGET_ROOTFS_CPIO=y`: make Buildroot generate `images/rootfs.cpio` in addition to the other images. -+ -It is also possible to compress that image with other options. -* `qemu -initrd`: make QEMU put the image into memory and tell the kernel about it. -* `CONFIG_BLK_DEV_INITRD=y`: Compile the kernel with initrd support, see also: https://unix.stackexchange.com/questions/67462/linux-kernel-is-not-finding-the-initrd-correctly/424496#424496 -+ -Buildroot forces that option when `BR2_TARGET_ROOTFS_CPIO=y` is given - -https://unix.stackexchange.com/questions/89923/how-does-linux-load-the-initrd-image asks how the mechanism works in more detail. - -=== initrd in desktop distros - -Most modern desktop distributions have an initrd in their root disk to do early setup. - -The rationale for this is described at: https://en.wikipedia.org/wiki/Initial_ramdisk - -One obvious use case is having an encrypted root filesystem: you keep the initrd in an unencrypted partition, and then setup decryption from there. - -I think GRUB then knows read common disk formats, and then loads that initrd to memory with a `/boot/grub/grub.cfg` directive of type: - -.... -initrd /initrd.img-4.4.0-108-generic -.... - -Related: https://stackoverflow.com/questions/6405083/initrd-and-booting-the-linux-kernel - -=== initramfs - -initramfs is just like <>, but you also glue the image directly to the kernel image itself. - -So the only argument that QEMU needs is the `-kernel`, no `-drive` not even `-initrd`! Pretty cool. - -Try it out with: - -.... -./build -I -l && ./run -I -.... - -The `-l` (ell) should only be used the first time you move to / from a different root filesystem method (ext2 or cpio) to initramfs to overcome: https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-custom-config-file-and-run-make-linux-reconfi - -.... -./build -I && ./run -I -.... - -It is interesting to see how this increases the size of the kernel image if you do a: - -.... -ls -lh "$(./getvar linux_image)" -.... - -before and after using initramfs, since the `.cpio` is now glued to the kernel image. - -In the background, it uses `BR2_TARGET_ROOTFS_INITRAMFS`, and this makes the kernel config option `CONFIG_INITRAMFS_SOURCE` point to the CPIO that will be embedded in the kernel image. - -http://nairobi-embedded.org/initramfs_tutorial.html shows a full manual setup. - -=== gem5 initrd - -TODO we were not able to get it working yet: https://stackoverflow.com/questions/49261801/how-to-boot-the-linux-kernel-with-initrd-or-initramfs-with-gem5 - == Linux kernel === Linux kernel configuration @@ -2989,7 +3060,7 @@ By default, we use a `.config` that is a mixture of: Use just your own exact `.config` instead: .... -./build -K data/myconfig -l +./build-buildroot -K data/myconfig -l .... Beware that Buildroot can `sed` override some of the configurations we make no matter what, e.g. it forces `CONFIG_BLK_DEV_INITRD=y` when `BR2_TARGET_ROOTFS_CPIO` is on, so you might want to double check as explained at <>. TODO check if there is a way to prevent that patching and maybe patch Buildroot for it, it is too fuzzy. People should be able to just build with whatever `.config` they want. @@ -2997,7 +3068,7 @@ Beware that Buildroot can `sed` override some of the configurations we make no m Modify a single option: .... -./build -C 'CONFIG_FORTIFY_SOURCE=y' -l +./build-buildroot -C 'CONFIG_FORTIFY_SOURCE=y' -l .... Use an extra kernel config fragment file: @@ -3007,7 +3078,7 @@ printf ' CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y ' > myconfig -./build -c 'myconfig' -l +./build-buildroot -c 'myconfig' -l .... `-K`, `-c`, `-C` can all be used at the same time. Options passed via `-C` take precedence over `-c`, which takes precedence over `-K`. @@ -3044,7 +3115,7 @@ CONFIG_IKCONFIG_PROC=y From host: .... -cat "$(./getvar linux_custom_dir)/.config" +cat "$(./getvar linux_build_dir)/.config" .... Just for fun link:https://stackoverflow.com/a/14958263/895245[]: @@ -3100,7 +3171,7 @@ cat /proc/version or in the source: .... -cd linux +cd "$(./getvar linux_src_dir)" git log | grep -E ' Linux [0-9]+\.' | head .... @@ -3138,6 +3209,277 @@ Those commits change `BR2_LINUX_KERNEL_LATEST_VERSION` in `/linux/Config.in`. You should then look up if there is a branch that supports that kernel. Staying on branches is a good idea as they will get backports, in particular ones that fix the build as newer host versions come out. +=== Kernel command line parameters + +Bootloaders can pass a string as input to the Linux kernel when it is booting to control its behaviour, much like the `execve` system call does to userland processes. + +This allows us to control the behaviour of the kernel without rebuilding anything. + +With QEMU, QEMU itself acts as the bootloader, and provides the `-append` option and we expose it through `./run --kernel-cli`, e.g.: + +.... +./run --kernel-cli 'foo bar' +.... + +Then inside the host, you can check which options were given with: + +.... +cat /proc/cmdline +.... + +They are also printed at the beginning of the boot message: + +.... +dmesg | grep "Command line" +.... + +See also: + +* https://unix.stackexchange.com/questions/48601/how-to-display-the-linux-kernel-command-line-parameters-given-for-the-current-bo +* https://askubuntu.com/questions/32654/how-do-i-find-the-boot-parameters-used-by-the-running-kernel + +The arguments are documented in the kernel documentation: https://www.kernel.org/doc/html/v4.14/admin-guide/kernel-parameters.html + +When dealing with real boards, extra command line options are provided on some magic bootloader configuration file, e.g.: + +* GRUB configuration files: https://askubuntu.com/questions/19486/how-do-i-add-a-kernel-boot-parameter +* Raspberry pi `/boot/cmdline.txt` on a magic partition: https://raspberrypi.stackexchange.com/questions/14839/how-to-change-the-kernel-commandline-for-archlinuxarm-on-raspberry-pi-effectly + +==== Kernel command line parameters escaping + +Double quotes can be used to escape spaces as in `opt="a b"`, but double quotes themselves cannot be escaped, e.g. `opt"a\"b"` + +This even lead us to use base64 encoding with `--eval`! + +==== Kernel command line parameters definition points + +There are two methods: + +* `__setup` as in: ++ +.... +__setup("console=", console_setup); +.... +* `core_param` as in: ++ +.... +core_param(panic, panic_timeout, int, 0644); +.... + +`core_param` suggests how they are different: + +.... +/** + * core_param - define a historical core kernel parameter. + +... + + * core_param is just like module_param(), but cannot be modular and + * doesn't add a prefix (such as "printk."). This is for compatibility + * with __setup(), and it makes sense as truly core parameters aren't + * tied to the particular file they're in. + */ +.... + +==== norandmaps + +Disable userland address space randomization. Test it out by running <> twice: + +.... +./run --eval-busybox '/rand_check.out;/poweroff.out' +./run --eval-busybox '/rand_check.out;/poweroff.out' +.... + +If we remove it from our link:run[] script by hacking it up, the addresses shown by `rand_check.out` vary across boots. + +Equivalent to: + +.... +echo 0 > /proc/sys/kernel/randomize_va_space +.... + +=== printk + +`printk` is the most simple and widely used way of getting information from the kernel, so you should familiarize yourself with its basic configuration. + +We use `printk` a lot in our kernel modules, and it shows on the terminal by default, along with stdout and what you type. + +Hide all `printk` messages: + +.... +dmesg -n 1 +.... + +or equivalently: + +.... +echo 1 > /proc/sys/kernel/printk +.... + +See also: https://superuser.com/questions/351387/how-to-stop-kernel-messages-from-flooding-my-console + +Do it with a <> to affect the boot itself: + +.... +./run --kernel-cli 'loglevel=5' +.... + +and now only boot warning messages or worse show, which is useful to identify problems. + +Our default `printk` format is: + +.... +[TIMESTAMP] MESSAGE +.... + +e.g.: + +.... +<6>[ 2.979121] Freeing unused kernel memory: 2024K +.... + +where: + +* `LEVEL`: higher means less serious +* `TIMESTAMP`: seconds since boot + +This format is selected by the following boot options: + +* `console_msg_format=syslog`: add the `` part. Added in v4.16. +* `printk.time=y`: add the `[TIMESTAMP]` part + +The debug highest level is a bit more magic, see: <> for more info. + +==== ignore_loglevel + +.... +./run --kernel-cli 'ignore_loglevel' +.... + +enables all log levels, and is basically the same as: + +.... +./run --kernel-cli 'loglevel=8' +.... + +except that you don't need to know what is the maximum level. + +==== pr_debug + +https://stackoverflow.com/questions/28936199/why-is-pr-debug-of-the-linux-kernel-not-giving-any-output/49835405#49835405 + +Debug messages are not printable by default without recompiling. + +But the awesome `CONFIG_DYNAMIC_DEBUG=y` option which we enable by default allows us to do: + +.... +echo 8 > /proc/sys/kernel/printk +echo 'file kernel/module.c +p' > /sys/kernel/debug/dynamic_debug/control +/myinsmod.out /hello.ko +.... + +and we have a shortcut at: + +.... +/pr_debug.sh +.... + +Source: link:rootfs_overlay/pr_debug.sh[]. + +Syntax: https://www.kernel.org/doc/html/v4.11/admin-guide/dynamic-debug-howto.html + +Wildcards are also accepted, e.g. enable all messages from all files: + +.... +echo 'file * +p' > /sys/kernel/debug/dynamic_debug/control +.... + +TODO: why is this not working: + +.... +echo 'func sys_init_module +p' > /sys/kernel/debug/dynamic_debug/control +.... + +Enable messages in specific modules: + +.... +echo 8 > /proc/sys/kernel/printk +echo 'module myprintk +p' > /sys/kernel/debug/dynamic_debug/control +insmod /myprintk.ko +.... + +Source: link:packages/kernel_modules/myprintk.c[] + +This outputs the `pr_debug` message: + +.... +printk debug +.... + +but TODO: it also shows debug messages even without enabling them explicitly: + +.... +echo 8 > /proc/sys/kernel/printk +insmod /myprintk.ko +.... + +and it shows as enabled: + +.... +# grep myprintk /sys/kernel/debug/dynamic_debug/control +/linux-kernel-module-cheat/out/x86_64/buildroot/build/kernel_modules-1.0/./myprintk.c:12 [myprintk]myinit =p "pr_debug\012" +.... + +Enable `pr_debug` for boot messages as well, before we can reach userland and write to `/proc`: + +.... +./run --kernel-cli 'dyndbg="file * +p" loglevel=8' +.... + +Get ready for the noisiest boot ever, I think it overflows the `printk` buffer and funny things happen. + +===== pr_debug != printk(KERN_DEBUG + +When `CONFIG_DYNAMIC_DEBUG` is set, `printk(KERN_DEBUG` is not the exact same as `pr_debug(` since `printk(KERN_DEBUG` messages are visible with: + +.... +./run --kernel-cli 'initcall_debug logleve=8' +.... + +which outputs lines of type: + +.... +<7>[ 1.756680] calling clk_disable_unused+0x0/0x130 @ 1 +<7>[ 1.757003] initcall clk_disable_unused+0x0/0x130 returned 0 after 111 usecs +.... + +which are `printk(KERN_DEBUG` inside `init/main.c` in v4.16. + +Mentioned at: https://stackoverflow.com/questions/37272109/how-to-get-details-of-all-modules-drivers-got-initialized-probed-during-kernel-b + +This likely comes from the ifdef split at `init/main.c`: + +.... +/* If you are writing a driver, please use dev_dbg instead */ +#if defined(CONFIG_DYNAMIC_DEBUG) +#include + +/* dynamic_pr_debug() uses pr_fmt() internally so we don't need it here */ +#define pr_debug(fmt, ...) \ + dynamic_pr_debug(fmt, ##__VA_ARGS__) +#elif defined(DEBUG) +#define pr_debug(fmt, ...) \ + printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) +#else +#define pr_debug(fmt, ...) \ + no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) +#endif +.... + +=== Linux kernel entry point + +`start_kernel` is a good definition of it: ttps://stackoverflow.com/questions/18266063/does-kernel-have-main-function/33422401#33422401 + === Kernel module APIs ==== Kernel module parameters @@ -3157,7 +3499,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/params.c[] +* link:packages/kernel_modules/params.c[] * link:rootfs_overlay/params.sh[] As shown in the example, module parameters can also be read and modified at runtime from <>. @@ -3211,8 +3553,8 @@ Outcome: the test passes: Sources: -* link:kernel_module/dep.c[] -* link:kernel_module/dep2.c[] +* link:packages/kernel_modules/dep.c[] +* link:packages/kernel_modules/dep2.c[] * link:rootfs_overlay/dep.sh[] The kernel deduces dependencies based on the `EXPORT_SYMBOL` that each module uses. @@ -3310,7 +3652,7 @@ name = module_info version = 1.0 .... -Source: link:kernel_module/module_info.c[] +Source: link:packages/kernel_modules/module_info.c[] Some of those are also present on sysfs: @@ -3344,7 +3686,7 @@ vermagic: 4.17.0 SMP mod_unload modversions Module information is stored in a special `.modinfo` section of the ELF file: .... -./runtc readelf -SW "$(./getvar target_dir)/module_info.ko" +./runtc readelf -- -SW "$(./getvar target_dir)/module_info.ko" .... contains: @@ -3356,7 +3698,7 @@ contains: and: .... -./runtc readelf -x .modinfo "$(./getvar build_dir)/module_info.ko" +./runtc readelf -- -x .modinfo "$(./getvar build_dir)/module_info.ko" .... gives: @@ -3396,7 +3738,7 @@ Possible dmesg output: VERMAGIC_STRING = 4.17.0 SMP mod_unload modversions .... -Source: link:kernel_module/vermagic.c[] +Source: link:packages/kernel_modules/vermagic.c[] If we artificially create a mismatch with `MODULE_INFO(vermagic`, the insmod fails with: @@ -3410,7 +3752,7 @@ and `dmesg` says the expected and found vermagic found: vermagic_fail: version magic 'asdfqwer' should be '4.17.0 SMP mod_unload modversions ' .... -Source: link:kernel_module/vermagic_fail.c[] +Source: link:packages/kernel_modules/vermagic_fail.c[] The kernel's vermagic is defined based on compile time configurations at link:https://github.com/torvalds/linux/blob/v4.17/include/linux/vermagic.h#L35[include/linux/vermagic.h]: @@ -3458,7 +3800,7 @@ init_module cleanup_module .... -Source: link:kernel_module/module_init.c[] +Source: link:packages/kernel_modules/module_init.c[] TODO why were `module_init` and `module_exit` created? https://stackoverflow.com/questions/3218320/what-is-the-difference-between-module-init-and-init-module-in-a-linux-kernel-mod @@ -3473,8 +3815,8 @@ insmod /oops.ko Source: -* link:kernel_module/panic.c[] -* link:kernel_module/oops.c[] +* link:packages/kernel_modules/panic.c[] +* link:packages/kernel_modules/oops.c[] A panic can also be generated with: @@ -3489,7 +3831,7 @@ How to generate them: * https://unix.stackexchange.com/questions/66197/how-to-cause-kernel-panic-with-a-single-command * https://stackoverflow.com/questions/23484147/generate-kernel-oops-or-crash-in-the-code -When a panic happens, <> does not work as it normally does, and it is hard to get the logs if on are on <>: +When a panic happens, <> does not work as it normally does, and it is hard to get the logs if on are on <>: * https://superuser.com/questions/848412/scrolling-up-the-failed-screen-with-kernel-panic * https://superuser.com/questions/269228/write-qemu-booting-virtual-machine-output-to-a-file @@ -3565,7 +3907,7 @@ info line *(myinit+0x1d) which gives us the correct line: .... -Line 7 of "/linux-kernel-module-cheat/out/x86_64/buildroot/build/kernel_module-1.0/./panic.c" starts at address 0xbf00001c and ends at 0xbf00002c . +Line 7 of "/linux-kernel-module-cheat/out/x86_64/buildroot/build/kernel_modules-1.0/./panic.c" starts at address 0xbf00001c and ends at 0xbf00002c . .... as explained at: https://stackoverflow.com/questions/8545931/using-gdb-to-convert-addresses-to-lines/27576029#27576029 @@ -3573,10 +3915,10 @@ as explained at: https://stackoverflow.com/questions/8545931/using-gdb-to-conver The exact same thing can be done post mortem with: .... -./runtc gdb \ +./runtc gdb -- \ -batch \ -ex 'info line *(myinit+0x1d)' \ - "$(./getvar build_dir)/kernel_module-1.0/panic.ko" \ + "$(./getvar build_dir)/kernel_modules-1.0/panic.ko" \ ; .... @@ -3619,8 +3961,8 @@ before hanging forever. We can make gem5 ff52563a214c71fcd1e21e9f00ad839612032e3b `fs.py` quit instead of hang with `system.panic_on_panic`: .... -patch -d gem5/gem5 -p1 < patches/manual/gem5-panic.patch -./run -aa -F 'echo c > /proc/sysrq-trigger' -g +patch -d "$(./getvar gem5_src_dir)" -p1 < patches/manual/gem5-panic.patch +./run --arch arm --eval-busybox 'echo c > /proc/sysrq-trigger' --gem5 .... Source: link:patches/manual/gem5-panic.patch[]. @@ -3693,7 +4035,7 @@ If `CONFIG_KALLSYMS=n`, then addresses are shown on traces instead of symbol plu In v4.16 it does not seem possible to configure that at runtime. GDB step debugging with: .... -./run -F 'insmod /dump_stack.ko' -du -U dump_stack +./run --eval-busybox 'insmod /dump_stack.ko' --debug-guest --tmux --tmux-args dump_stack .... shows that traces are printed at `arch/x86/kernel/dumpstack.c`: @@ -3795,7 +4137,7 @@ info line *(myinit+0x18) which gives us the correct line: .... -Line 7 of "/linux-kernel-module-cheat/out/arm/buildroot/build/kernel_module-1.0/./panic.c" starts at address 0xbf00001c and ends at 0xbf00002c . +Line 7 of "/linux-kernel-module-cheat/out/arm/buildroot/build/kernel_modules-1.0/./panic.c" starts at address 0xbf00001c and ends at 0xbf00002c . .... This-did not work on `arm` due to <> so we need to either: @@ -3811,7 +4153,7 @@ The `dump_stack` function produces a stack trace much like panic and oops, but c insmod /dump_stack.ko .... -Source: link:kernel_module/dump_stack.c[] +Source: link:packages/kernel_modules/dump_stack.c[] ==== WARN_ON @@ -3824,7 +4166,7 @@ echo 1 > /proc/sys/kernel/panic_on_warn insmod /warn_on.ko .... -Source: link:kernel_module/warn_on.c[] +Source: link:packages/kernel_modules/warn_on.c[] Can also be activated with the `panic_on_warn` boot parameter. @@ -3856,7 +4198,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/debugfs.c[] +* link:packages/kernel_modules/debugfs.c[] * link:rootfs_overlay/debugfs.sh[] Debugfs is made specifically to help test kernel stuff. Just mount, set <>, and we are done. @@ -3897,7 +4239,7 @@ Procfs can run all system calls, including ones that debugfs can't, e.g. < Sources: -* link:kernel_module/procfs.c[] +* link:packages/kernel_modules/procfs.c[] * link:rootfs_overlay/procfs.sh[] Bibliography: https://stackoverflow.com/questions/8516021/proc-create-example-for-kernel-module/18924359#18924359 @@ -3919,7 +4261,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/sysfs.c[] +* link:packages/kernel_modules/sysfs.c[] * link:rootfs_overlay/sysfs.sh[] Vs procfs: @@ -3960,7 +4302,7 @@ Sources: * link:rootfs_overlay/character_device.sh[] * link:rootfs_overlay/mknoddev.sh[] -* link:kernel_module/character_device.c[] +* link:packages/kernel_modules/character_device.c[] Unlike <> entires, character device files are created with userland `mknod` or `mknodat` syscalls: @@ -4013,7 +4355,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/character_device_create.c[] +* link:packages/kernel_modules/character_device_create.c[] * link:rootfs_overlay/character_device_create.sh[] Bibliography: https://stackoverflow.com/questions/5970595/how-to-create-a-device-node-from-the-init-module-code-of-a-linux-kernel-module/45531867#45531867 @@ -4039,7 +4381,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/fops.c[] +* link:packages/kernel_modules/fops.c[] * link:rootfs_overlay/fops.sh[] Then give this a try: @@ -4069,7 +4411,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/seq_file.c[] +* link:packages/kernel_modules/seq_file.c[] * link:rootfs_overlay/seq_file.sh[] In this example we create a debugfs file that behaves just like a file that contains: @@ -4106,7 +4448,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/seq_file_single_open.c[] +* link:packages/kernel_modules/seq_file_single_open.c[] * link:rootfs_overlay/seq_file_single_open.sh[] This example produces a debugfs file that behaves like a file that contains: @@ -4128,8 +4470,8 @@ Outcome: `jiffies` gets printed to stdout every second from userland. Sources: -* link:kernel_module/poll.c[] -* link:kernel_module/poll.c[] +* link:packages/kernel_modules/poll.c[] +* link:packages/kernel_modules/poll.c[] * link:rootfs_overlay/poll.sh[] Typically, we are waiting for some hardware to make some piece of data available available to the kernel. @@ -4157,9 +4499,9 @@ Outcome: the test passes: Sources: -* link:kernel_module/ioctl.c[] -* link:kernel_module/ioctl.h[] -* link:kernel_module/user/ioctl.c[] +* link:packages/kernel_modules/ioctl.c[] +* link:packages/kernel_modules/ioctl.h[] +* link:packages/kernel_modules/user/ioctl.c[] * link:rootfs_overlay/ioctl.sh[] `ioctl` is one of the most important methods of communication with real device drivers, which often take several fields as input. @@ -4206,8 +4548,8 @@ Outcome: the test passes: Sources: -* link:kernel_module/mmap.c[] -* link:kernel_module/user/mmap.c[] +* link:packages/kernel_modules/mmap.c[] +* link:packages/kernel_modules/user/mmap.c[] * link:rootfs_overlay/mmap.sh[] In this example, we make a tiny 4 byte kernel buffer available to user-space, and we then modify it on userspace, and check that the kernel can see the modification. @@ -4242,9 +4584,9 @@ Outcome: the test passes: Sources: -* link:kernel_module/anonymous_inode.c[] -* link:kernel_module/anonymous_inode.h[] -* link:kernel_module/user/anonymous_inode.c[] +* link:packages/kernel_modules/anonymous_inode.c[] +* link:packages/kernel_modules/anonymous_inode.h[] +* link:packages/kernel_modules/user/anonymous_inode.c[] * link:rootfs_overlay/anonymous_inode.sh[] This example gets an anonymous inode via <> from a debugfs entry by using `anon_inode_getfd`. @@ -4270,9 +4612,9 @@ Outcome: the test passes: Sources: -* link:kernel_module/netlink.c[] -* link:kernel_module/netlink.h[] -* link:kernel_module/user/netlink.c[] +* link:packages/kernel_modules/netlink.c[] +* link:packages/kernel_modules/netlink.h[] +* link:packages/kernel_modules/user/netlink.c[] * link:rootfs_overlay/netlink.sh[] Launch multiple user requests in parallel to stress our socket: @@ -4297,7 +4639,7 @@ Kernel threads are managed exactly like userland threads; they also have a backi insmod /kthread.ko .... -Source: link:kernel_module/kthread.c[] +Source: link:packages/kernel_modules/kthread.c[] Outcome: dmesg counts from `0` to `9` once every second infinitely many times: @@ -4335,7 +4677,7 @@ Let's launch two threads and see if they actually run in parallel: insmod /kthreads.ko .... -Source: link:kernel_module/kthreads.c[] +Source: link:packages/kernel_modules/kthreads.c[] Outcome: two threads count to dmesg from `0` to `9` in parallel. @@ -4369,7 +4711,7 @@ Count to dmesg every one second from `0` up to `n - 1`: insmod /sleep.ko n=5 .... -Source: link:kernel_module/sleep.c[] +Source: link:packages/kernel_modules/sleep.c[] The sleep is done with a call to link:https://github.com/torvalds/linux/blob/v4.17/kernel/time/timer.c#L1984[`usleep_range`] directly inside `module_init` for simplicity. @@ -4394,7 +4736,7 @@ Stop counting: rmmod workqueue_cheat .... -Source: link:kernel_module/workqueue_cheat.c[] +Source: link:packages/kernel_modules/workqueue_cheat.c[] The workqueue thread is killed after the worker function returns. @@ -4418,19 +4760,19 @@ rmmod work_from_work The sleep is done indirectly through: link:https://github.com/torvalds/linux/blob/v4.17/include/linux/workqueue.h#L522[`queue_delayed_work`], which waits the specified time before scheduling the work. -Source: link:kernel_module/work_from_work.c[] +Source: link:packages/kernel_modules/work_from_work.c[] ==== schedule Let's block the entire kernel! Yay: ..... -./run -F 'dmesg -n 1;insmod /schedule.ko schedule=0' +./run --eval-busybox 'dmesg -n 1;insmod /schedule.ko schedule=0' ..... Outcome: the system hangs, the only way out is to kill the VM. -Source: link:kernel_module/schedule.c[] +Source: link:packages/kernel_modules/schedule.c[] kthreads only allow interrupting if you call `schedule()`, and the `schedule=0` <> turns it off. @@ -4439,7 +4781,7 @@ Sleep functions like `usleep_range` also end up calling schedule. If we allow `schedule()` to be called, then the system becomes responsive: ..... -./run -F 'dmesg -n 1;insmod /schedule.ko schedule=1' +./run --eval-busybox 'dmesg -n 1;insmod /schedule.ko schedule=1' ..... @@ -4452,7 +4794,7 @@ dmesg -w The system also responds if we <>: .... -./run -c 2 -F 'dmesg -n 1;insmod /schedule.ko schedule=0' +./run --cpus 2 --eval-busybox 'dmesg -n 1;insmod /schedule.ko schedule=0' .... ==== Wait queues @@ -4486,7 +4828,7 @@ Stop the count: rmmod wait_queue .... -Source: link:kernel_module/wait_queue.c[] +Source: link:packages/kernel_modules/wait_queue.c[] This example launches three threads: @@ -4514,7 +4856,7 @@ Stop counting: rmmod timer .... -Source: link:kernel_module/timer.c[] +Source: link:packages/kernel_modules/timer.c[] Timers are callbacks that run when an interrupt happens, from the interrupt context itself. @@ -4532,10 +4874,10 @@ Bibliography: Brute force monitor every shared interrupt that will accept us: .... -./run -F 'insmod /irq.ko' -x +./run --eval-busybox 'insmod /irq.ko' --graphic .... -Source: link:kernel_module/irq.c[]. +Source: link:packages/kernel_modules/irq.c[]. Now try the following: @@ -4608,7 +4950,7 @@ CONFIG_DUMMY_IRQ=m And then you can do .... -./run -x +./run --graphic .... and in guest: @@ -4641,7 +4983,7 @@ So so see something interesting, you need to monitor an interrupt that is more r ==== /proc/interrupts -In the guest on <>: +In the guest with <>: .... watch -n 1 cat /proc/interrupts @@ -4679,7 +5021,7 @@ Outcome: the test passes: Sources: -* link:kernel_module/kstrto.c[] +* link:packages/kernel_modules/kstrto.c[] * link:rootfs_overlay/kstrto.sh[] Bibliography: https://stackoverflow.com/questions/6139493/how-convert-char-to-int-in-linux-kernel/49811658#49811658 @@ -4693,7 +5035,7 @@ insmod /virt_to_phys.ko cat /sys/kernel/debug/lkmc_virt_to_phys .... -Source: link:kernel_module/virt_to_phys.c[] +Source: link:packages/kernel_modules/virt_to_phys.c[] Sample output: @@ -4750,7 +5092,7 @@ First get a virtual address to play with: /virt_to_phys_test.out & .... -Source: link:kernel_module/user/virt_to_phys_test.c[] +Source: link:packages/kernel_modules/user/virt_to_phys_test.c[] Sample output: @@ -4777,7 +5119,7 @@ Sample output physical address: 0x7c7b800 .... -Source: link:kernel_module/user/virt_to_phys_user.c[] +Source: link:packages/kernel_modules/user/virt_to_phys_user.c[] Now we can verify that `virt_to_phys_user.out` gave the correct physical address in the following ways: @@ -4894,7 +5236,7 @@ vaddr pfn soft-dirty file/shared swapped present library 7ffff78ec000 1fd4 0 1 0 1 /lib/libuClibc-1.0.30.so .... -Source: link:kernel_module/user/pagemap_dump.c[] +Source: link:packages/kernel_modules/user/pagemap_dump.c[] Adapted from: https://github.com/dwks/pagemap/blob/8a25747bc79d6080c8b94eac80807a4dceeda57a/pagemap2.c @@ -4903,7 +5245,7 @@ Meaning of the flags: * `vaddr`: first virtual address of a page the belongs to the process. Notably: + .... -./runtc readelf -l "$(./getvar build_dir)/kernel_module-1.0/user/virt_to_phys_test.out" +./runtc readelf -- -l "$(./getvar build_dir)/kernel_modules-1.0/user/virt_to_phys_test.out" .... + contains: @@ -4970,7 +5312,7 @@ I hope to have examples of all methods some day, since I'm obsessed with visibil ==== CONFIG_PROC_EVENTS -Logs proc events such as process creation to a link:kernel_module/netlink.c[netlink socket]. +Logs proc events such as process creation to a link:packages/kernel_modules/netlink.c[netlink socket]. We then have a userland program that listens to the events and prints them out: @@ -4989,7 +5331,7 @@ a # .... -Source: link:kernel_module/user/proc_events.c[] +Source: link:packages/kernel_modules/user/proc_events.c[] TODO: why `exit: tid=79` shows after `exit: tid=80`? @@ -5015,8 +5357,8 @@ TODO can you get process data such as UID and process arguments? It seems not si 0111ca406bdfa6fd65a2605d353583b4c4051781 was failing with: .... ->>> kernel_module 1.0 Building -/usr/bin/make -j8 -C '/linux-kernel-module-cheat//out/aarch64/buildroot/build/kernel_module-1.0/user' BR2_PACKAGE_OPENBLAS="" CC="/linux-kernel-module-cheat//out/aarch64/buildroot/host/bin/aarch64-buildroot-linux-uclibc-gcc" LD="/linux-kernel-module-cheat//out/aarch64/buildroot/host/bin/aarch64-buildroot-linux-uclibc-ld" +>>> kernel_modules 1.0 Building +/usr/bin/make -j8 -C '/linux-kernel-module-cheat//out/aarch64/buildroot/build/kernel_modules-1.0/user' BR2_PACKAGE_OPENBLAS="" CC="/linux-kernel-module-cheat//out/aarch64/buildroot/host/bin/aarch64-buildroot-linux-uclibc-gcc" LD="/linux-kernel-module-cheat//out/aarch64/buildroot/host/bin/aarch64-buildroot-linux-uclibc-ld" /linux-kernel-module-cheat//out/aarch64/buildroot/host/bin/aarch64-buildroot-linux-uclibc-gcc -ggdb3 -fopenmp -O0 -std=c99 -Wall -Werror -Wextra -o 'proc_events.out' 'proc_events.c' In file included from /linux-kernel-module-cheat//out/aarch64/buildroot/host/aarch64-buildroot-linux-uclibc/sysroot/usr/include/signal.h:329:0, from proc_events.c:12: @@ -5166,7 +5508,7 @@ TODO: can you get function arguments? https://stackoverflow.com/questions/276087 kprobes is an instrumentation mechanism that injects arbitrary code at a given address in a trap instruction, much like GDB. Oh, the good old kernel. :-) .... -./build -C 'CONFIG_KPROBES=y' +./build-buildroot -C 'CONFIG_KPROBES=y' .... Then on guest: @@ -5185,7 +5527,7 @@ Outcome: dmesg outputs on every fork: <_do_fork> post_handler: p->addr = 0x00000000e1360063, flags = 0x246 .... -Source: link:kernel_module/kprobe_example.c[] +Source: link:packages/kernel_modules/kprobe_example.c[] TODO: it does not work if I try to immediately launch `sleep`, why? @@ -5248,24 +5590,24 @@ Results (boot not excluded): QEMU: .... -./trace-boot -a x86_64 +./trace-boot --arch x86_64 .... sample output: .... -instruction count all: 1833863 -entry address: 0x1000000 -instruction count firmware: 20708 +instructions 1833863 +entry_address 0x1000000 +instructions_firmware 20708 .... gem5: .... -./run -a aarch64 -g -E 'm5 exit' +./run --arch aarch64 --gem5 --eval 'm5 exit' # Or: -# ./run -a aarch64 -g -E 'm5 exit' -- --cpu-type=HPI --caches -./gem5-stat -a aarch64 sim_insts +# ./run --arch aarch64 --gem5 --eval 'm5 exit' -- --cpu-type=HPI --caches +./gem5-stat --arch aarch64 sim_insts .... Notes: @@ -5275,7 +5617,7 @@ Notes: It can be found from: + .... -./runtc readelf -e "$(./getvar vmlinux)" | grep Entry +./runtc readelf -- -e "$(./getvar vmlinux)" | grep Entry .... + TODO confirm further. If I try to break there with: @@ -5294,11 +5636,11 @@ It only appears once on every log I've seen so far, checked with `grep 0x1000000 + Then when we count the instructions that run before the kernel entry point, there is only about 100k instructions, which is insignificant compared to the kernel boot itself. + -TODO `-a arm` and `-a aarch64` does not count firmware instructions properly because the entry point address of the ELF file does not show up on the trace at all. +TODO `--arch arm` and `--arch aarch64` does not count firmware instructions properly because the entry point address of the ELF file (`ffffff8008080000` for `aarch64`) does not show up on the trace at all. Tested on link:http://github.com/cirosantilli/linux-kernel-module-cheat/commit/f8c0502bb2680f2dbe7c1f3d7958f60265347005[f8c0502bb2680f2dbe7c1f3d7958f60265347005]. * We can also discount the instructions after `init` runs by using `readelf` to get the initial address of `init`. One easy way to do that now is to just run: + .... -./rungdb-user kernel_module-1.0/user/poweroff.out main +./rungdb-user kernel_modules-1.0/user/poweroff.out main .... + And get that from the traces, e.g. if the address is `4003a0`, then we search: @@ -5327,8 +5669,8 @@ Make it harder to get hacked and easier to notice that you were, at the cost of Detects buffer overflows for us: .... -./build -C 'CONFIG_FORTIFY_SOURCE=y' -L fortify -k -./run -F 'insmod /strlen_overflow.ko' -L fortify +./build-buildroot -C 'CONFIG_FORTIFY_SOURCE=y' --linux-build-id fortify --kernel-modules +./run --eval-busybox 'insmod /strlen_overflow.ko' --linux-build-id fortify .... Possible dmesg output: @@ -5349,7 +5691,7 @@ TODO not always reproducible. Find a more reproducible failure. I could not obse insmod /memcpy_overflow.ko .... -Source: link:kernel_module/strlen_overflow.c[] +Source: link:packages/kernel_modules/strlen_overflow.c[] Bibliography: https://www.reddit.com/r/hacking/comments/8h4qxk/what_a_buffer_overflow_in_the_linux_kernel_looks/ @@ -5391,7 +5733,7 @@ UIO interface in a nutshell: Sources: -* link:kernel_module/user/uio_read.c[] +* link:packages/kernel_modules/user/uio_read.c[] * link:rootfs_overlay/uio_read.sh[] Bibliography: @@ -5414,7 +5756,7 @@ Bibliography: [[fbcon]] ==== Linux kernel console fun -Requires <>. +Requires <>. You can also try those on the `Ctrl-Alt-F3` of your Ubuntu host, but it is much more fun inside a VM! @@ -5441,7 +5783,7 @@ TODO: font and keymap. Mentioned at: https://cmcenroe.me/2017/05/05/linux-consol ==== Linux kernel magic keys -Requires <>. +Requires <>. Let's have some fun. @@ -5519,10 +5861,10 @@ echo 0 > /proc/sys/kernel/ctrl-alt-del Minimal example: .... -./run -e 'init=/ctrl_alt_del.out' -x +./run --kernel-cli 'init=/ctrl_alt_del.out' --graphic .... -Source: link:kernel_module/user/ctrl_alt_del.c[] +Source: link:packages/kernel_modules/user/ctrl_alt_del.c[] When you hit `Ctrl-Alt-Del` in the guest, our tiny init handles a `SIGINT` sent by the kernel and outputs to stdout: @@ -5630,8 +5972,8 @@ tty63::respawn:-/bin/sh # Leave one serial empty. #::respawn:/sbin/getty -L ttyS3 0 vt100 ' >> rootfs_overlay/etc/inittab -./build -./run -x -- \ +./build-buildroot +./run --graphic -- \ -serial telnet::1235,server,nowait \ -serial vc:800x600 \ -serial telnet::1236,server,nowait \ @@ -5772,7 +6114,7 @@ getty: setsid: Operation not permitted The following however works: .... -./run -E 'getty 0 tty1 & getty 0 tty2 & getty 0 tty3 & sleep 99999999' -x +./run --eval 'getty 0 tty1 & getty 0 tty2 & getty 0 tty3 & sleep 99999999' --graphic .... presumably because it is being called from `init` directly? @@ -5807,7 +6149,7 @@ Instead, the shell appears on `/dev/tty7`. ==== CONFIG_LOGO -If you run in <>, then you get a Penguin image for <> above the console! https://askubuntu.com/questions/80938/is-it-possible-to-get-the-tux-logo-on-the-text-based-boot +If you run in <>, then you get a Penguin image for <> above the console! https://askubuntu.com/questions/80938/is-it-possible-to-get-the-tux-logo-on-the-text-based-boot This is due to the link:https://github.com/torvalds/linux/blob/v4.17/drivers/video/logo/Kconfig#L5[`CONFIG_LOGO=y`] option which we enable by default. @@ -5816,7 +6158,7 @@ This is due to the link:https://github.com/torvalds/linux/blob/v4.17/drivers/vid When `CONFIG_LOGO=y` is set, the logo can be disabled at boot with: .... -./run -e 'logo.nologo' +./run --kernel-cli 'logo.nologo' .... * https://stackoverflow.com/questions/39872463/how-can-i-disable-the-startup-penguins-and-boot-text-on-linaro-ubuntu @@ -5832,19 +6174,19 @@ Looks like a recompile is needed to modify the image... DRM / DRI is the new interface that supersedes `fbdev`: .... -./build -B 'BR2_PACKAGE_LIBDRM=y' -k -./run -F '/libdrm_modeset.out' -x +./build-buildroot --buildroot-config 'BR2_PACKAGE_LIBDRM=y' --kernel-modules +./run --eval-busybox '/libdrm_modeset.out' --graphic .... -Source: link:kernel_module/user/libdrm_modeset.c[] +Source: link:packages/kernel_modules/user/libdrm_modeset.c[] Outcome: for a few seconds, the screen that contains the terminal gets taken over by changing colors of the rainbow. TODO not working for `aarch64`, it takes over the screen for a few seconds and the kernel messages disappear, but the screen stays black all the time. .... -./build -B 'BR2_PACKAGE_LIBDRM=y' -k -./run -F '/libdrm_modeset.out' -x +./build-buildroot --buildroot-config 'BR2_PACKAGE_LIBDRM=y' --kernel-modules +./run --eval-busybox '/libdrm_modeset.out' --graphic .... <> however worked, which means that it must be a bug with this demo? @@ -5863,13 +6205,13 @@ crw------- 1 root root 226, 0 May 28 09:41 card0 Try creating new displays: .... -./run -a A -x -- -device virtio-gpu-pci +./run --arch aarch64 --graphic -- -device virtio-gpu-pci .... to see multiple `/dev/dri/cardN`, and then use a different display with: .... -./run -F '/libdrm_modeset.out' -x +./run --eval-busybox '/libdrm_modeset.out' --graphic .... Bibliography: @@ -5884,7 +6226,7 @@ Tested on: link:http://github.com/cirosantilli/linux-kernel-module-cheat/commit/ ==== kmscube .... -./build -b br2/kmscube +./build-buildroot --buildroot-config-fragment br2/kmscube .... Outcome: a colored spinning cube coded in OpenGL + EGL takes over your display and spins forever: https://www.youtube.com/watch?v=CqgJMgfxjsk @@ -5951,9 +6293,9 @@ TODO get working. Looks like a more raw alternative to libdrm: .... -./build -B 'BR2_PACKABE_LIBDRI2=y' -wget -O kernel_module/user/dri2test.c https://raw.githubusercontent.com/robclark/libdri2/master/test/dri2test.c -./build -k +./build-buildroot --buildroot-config 'BR2_PACKABE_LIBDRI2=y' +wget -O packages/kernel_modules/user/dri2test.c https://raw.githubusercontent.com/robclark/libdri2/master/test/dri2test.c +./build-buildroot --kernel-modules .... but then I noticed that that example requires multiple files, and I don't feel like integrating it into our build. @@ -5975,7 +6317,7 @@ C userland test suite. Buildroot already has a package, so it is trivial to build it: .... -./build -B 'BR2_PACKAGE_LTP_TESTSUITE=y' +./build-buildroot --buildroot-config 'BR2_PACKAGE_LTP_TESTSUITE=y' .... Then try it out with: @@ -5994,8 +6336,8 @@ TODO a large chunk of tests, the Open POSIX test suite, is disabled with a comme POSIX userland stress. Two versions: .... -./build -B 'BR2_PACKAGE_STRESS=y' -./build -B 'BR2_PACKAGE_STRESS_NG=y' +./build-buildroot --buildroot-config 'BR2_PACKAGE_STRESS=y' +./build-buildroot --buildroot-config 'BR2_PACKAGE_STRESS_NG=y' .... Websites: @@ -6027,6 +6369,73 @@ kill %1 Some QEMU specific features to play with and limitations to cry over. +=== Disk persistency + +We disable disk persistency for both QEMU and gem5 by default, to prevent the emulator from putting the image in an unknown state. + +For QEMU, this is done by passing the `snapshot` option to `-drive`, and for gem5 it is the default behaviour. + +If you hack up our link:run[] script to remove that option, then: + +.... +./run --eval-busybox 'date >f;poweroff' + +.... + +followed by: + +.... +./run --eval-busybox 'cat f' +.... + +gives the date, because `poweroff` without `-n` syncs before shutdown. + +The `sync` command also saves the disk: + +.... +sync +.... + +When you do: + +.... +./build-buildroot +.... + +the disk image gets overwritten by a fresh filesystem and you lose all changes. + +Remember that if you forcibly turn QEMU off without `sync` or `poweroff` from inside the VM, e.g. by closing the QEMU window, disk changes may not be saved. + +Persistency is also turned off when booting from <> with a CPIO instead of with a disk. + +Disk persistency is useful to re-run shell commands from the history of a previous session with `Ctrl-R`, but we felt that the loss of determinism was not worth it. + +==== gem5 disk persistency + +TODO how to make gem5 disk writes persistent? + +As of cadb92f2df916dbb47f428fd1ec4932a2e1f0f48 there are some `read_only` entries in the `config.ini` under cow sections, but hacking them to true did not work: + +.... +diff --git a/configs/common/FSConfig.py b/configs/common/FSConfig.py +index 17498c42b..76b8b351d 100644 +--- a/configs/common/FSConfig.py ++++ b/configs/common/FSConfig.py +@@ -60,7 +60,7 @@ os_types = { 'alpha' : [ 'linux' ], + } + + class CowIdeDisk(IdeDisk): +- image = CowDiskImage(child=RawDiskImage(read_only=True), ++ image = CowDiskImage(child=RawDiskImage(read_only=False), + read_only=False) + + def childImage(self, ci): +.... + +The directory of interest is `src/dev/storage`. + +qcow2 does not appear supported, there are not hits in the source tree, and there is a mention on Nate's 2009 wishlist: http://gem5.org/Nate%27s_Wish_List + === Snapshot QEMU allows us to take snapshots at any time through the monitor. @@ -6038,7 +6447,7 @@ qcow2 filesystems must be used for that to work. To test it out, login into the VM with and run: .... -./run -F 'umount /mnt/9p /mnt/out;/count.sh' +./run --eval-busybox 'umount /mnt/9p /mnt/out;/count.sh' .... On another shell, take a snapshot: @@ -6168,7 +6577,7 @@ insmod /pci_min.ko Sources: -* Kernel module: link:kernel_module/pci_min.c[]. +* Kernel module: link:packages/kernel_modules/pci_min.c[]. * QEMU device: https://github.com/cirosantilli/qemu/blob/lkmc/hw/misc/lkmc_pci_min.c Outcome: @@ -6188,7 +6597,7 @@ What happened: * our hardware model is coded such that it generates an interrupt when written to * the Linux kernel interrupt handler write to another register, which tells the hardware to stop sending interrupts -Kernel messages and printks from inside QEMU are shown all together, to see that more clearly, run in <> instead. +Kernel messages and printks from inside QEMU are shown all together, to see that more clearly, run in <> instead. We don't enable the device by default because it does not work for vanilla QEMU, which we often want to test with this repository. @@ -6207,7 +6616,7 @@ This tests a lot of features of the edu device, to understand the results, compa Sources: -* kernel module: link:kernel_module/qemu_edu.c[] +* kernel module: link:packages/kernel_modules/qemu_edu.c[] * QEMU device: https://github.com/qemu/qemu/blob/v2.12.0/hw/misc/edu.c * test script: link:rootfs_overlay/qemu_edu.sh[] @@ -6449,7 +6858,7 @@ where link:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0190b/ Then compile with: .... -./build -aa -b br2/gpio -c kernel_config_fragment/gpio -l +./build-buildroot --arch arm --buildroot-config-fragment br2/gpio -c kernel_config_fragment/gpio -l .... then test it out with: @@ -6462,8 +6871,6 @@ Source: link:rootfs_overlay/gpio.sh[] Buildroot's Linux tools package provides some GPIO CLI tools: `lsgpio`, `gpio-event-mon`, `gpio-hammer`, TODO document them here. -Those broke MIPS build in 2017-02: https://bugs.busybox.net/show_bug.cgi?id=10276 and so we force disable them in our MIPS build currently. - ==== LEDs TODO: broken when `arm` moved to `-M virt`, same as <>. @@ -6484,7 +6891,7 @@ static void arm_sysctl_write(void *opaque, hwaddr offset, and then rebuild with: .... -./build -aa -c kernel_config_fragment/leds -lq +./build-buildroot --arch arm -c kernel_config_fragment/leds -lq .... But beware that one of the LEDs has a heartbeat trigger by default (specified on dts), so it will produce a lot of output. @@ -6516,15 +6923,14 @@ Using this device now requires checking out to the branch: .... git checkout platform-device +git submodule sync .... before building, it does not work on master. -The module itself can be found at: https://github.com/cirosantilli/linux-kernel-module-cheat/blob/platform-device/kernel_module/platform_device.c +Rationale: we found out that the kernels that build for `qemu -M versatilepb` don't work on gem5 because `versatilepb` is an old pre-v7 platform, and gem5 requires armv7. So we migrated over to `-M virt` to have a single kernel for both gem5 and QEMU, and broke this since the single kernel was more important. TODO port to `-M virt`. -Rationale: we found out that the kernels that build for `qemu -M versatilepb` don't work on gem5 because `versatilepb` is an old pre-v7 platform, and gem5 requires armv7. - -At the same time, we also found out that Versatile Express (`vexpress`) does support armv7, so maybe we could port it over, but I had lost interest at that point, and decided to just go with the simpler `-M virt` machine instead. +The module itself can be found at: https://github.com/cirosantilli/linux-kernel-module-cheat/blob/platform-device/kernel_modules/platform_device.c Uses: @@ -6703,7 +7109,7 @@ We us this exact procedure to connect to <>. Not enabled by default due to the build / runtime overhead. To enable, build with: .... -./build -B 'BR2_PACKAGE_OPENSSH=y' +./build-buildroot --buildroot-config 'BR2_PACKAGE_OPENSSH=y' .... Then inside the guest turn on sshd: @@ -6744,19 +7150,21 @@ Bibliography: https://serverfault.com/questions/769874/how-to-forward-a-port-fro ==== Secondary disk -A simpler and possibly less overhead alternative to <<9P>> would be to generate a secondary disk image with the bencmark you want to rebuild. +A simpler and possibly less overhead alternative to <<9P>> would be to generate a secondary disk image with the benchmark you want to rebuild. Then you can `umount` and re-mount on guest without reboot. We don't support this yet, but it should not be too hard to hack it up, maybe by hooking into link:rootfs_post_build_script[]. +This was not possible from gem5 `fs.py` as of 60600f09c25255b3c8f72da7fb49100e2682093a: https://stackoverflow.com/questions/50862906/how-to-attach-multiple-disk-images-in-a-simulation-with-gem5-fs-py/51037661#51037661 + === QEMU user mode This has nothing to do with the Linux kernel, but it is cool: .... sudo apt-get install qemu-user -./build -a arm +./build-buildroot --arch arm cd "$(./getvar target_dir)" qemu-arm -L . bin/ls .... @@ -6768,7 +7176,7 @@ The reason this is cool, is that `ls` is not statically compiled, but since we h In other words, much cooler than: .... -./runtc -a arm gcc -static ./kernel_module/user/hello.c +./runtc --arch arm gcc -- -static ./packages/kernel_modules/user/hello.c qemu-arm a.out .... @@ -6785,13 +7193,13 @@ Anyways, this warns us that the userland emulation will likely not be reliable, GDB step debugging is also possible with: .... -cd "$(./getvar -a arm target_dir)" -qemu-arm -g 1234 -L . ../build/kernel_module-1.0/user/myinsmod.out +cd "$(./getvar --arch arm target_dir)" +qemu-arm -g 1234 -L . ../build/kernel_modules-1.0/user/myinsmod.out ../host/usr/bin/arm-buildroot-linux-uclibcgnueabihf-gdb \ --nh \ -ex 'set architecture arm' \ -ex 'set sysroot .' \ - -ex 'file ../build/kernel_module-1.0/user/myinsmod.out' \ + -ex 'file ../build/kernel_modules-1.0/user/myinsmod.out' \ -ex 'target remote localhost:1234' \ -ex 'break main' \ -ex 'continue' \ @@ -6819,12 +7227,12 @@ First we try some `-static` sanity checks. Works and prints `hello`: .... -./runtc -a x86_64 gcc -static -o x86_64.out ./kernel_module/user/hello.c -./runtc -a arm gcc -static -o arm.out ./kernel_module/user/hello.c -./runtc -a aarch64 gcc -static -o aarch64.out ./kernel_module/user/hello.c -"$(./getvar -a x86_64 -g exec)" ./gem5/gem5/configs/example/se.py -c ./x86_64.out -"$(./getvar -a arm -g exec)" ./gem5/gem5/configs/example/se.py -c ./arm.out -"$(./getvar -a aarch64 -g exec)" ./gem5/gem5/configs/example/se.py -c ./aarch64.out +./runtc --arch x86_64 gcc -- -static -o x86_64.out ./packages/kernel_modules/user/hello.c +./runtc --arch arm gcc -- -static -o arm.out ./packages/kernel_modules/user/hello.c +./runtc --arch aarch64 gcc -- -static -o aarch64.out ./packages/kernel_modules/user/hello.c +"$(./getvar --arch x86_64 --gem5 exec)" "$(./getvar gem5_se_file)" -c ./x86_64.out +"$(./getvar --arch arm --gem5 exec)" "$(./getvar gem5_se_file)" -c ./arm.out +"$(./getvar --arch aarch64 --gem5 exec)" "$(./getvar gem5_se_file)" -c ./aarch64.out .... But I think this is unreliable, and only works because we are using uclibc which does not check the kernel version as glibc does: https://stackoverflow.com/questions/48959349/how-to-solve-fatal-kernel-too-old-when-running-gem5-in-syscall-emulation-se-m/50542301#50542301 @@ -6832,9 +7240,9 @@ But I think this is unreliable, and only works because we are using uclibc which Ignoring that insanity, we then try it with dynamically linked executables: .... -"$(./getvar -a x86_64 -g exec)" ./gem5/gem5/configs/example/se.py -c "$(./getvar -a x86_64 -g target_dir)/hello.out" -"$(./getvar -a arm -g exec)" ./gem5/gem5/configs/example/se.py -c "$(./getvar -a arm -g target_dir)/hello.out" -"$(./getvar -a aarch64 -g exec)" ./gem5/gem5/configs/example/se.py -c "$(./getvar -a aarch64 -g target_dir)/hello.out" +"$(./getvar --arch x86_64 --gem5 exec)" "$(./getvar gem5_se_file)" -c "$(./getvar --arch x86_64 --gem5 target_dir)/hello.out" +"$(./getvar --arch arm --gem5 exec)" "$(./getvar gem5_se_file)" -c "$(./getvar --arch arm --gem5 target_dir)/hello.out" +"$(./getvar --arch aarch64 --gem5 exec)" "$(./getvar gem5_se_file)" -c "$(./getvar --arch aarch64 --gem5 target_dir)/hello.out" .... But at 185c2730cc78d5adda683d76c0e3b35e7cb534f0 they fail with: @@ -6843,7 +7251,7 @@ But at 185c2730cc78d5adda683d76c0e3b35e7cb534f0 they fail with: fatal: Unable to open dynamic executable's interpreter. .... -and `cd "$(./getvar -a aarch64 target_dir)` did not help: https://stackoverflow.com/questions/50542222/how-to-run-a-dynamically-linked-executable-syscall-emulation-mode-se-py-in-gem5 +and `cd "$(./getvar --arch aarch64 target_dir)` did not help: https://stackoverflow.com/questions/50542222/how-to-run-a-dynamically-linked-executable-syscall-emulation-mode-se-py-in-gem5 The current FAQ says it is not possible to use dynamic executables: http://gem5.org/Frequently_Asked_Questions but I don't trust it, and then these presentations mention it: @@ -6860,13 +7268,14 @@ gem5 user mode: .... make \ - -C "$(./getvar -a arm build_dir)/dhrystone-2" \ - CC=""$(./getvar buildroot_out_dir)/host/usr/bin/arm-buildroot-linux-uclibcgnueabihf-gcc" \ + -C "$(./getvar --arch arm build_dir)/dhrystone-2" \ + CC="$(./runtc --arch arm --dry gcc)" \ CFLAGS=-static \ ; -time "$(./getvar -a arm -g exec)" \ - ./gem5/gem5/configs/example/se.py \ - -c "$(./getvar -a arm build_dir)/dhrystone-2/dhrystone" \ +time \ + "$(./getvar --arch arm --gem5 exec)" \ + "$(./getvar --arch arm gem5_se_file)" \ + -c "$(./getvar --arch arm build_dir)/dhrystone-2/dhrystone" \ -o 100000 \ ; .... @@ -6874,22 +7283,29 @@ time "$(./getvar -a arm -g exec)" \ gem5 full system: .... -printf 'm5 exit' > data/readfile -./run -a a -g -F '/gem5.sh' -printf 'dhrystone 100000' > data/readfile -time ./run -a a -l 1 -g +time \ + ./run \ + --arch arm \ + --eval-busybox '/gem5.sh' \ + --gem5 + --gem5-readfile 'dhrystone 100000' \ +; .... QEMU user mode: .... -time qemu-arm "$(./getvar -a arm build_dir)/dhrystone-2/dhrystone" 100000000 +time qemu-arm "$(./getvar --arch arm build_dir)/dhrystone-2/dhrystone" 100000000 .... QEMU full system: .... -time ./run -a a -F 'time dhrystone 100000000;/poweroff.out' +time \ + ./run \ + --arch arm \ + --eval-busybox 'time dhrystone 100000000;/poweroff.out' \ +; .... Result on <> at bad30f513c46c1b0995d3a10c0d9bc2a33dc4fa0: @@ -6903,7 +7319,7 @@ Result on <> at bad30f513c46c1b0995d3a10c0d9bc2a33dc4fa0: The QEMU monitor is a terminal that allows you to send text commands to the QEMU VM: https://en.wikibooks.org/wiki/QEMU/Monitor -Accessed it in either <> and <>: +On another terminal, run: .... ./qemumonitor @@ -6959,6 +7375,14 @@ Getting everything to work required careful choice of QEMU command line options: Peter Maydell said potentially not possible nicely as of August 2018: https://stackoverflow.com/questions/51747744/how-to-run-a-qemu-monitor-command-from-inside-the-guest/51764110#51764110 +It is also worth looking into the QEMU Guest Agent tool `qemu-gq` that can be enabled with: + +.... +./build-buildroot --buildroot-config 'BR2_PACKAGE_QEMU=y' +.... + +See also: https://superuser.com/questions/930588/how-to-pass-commands-noninteractively-to-running-qemu-from-the-guest-qmp-via-te + ==== QEMU monitor from GDB When doing <> it is possible to send QEMU monitor commands through the GDB `monitor` command, which saves you the trouble of opening yet another shell. @@ -6977,7 +7401,7 @@ When you start hacking QEMU or gem5, it is useful to see what is going on inside This is of course trivial since they are just regular userland programs on the host, but we make it a bit easier with: .... -./run -D +./run --debug-vm .... Then you could: @@ -6993,7 +7417,7 @@ And in QEMU: /qemu_edu.sh .... -When in <>, using `-D` makes Ctrl-C not get passed to the QEMU guest anymore: it is instead captured by GDB itself, so allow breaking. So e.g. you won't be able to easily quit from a guest progra like: +When in <>, using `--debug-vm` makes Ctrl-C not get passed to the QEMU guest anymore: it is instead captured by GDB itself, so allow breaking. So e.g. you won't be able to easily quit from a guest program like: .... sleep 10 @@ -7003,6 +7427,30 @@ In graphic mode, make sure that you never click inside the QEMU graphic while de You can still send key presses to QEMU however even without the mouse capture, just either click on the title bar, or alt tab to give it focus. +==== Debug gem5 Python scripts + +Start pdb at the first instruction: + +.... +./run --gem5 --gem5-exe-args='--pdb' --terminal +.... + +Requires `--terminal` as we must be on foreground. + +Alternatively, you can add to the point of the code where you want to break the usual: + +.... +import ipdb; ipdb.set_trace() +.... + +and then run with: + +.... +./run --gem5 --terminal +.... + +TODO test PyCharm: https://stackoverflow.com/questions/51982735/writing-gem5-configuration-scripts-with-pycharm + === Tracing QEMU can log several different events. @@ -7010,7 +7458,7 @@ QEMU can log several different events. The most interesting are events which show instructions that QEMU ran, for which we have a helper: .... -./trace-boot -a x86_64 +./trace-boot --arch x86_64 .... Under the hood, this uses QEMU's `-trace` option. @@ -7018,19 +7466,19 @@ Under the hood, this uses QEMU's `-trace` option. You can then inspect the instructions with: .... -less "$(./getvar -a x86_64 run_dir)/trace.txt" +less "$(./getvar --arch x86_64 run_dir)/trace.txt" .... Get the list of available trace events: .... -./run -T help +./run --trace help .... Enable other specific trace events: .... -./run -T trace1,trace2 +./run --trace trace1,trace2 ./qemu-trace2txt -a "$arch" less "$(./getvar -a "$arch" run_dir)/trace.txt" .... @@ -7069,7 +7517,7 @@ QEMU also has a second trace mechanism in addition to `-trace`, find out the eve Let's pick the one that dumps executed instructions, `in_asm`: .... -./run -E '/poweroff.out' -- -D out/trace.txt -d in_asm +./run --eval '/poweroff.out' -- -D out/trace.txt -d in_asm less out/trace.txt .... @@ -7095,8 +7543,8 @@ TODO: what is the point of having two mechanisms, `-trace` and `-d`? `-d` traci We can further use Binutils' `addr2line` to get the line that corresponds to each address: .... -./trace-boot -a x86_64 && ./trace2line -a x86_64 -less "$(./getvar -a x86_64 run_dir)/trace-lines.txt" +./trace-boot --arch x86_64 && ./trace2line --arch x86_64 +less "$(./getvar --arch x86_64 run_dir)/trace-lines.txt" .... The format is as follows: @@ -7126,15 +7574,15 @@ This awesome feature allows you to examine a single run as many times as you wou .... # Record a run. -./run -F '/rand_check.out;/poweroff.out;' -r +./run --eval-busybox '/rand_check.out;/poweroff.out;' --record # Replay the run. -./run -F '/rand_check.out;/poweroff.out;' -R +./run --eval-busybox '/rand_check.out;/poweroff.out;' --replay .... A convenient shortcut to do both at once to test the feature is: .... -./qemurr -F '/rand_check.out;/poweroff.out;' +./qemurr --eval-busybox '/rand_check.out;/poweroff.out;' .... By comparing the terminal output of both runs, we can see that they are the exact same, including things which normally differ across runs: @@ -7161,7 +7609,7 @@ EXT4-fs (sda): re-mounted. Opts: block_validity,barrier,user_xattr TODO replay with network gets stuck: .... -./qemurr -F '/sbin/ifup -a;wget -S google.com;/poweroff.out;' +./qemurr --eval-busybox '/sbin/ifup -a;wget -S google.com;/poweroff.out;' .... after the message: @@ -7179,8 +7627,8 @@ TODO `arm` and `aarch64` only seem to work with initrd since I cannot plug a wor Then, when I tried with <> and no disk: .... -./build -a A -i -./qemurr -a A -F '/rand_check.out;/poweroff.out;' -i +./build-buildroot --arch aarch64 -i +./qemurr --arch aarch64 --eval-busybox '/rand_check.out;/poweroff.out;' -i .... QEMU crashes with: @@ -7200,8 +7648,8 @@ TODO get working. QEMU replays support checkpointing, and this allows for a simplistic "reverse debugging" implementation proposed at https://lists.gnu.org/archive/html/qemu-devel/2018-06/msg00478.html on the unmerged link:https://github.com/ispras/qemu/tree/rr-180725[]: .... -./run -F '/rand_check.out;/poweroff.out;' -r -./run -F '/rand_check.out;/poweroff.out;' -R -d +./run --eval-busybox '/rand_check.out;/poweroff.out;' --record +./run --eval-busybox '/rand_check.out;/poweroff.out;' --replay --debug-guest .... On another shell: @@ -7227,7 +7675,7 @@ and we are back at `start_kernel` TODO: is there any way to distinguish which instruction runs on each core? Doing: .... -./run -a x86_64 -c 2 -E '/poweroff.out' -T exec_tb +./run --arch x86_64 --cpus 2 --eval '/poweroff.out' --trace exec_tb ./qemu-trace2txt .... @@ -7252,21 +7700,21 @@ gem5 unlike QEMU is deterministic by default without needing to replay traces But it also provides a tracing mechanism documented at: link:http://www.gem5.org/Trace_Based_Debugging[] to allow easily inspecting certain aspects of the system: .... -./run -a aarch64 -E 'm5 exit' -g -T Exec -less "$(./getvar -a aarch64 run_dir)/trace.txt" +./run --arch aarch64 --eval 'm5 exit' --gem5 --trace Exec +less "$(./getvar --arch aarch64 run_dir)/trace.txt" .... List all available debug flags: .... -./run -a aarch64 -G --debug-help -g +./run --arch aarch64 --gem5-exe-args='--debug-help' --gem5 .... but to understand most of them you have to look at the source code: .... -less gem5/gem5/src/cpu/SConscript -less gem5/gem5/src/cpu/exetrace.cc +less "$(./getvar gem5_src_dir)/src/cpu/SConscript" +less "$(./getvar gem5_src_dir)/src/cpu/exetrace.cc" .... As can be seen on the `Sconstruct`, `Exec` is just an alias that enables a set of flags. @@ -7310,8 +7758,8 @@ The best way to verify all of this is to write some bare metal code: https://sta Trace the source lines just like <> with: .... -./trace-boot -a aarch64 -g && ./trace2line -a aarch64 -g -less "$(./getvar -a aarch64 run_dir)/trace-lines.txt" +./trace-boot --arch aarch64 --gem5 && ./trace2line --arch aarch64 --gem5 +less "$(./getvar --arch aarch64 run_dir)/trace-lines.txt" .... TODO: 7452d399290c9c1fc6366cdad129ef442f323564 `./trace2line` this is too slow and takes hours. QEMU's processing of 170k events takes 7 seconds. gem5's processing is analogous, but there are 140M events, so it should take 7000 seconds ~ 2 hours which seems consistent with what I observe, so maybe there is no way to speed this up... The workaround is to just use gem5's `ExecSymbol` to get function granularity, and then GDB individually if line detail is needed? @@ -7334,32 +7782,6 @@ Using text mode is another workaround if you don't need GUI features. == gem5 -=== gem5 getting started - -gem5 is a system simulator, much <>: http://gem5.org/ - -For the most part, just add the `-g` option to all commands and everything should magically work: - -.... -./configure -g && ./build -a arm -g && ./run -a arm -g -.... - -To get a terminal, either open a new shell and run: - -.... -./gem5-shell -.... - -or use `./run -u` if you are using tmux, which I highly recommend: <>. - -TODO: `arm` broken on kernel 4.18 with: - -.... -Internal error: Oops - undefined instruction -.... - -Workaround by checking out kernel 4.17 as explained at <>. Bisected down to kernel 1d4238c56f9816ce0f9c8dbe42d7f2ad81cb6613, gem5 is not implementing the `CSDB` instruction. - === gem5 vs QEMU * advantages of gem5: @@ -7408,51 +7830,39 @@ On the other hand, the chip makers tend to upstream less, and the project become OK, this is why we used gem5 in the first place, performance measurements! -Let's benchmark https://en.wikipedia.org/wiki/Dhrystone[Dhrystone] which Buildroot provides. +Let's see how many cycles https://en.wikipedia.org/wiki/Dhrystone[Dhrystone], which Buildroot provides, takes for a few different input parameters. -A flexible setup is: +A flexible setup is demonstrated at: .... -arch=aarch64 -cmd="./run -a '$arch' -g -F '/gem5.sh'" -# These cache sizes roughly match the ARM Cortex A75 -# https://en.wikipedia.org/wiki/ARM_Cortex-A75 -restore='-l 1 -- --cpu-type=HPI --restore-with-cpu=HPI --caches --l2cache --l1d_size=128kB --l1i_size=1024kB --l2_size=256kB' - -# Generate a checkpoint after Linux boots, using the faster and less detailed CPU. -# The boot takes a while, be patient young Padawan. -printf '' > data/readfile -eval "$cmd" - -# Restore the most recent checkpoint taken with the more detailed and slower HPI CPU, -# and run the benchmark with parameter 1.000. We skip the boot completely, saving time! -printf 'dhrystone 1000' > data/readfile -eval "${cmd} ${restore}" -./gem5-stat -a "$arch" - -# Now run again with another parameter 10.000. -# This one should take more cycles! -printf 'dhrystone 10000' > data/readfile -eval "${cmd} ${restore}" -./gem5-stat -a "$arch" - -# Get an interactive shell at the end of the restore -# if you need to debug something more interactively. -printf 'sh' > data/readfile -eval "${cmd} ${restore}" +./gem5-bench-dhrystone +cat out/gem5-bench-dhrystone.txt .... +Source: link:gem5-bench-dhrystone[] + +Sample output: + +.... +n cycles +1000 12898577 +10000 23441629 +100000 128428617 +.... + +so as expected, the Dhrystone run with a larger input parameter `100000` took more cycles than the ones with smaller input parameters. + The `gem5-stats` commands output the approximate number of CPU cycles it took Dhrystone to run. -For more serious tests, you will likely want to automate logging the commands ran and results to files, a good example is: link:gem5-bench-cache[]. +Another interesting example can be found at: link:gem5-bench-cache[]. A more naive and simpler to understand approach would be a direct: .... -./run -a aarch64 -g -E 'm5 checkpoint;m5 resetstats;dhrystone 10000;m5 exit' +./run --arch aarch64 --gem5 --eval 'm5 checkpoint;m5 resetstats;dhrystone 10000;m5 exit' .... -but the problem is that this method does not allow to easily run a different script without running the boot again, see: <> +but the problem is that this method does not allow to easily run a different script without running the boot again, see: <>. Now you can play a fun little game with your friends: @@ -7463,7 +7873,7 @@ Now you can play a fun little game with your friends: To find out why your program is slow, a good first step is to have a look at the statistics for the run: .... -cat "$(./getvar -a aarch64 m5out_dir)/stats.txt" +cat "$(./getvar --arch aarch64 m5out_dir)/stats.txt" .... Whenever we run `m5 dumpstats` or `m5 exit`, a section with the following format is added to that file: @@ -7499,7 +7909,7 @@ The rabbit hole is likely deep, but let's scratch a bit of the surface. ===== Number of cores .... -./run -a arm -c 2 -g +./run --arch arm --cpus 2 --gem5 .... Check with: @@ -7517,7 +7927,7 @@ https://stackoverflow.com/questions/50248067/how-to-run-a-gem5-arm-aarch64-full- https://stackoverflow.com/questions/49624061/how-to-run-gem5-simulator-in-fs-mode-without-cache/49634544#49634544 -A quick `+./run -g -- -h+` leads us to the options: +A quick `+./run --gem5 -- -h+` leads us to the options: .... --caches @@ -7577,48 +7987,48 @@ Behaviour breakdown: So we take a performance measurement approach instead: .... -./gem5-bench-cache -a aarch64 -cat "$(./getvar -a aarch64 run_dir)bench-cache.txt" +./gem5-bench-cache --arch aarch64 +cat "$(./getvar --arch aarch64 run_dir)/bench-cache.txt" .... which gives: .... -n 1000 -cmd ./run -a arm -g -l 1 -- --caches --l2cache --l1d_size=1024 --l1i_size=1024 --l2_size=1024 --l3_size=1024 --cpu-type=HPI --restore-with-cpu=HPI -time 24.71 +cmd ./run --gem5 --arch aarch64 --gem5-readfile "dhrystone 1000" --gem5-restore 1 -- --caches --l2cache --l1d_size=1024 --l1i_size=1024 --l2_size=1024 --l3_size=1024 --cpu-type=HPI --restore-with-cpu=HPI +time 23.82 exit_status 0 -cycles 52386455 -instructions 4555081 -cmd ./run -a arm -g -l 1 -- --caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB --cpu-type=HPI --restore-with-cpu=HPI -time 17.44 -exit_status 0 -cycles 6683355 -instructions 4466051 +cycles 93284622 +instructions 4393457 -n 10000 -cmd ./run -a arm -g -l 1 -- --caches --l2cache --l1d_size=1024 --l1i_size=1024 --l2_size=1024 --l3_size=1024 --cpu-type=HPI --restore-with-cpu=HPI -time 52.90 +cmd ./run --gem5 --arch aarch64 --gem5-readfile "dhrystone 1000" --gem5-restore 1 -- --caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB --cpu-type=HPI --restore-with-cpu=HPI +time 14.91 exit_status 0 -cycles 165704397 -instructions 11531136 -cmd ./run -a arm -g -l 1 -- --caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB --cpu-type=HPI --restore-with-cpu=HPI -time 36.19 -exit_status 0 -cycles 16182925 -instructions 11422585 +cycles 10128985 +instructions 4211458 -n 100000 -cmd ./run -a arm -g -l 1 -- --caches --l2cache --l1d_size=1024 --l1i_size=1024 --l2_size=1024 --l3_size=1024 --cpu-type=HPI --restore-with-cpu=HPI -time 325.09 +cmd ./run --gem5 --arch aarch64 --gem5-readfile "dhrystone 10000" --gem5-restore 1 -- --caches --l2cache --l1d_size=1024 --l1i_size=1024 --l2_size=1024 --l3_size=1024 --cpu-type=HPI --restore-with-cpu=HPI +time 51.87 exit_status 0 -cycles 1295703657 -instructions 81189411 -cmd ./run -a arm -g -l 1 -- --caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB --cpu-type=HPI --restore-with-cpu=HPI -time 250.74 +cycles 188803630 +instructions 12401336 + +cmd ./run --gem5 --arch aarch64 --gem5-readfile "dhrystone 10000" --gem5-restore 1 -- --caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB --cpu-type=HPI --restore-with-cpu=HPI +time 35.35 exit_status 0 -cycles 110585681 -instructions 80899588 +cycles 20715757 +instructions 12192527 + +cmd ./run --gem5 --arch aarch64 --gem5-readfile "dhrystone 100000" --gem5-restore 1 -- --caches --l2cache --l1d_size=1024 --l1i_size=1024 --l2_size=1024 --l3_size=1024 --cpu-type=HPI --restore-with-cpu=HPI +time 339.07 +exit_status 0 +cycles 1176559936 +instructions 94222791 + +cmd ./run --gem5 --arch aarch64 --gem5-readfile "dhrystone 100000" --gem5-restore 1 -- --caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB --cpu-type=HPI --restore-with-cpu=HPI +time 240.37 +exit_status 0 +cycles 125666679 +instructions 91738770 .... We make the following conclusions: @@ -7643,7 +8053,7 @@ TODO: now to verify this with the Linux kernel? Besides raw performance benchmar ===== Memory size .... -./run -a arm -m 512M +./run --arch arm --memory 512M .... and verify inside the guest with: @@ -7668,7 +8078,7 @@ and also: `gem5-dist`: https://publish.illinois.edu/icsl-pdgem5/ Clock frequency: TODO how does it affect performance in benchmarks? .... -./run -a aarch64 -g -- --cpu-clock 10000000 +./run --arch aarch64 --gem5 -- --cpu-clock 10000000 .... Check with: @@ -7680,7 +8090,7 @@ m5 resetstats && sleep 10 && m5 dumpstats and then: .... -./gem5-stat -a aarch64 +./gem5-stat --arch aarch64 .... TODO: why doesn't this exist: @@ -7711,9 +8121,13 @@ https://stackoverflow.com/questions/6147242/heap-vs-binary-search-tree-bst/29548 Usage: .... -printf '/bst_vs_heap.out' > data/readfile -./run -aA -g -F '/gem5.sh' -./bst-vs-heap > bst_vs_heap.dat +./run \ + --arch aarch64 \ + --eval-busybox '/gem5.sh' \ + --gem5 \ + --gem5-readfile '/bst_vs_heap.out' \ +; +./bst-vs-heap --arch aarch64 --gem5 > bst_vs_heap.dat .... and then feed `bst_vs_heap.dat` into: https://github.com/cirosantilli/cpp-cheat/blob/9d0f77792fc8e55b20b6ee32018761ef3c5a3f2f/cpp/interactive/bst_vs_heap.gnuplot @@ -7721,7 +8135,7 @@ and then feed `bst_vs_heap.dat` into: https://github.com/cirosantilli/cpp-cheat/ Sources: * link:bst-vs-heap[] -* link:kernel_module/user/bst_vs_heap.cpp[] +* link:packages/kernel_modules/user/bst_vs_heap.cpp[] ===== OpenMP @@ -7731,15 +8145,15 @@ Implemented by GCC itself, so just a toolchain configuration, no external libs, /openmp.out .... -Source: link:kernel_module/user/openmp.c[] +Source: link:packages/kernel_modules/user/openmp.c[] ===== BLAS Buildroot supports it, which makes everything just trivial: .... -./build -B 'BR2_PACKAGE_OPENBLAS=y' -k -./run -F '/openblas.out; echo $?' +./build-buildroot --buildroot-config 'BR2_PACKAGE_OPENBLAS=y' --kernel-modules +./run --eval-busybox '/openblas.out; echo $?' .... Outcome: the test passes: @@ -7748,7 +8162,7 @@ Outcome: the test passes: 0 .... -Source: link:kernel_module/user/openblas.c[] +Source: link:packages/kernel_modules/user/openblas.c[] The test performs a general matrix multiplication: @@ -7777,13 +8191,13 @@ cblas_dgemm( CblasColMajor, CblasNoTrans, CblasTrans,3,3,2 ,1, A,3, B, Header only linear algebra library with a mainline Buildroot package: .... -./build -B 'BR2_PACKAGE_EIGEN=y' -k +./build-buildroot --buildroot-config 'BR2_PACKAGE_EIGEN=y' --kernel-modules .... Just create an array and print it: .... -./run -F '/eigen.out' +./run --eval-busybox '/eigen.out' .... Output: @@ -7793,7 +8207,7 @@ Output: 2.5 1.5 .... -Source: link:kernel_module/user/eigen.cpp[] +Source: link:packages/kernel_modules/user/eigen.cpp[] This example just creates a matrix and prints it out. @@ -7811,7 +8225,10 @@ There are two ways to run PARSEC with this repo: ====== PARSEC benchmark without parsecmgmt .... -configure -gpq && ./build -a arm -B 'BR2_PACKAGE_PARSEC_BENCHMARK=y' -g && ./run -a arm -g +./configure -gpq && \ +./build-buildroot --arch arm --buildroot-config 'BR2_PACKAGE_PARSEC_BENCHMARK=y' --gem5 && \ +./run --arch arm --gem5 && \ +:; .... Once inside the guest, launch one of the `test` input sized benchmarks manually as in: @@ -7837,11 +8254,11 @@ but this might be a bit time consuming in gem5. Running a benchmark of a size different than `test`, e.g. `simsmall`, requires a rebuild with: .... -./build \ - -a arm \ - -B 'BR2_PACKAGE_PARSEC_BENCHMARK=y' \ - -B 'BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE="simsmall"' \ - -g \ +./build-buildroot \ + --arch arm \ + --buildroot-config 'BR2_PACKAGE_PARSEC_BENCHMARK=y' \ + --buildroot-config 'BR2_PACKAGE_PARSEC_BENCHMARK_INPUT_SIZE="simsmall"' \ + --gem5 \ -- parsec-benchmark-reconfigure \ ; .... @@ -7895,12 +8312,12 @@ but it simply is not feasible in gem5 because it takes too long. If you still want to run this, try it out with: .... -./build \ - -a aarch64 \ - -B 'BR2_PACKAGE_PARSEC_BENCHMARK=y' \ - -B 'BR2_PACKAGE_PARSEC_BENCHMARK_PARSECMGMT=y' \ - -B 'BR2_TARGET_ROOTFS_EXT2_SIZE="3G"' \ - -g \ +./build-buildroot \ + --arch aarch64 \ + --buildroot-config 'BR2_PACKAGE_PARSEC_BENCHMARK=y' \ + --buildroot-config 'BR2_PACKAGE_PARSEC_BENCHMARK_PARSECMGMT=y' \ + --buildroot-config 'BR2_TARGET_ROOTFS_EXT2_SIZE="3G"' \ + --gem5 \ -- parsec-benchmark-reconfigure \ ; .... @@ -7920,18 +8337,18 @@ If you want to remove PARSEC later, Buildroot doesn't provide an automated packa .... rm -rf \ - ./out/common/dl/parsec-* \ - "$(./getvar buildroot_out_dir)"/build/parsec-* \ - "$(./getvar buildroot_out_dir)"/build/packages-file-list.txt \ - "$(./getvar buildroot_out_dir)"/images/rootfs.* \ - "$(./getvar buildroot_out_dir)"/target/parsec-* \ + "$(./getvar dl_dir)"/parsec-* \ + "$(./getvar buildroot_build_dir)"/build/parsec-* \ + "$(./getvar buildroot_build_dir)"/build/packages-file-list.txt \ + "$(./getvar buildroot_build_dir)"/images/rootfs.* \ + "$(./getvar buildroot_build_dir)"/target/parsec-* \ ; -./build -a arm -g +./build-buildroot --arch arm --gem5 .... ====== PARSEC benchmark hacking -If you end up going inside link:parsec-benchmark/parsec-benchmark[] to hack up the benchmark (you will!), these tips will be helpful. +If you end up going inside link:submodules/parsec-benchmark[] to hack up the benchmark (you will!), these tips will be helpful. Buildroot was not designed to deal with large images, and currently cross rebuilds are a bit slow, due to some image generation and validation steps. @@ -7942,7 +8359,7 @@ A few workarounds are: If you do this, don't forget to do a: + .... -cd parsec-benchmark/parsec-benchmark +cd "$(./getvar parsec_src_dir)" git clean -xdf . .... before going for the cross compile build. @@ -7952,7 +8369,12 @@ before going for the cross compile build. Don't forget to explicitly rebuild PARSEC with: + .... -./build -a arm -B 'BR2_PACKAGE_PARSEC_BENCHMARK=y' -g parsec-benchmark-reconfigure +./build-buildroot \ + --arch arm \ + --buildroot-config 'BR2_PACKAGE_PARSEC_BENCHMARK=y' \ + --gem5 \ + -- parsec-benchmark-reconfigure \ +; .... + You may also want to test if your patches are still functionally correct inside of QEMU first, which is a faster emulator. @@ -7963,7 +8385,7 @@ You may also want to test if your patches are still functionally correct inside Analogous <>: .... -./run -a arm -e 'init=/poweroff.out' -g +./run --arch arm --kernel-cli 'init=/poweroff.out' --gem5 .... Internals: when we give `--command-line=` to gem5, it overrides default command lines, including some mandatory ones which are required to boot properly. @@ -7973,7 +8395,7 @@ Our run script hardcodes the require options in the default `--command-line` and To find the default options in the first place, we removed `--command-line` and ran: .... -./run -a arm -g +./run --arch arm --gem5 .... and then looked at the line of the Linux kernel that starts with: @@ -7989,13 +8411,13 @@ Kernel command line: Analogous <>, on the first shell: .... -./run -a arm -d -g +./run --arch arm --debug-guest --gem5 .... On the second shell: .... -./rungdb -a arm -g +./rungdb --arch arm --gem5 .... On a third shell: @@ -8060,7 +8482,7 @@ I press `n`, it just runs the program until the end, instead of stopping on the TODO: .... -./rungdb-user -a arm -g gem5-1.0/gem5/util/m5/m5 main +./rungdb-user --arch arm --gem5 gem5-1.0/gem5/util/m5/m5 main .... breaks when `m5` is run on guest, but does not show the source code. @@ -8072,7 +8494,7 @@ Analogous to QEMU's <>, but better since it can be started from inside Documentation: http://gem5.org/Checkpoints .... -./run -a arm -g +./run --arch arm --gem5 .... In the guest, wait for the boot to end and run: @@ -8086,10 +8508,10 @@ where <> is a guest utility present inside the gem5 tree which we cross-comp To restore the checkpoint, kill the VM and run: .... -./run -a arm -g -l 1 +./run --arch arm --gem5 --gem5-restore 1 .... -The `-l` option restores the checkpoint that was created most recently. +The `--gem5-restore` option restores the checkpoint that was created most recently. Let's create a second checkpoint to see how it works, in guest: @@ -8101,10 +8523,10 @@ m5 checkpoint Kill the VM, and try it out: .... -./run -a arm -g -l 1 +./run --arch arm --gem5 --gem5-restore 1 .... -Here we use `-l 1` again, since the second snapshot we took is now the most recent one +Here we use `--gem5-restore 1` again, since the second snapshot we took is now the most recent one Now in the guest: @@ -8112,18 +8534,18 @@ Now in the guest: cat f .... -contains the `date`. The file `f` wouldn't exist had we used the first checkpoint with `-l 2`, which is the second most recent snapshot taken. +contains the `date`. The file `f` wouldn't exist had we used the first checkpoint with `--gem5-restore 2`, which is the second most recent snapshot taken. If you automate things with <> as in: .... -./run -a arm -E 'm5 checkpoint;m5 resetstats;dhrystone 1000;m5 exit' -g +./run --arch arm --eval 'm5 checkpoint;m5 resetstats;dhrystone 1000;m5 exit' --gem5 .... Then there is no need to pass the kernel command line again to gem5 for replay: .... -./run -a arm -g -l 1 +./run --arch arm --gem5 --gem5-restore 1 .... since boot has already happened, and the parameters are already in the RAM of the snapshot. @@ -8133,7 +8555,7 @@ since boot has already happened, and the parameters are already in the RAM of th Checkpoints are stored inside the `m5out` directory at: .... -"$(./getvar -g run_dir)/m5out/cpt." +"$(./getvar --gem5 run_dir)/m5out/cpt." .... where `` is the cycle number at which the checkpoint was taken. @@ -8142,11 +8564,11 @@ where `` is the cycle number at which the checkpoint was taken. However, that interface is bad because if you had taken previous checkpoints, you have no idea what `N` to use, unless you memorize which checkpoint was taken at which cycle. -Therefore, just use our superior `-l` flag, which uses directory timestamps to determine which checkpoint you created most recently. +Therefore, just use our superior `--gem5-restore` flag, which uses directory timestamps to determine which checkpoint you created most recently. The `-r N` integer value is just pure `fs.py` sugar, the backend at `m5.instantiate` just takes the actual tracepoint directory path as input. -[[gem5-restore-new-scrip]] +[[gem5-restore-new-script]] ==== gem5 checkpoint restore and run a different script You want to automate running several tests from a single pristine post-boot state. @@ -8156,23 +8578,37 @@ The problem is that boot takes forever, and after the checkpoint, the memory and * hack up an existing rc script, since the disk is fixed * inject new kernel boot command line options, since those have already been put into memory by the bootloader -There is however one loophole: <>, which reads whatever is present on the host, so we can do it like: +There is however a few loopholes, <> being the simplest, as it reads whatever is present on the host. + +So we can do it like: .... -printf 'echo "setup run";m5 exit' > data/readfile -./run -a aarch64 -g -E 'm5 checkpoint;m5 readfile > a.sh;sh a.sh' -printf 'echo "first benchmark";m5 exit' > data/readfile -./run -a aarch64 -g -l 1 -printf 'echo "second benchmark";m5 exit' > data/readfile -./run -a aarch64 -g -l 1 +# Boot, checkpoint and exit. +printf 'echo "setup run";m5 exit' > "$(./getvar gem5_readfile)" +./run --gem5 --eval 'm5 checkpoint;m5 readfile > a.sh;sh a.sh' + +# Restore and run the first benchmark. +printf 'echo "first benchmark";m5 exit' > "$(./getvar gem5_readfile)" +./run --gem5 --gem5-restore 1 + +# Restore and run the second benchmark. +printf 'echo "second benchmark";m5 exit' > "$(./getvar gem5_readfile)" +./run --gem5 --gem5-restore 1 + +# If something weird happened, create an interactive shell to examine the system. +printf 'sh' > "$(./getvar gem5_readfile)" +./run --gem5 --gem5-restore 1 .... -Since this is such a common setup, we provide helper for it at: link:rootfs_overlay/gem5.sh[rootfs_overlay/gem5.sh]. +Since this is such a common setup, we provide some helpers for it as described at <>: + +* link:rootfs_overlay/gem5.sh[rootfs_overlay/gem5.sh]. This script is analogous to gem5's in-tree link:https://github.com/gem5/gem5/blob/2b4b94d0556c2d03172ebff63f7fc502c3c26ff8/configs/boot/hack_back_ckpt.rcS[hack_back_ckpt.rcS], but with less noise. +* `./run --gem5-readfile` is a convenient way to set the `m5 readfile` Other loophole possibilities include: * <<9p>> -* link:https://stackoverflow.com/questions/50862906/how-to-attach-multiple-disk-images-in-a-simulation-with-gem5-fs-py/51037661#51037661[create multiple disk images], and mount the benchmark from on one of them +* <> * `expect` as mentioned at: https://stackoverflow.com/questions/7013137/automating-telnet-session-using-bash-scripts + .... @@ -8184,6 +8620,8 @@ send "ls /\r" send "m5 exit\r" expect eof .... ++ +This is ugly however as it is not deterministic. https://www.mail-archive.com/gem5-users@gem5.org/msg15233.html @@ -8196,7 +8634,7 @@ A common combo is to boot Linux with a fast CPU, make a checkpoint and then repl An illustrative interactive run: .... -./run -a arm -g +./run --arch arm --gem5 .... In guest: @@ -8208,7 +8646,7 @@ m5 checkpoint And then restore the checkpoint with a different CPU: .... -./run -a arm -g -l 1 -- --caches --restore-with-cpu=HPI +./run --arch arm --gem5 --gem5-restore 1 -- --caches --restore-with-cpu=HPI .... === Pass extra options to gem5 @@ -8218,12 +8656,12 @@ Pass options to the `fs.py` script: * get help: + .... -./run -g -- -h +./run --gem5 -- -h .... * boot with the more detailed and slow `HPI` CPU model: + .... -./run -a arm -g -- --caches --cpu-type=HPI +./run --arch arm --gem5 -- --caches --cpu-type=HPI .... Pass options to the `gem5` executable itself: @@ -8231,7 +8669,7 @@ Pass options to the `gem5` executable itself: * get help: + .... -./run -G '-h' -g +./run --gem5-exe-args='-h' --gem5 .... === gem5 exit after a number of instructions @@ -8239,7 +8677,7 @@ Pass options to the `gem5` executable itself: Quit the simulation after `1024` instructions: .... -./run -g -- -I 1024 +./run --gem5 -- -I 1024 .... Can be nicely checked with <>. @@ -8247,7 +8685,7 @@ Can be nicely checked with <>. Cycles instead of instructions: .... -./run -g -- -m 1024 +./run --gem5 -- --memory 1024 .... Otherwise the simulation runs forever by default. @@ -8326,7 +8764,7 @@ m5 writefile myfileguest myfilehost Host: .... -cat "$(./getvar -a aarch64 -g m5out_dir)/myfilehost" +cat "$(./getvar --arch aarch64 --gem5 m5out_dir)/myfilehost" .... Does not work for subdirectories, gem5 crashes: @@ -8337,12 +8775,14 @@ m5 writefile myfileguest mydirhost/myfilehost ===== m5 readfile +Read a host file pointed to by the `fs.py --script` option to stdout. + https://stackoverflow.com/questions/49516399/how-to-use-m5-readfile-and-m5-execfile-in-gem5/49538051#49538051 Host: .... -date > data/readfile +date > "$(./getvar gem5_readfile)" .... Guest: @@ -8351,13 +8791,37 @@ Guest: m5 readfile .... +Outcome: date shows on guest. + +===== m5 initparam + +Ermm, just another <> that only takes integers and only from CLI options? Is this software so redundant? + +Host: + +.... +./run --gem5 --gem5-restore 1 -- --initparam 13 +./run --gem5 --gem5-restore 1 -- --initparam 42 +.... + +Guest: + +.... +m5 initparm +.... + +Outputs the given paramter. + ===== m5 execfile +Trivial combination of `m5 readfile` + execute the script. + Host: .... printf '#!/bin/sh -echo asdf' > data/readfile +echo asdf +' > "$(./getvar gem5_readfile)" .... Guest: @@ -8368,6 +8832,12 @@ chmod +x /tmp/execfile m5 execfile .... +Outcome: + +.... +adsf +.... + ==== m5ops instructions The executable `/m5ops.out` illustrates how to hard code with inline assembly the m5ops that you are most likely to hack into the benchmark you are analysing: @@ -8388,8 +8858,8 @@ The executable `/m5ops.out` illustrates how to hard code with inline assembly th Sources: -* link:kernel_module/user/m5ops.h[] -* link:kernel_module/user/m5ops.c[] +* link:packages/kernel_modules/user/m5ops.h[] +* link:packages/kernel_modules/user/m5ops.c[] That executable is of course a subset of <> and useless by itself: its goal is only illustrate how to hardcode some <> yourself as one-liners. @@ -8481,7 +8951,7 @@ Those patches look interesting, but it is obviously not possible to understand w So let's explain them one by one here as we understand them: -* `drm: Add component-aware simple encoder` allows you to see images through VNC: <> +* `drm: Add component-aware simple encoder` allows you to see images through VNC: <> * `gem5: Add support for gem5's extended GIC mode` adds support for more than 8 cores: https://stackoverflow.com/questions/50248067/how-to-run-a-gem5-arm-aarch64-full-system-simulation-with-fs-py-with-more-than-8/50248068#5024806 === m5term @@ -8505,11 +8975,13 @@ Lets try to understand some stats better. link:https://en.wikipedia.org/wiki/Time_Stamp_Counter[x86 instruction] that returns the cycle count since reset: .... -./build -kg && ./run -E '/rdtsc.out;m5 exit;' -g -./gem5-stat +./build-buildroot --gem5 --kernel-modules && \ +./run --eval '/rdtsc.out;m5 exit;' --gem5 && \ +./gem5-stat && \ +:; .... -Source: link:kernel_module/user/rdtsc.c[] +Source: link:packages/kernel_modules/user/rdtsc.c[] `rdtsc` outputs a cycle count which we compare with gem5's `gem5-stat`: @@ -8527,7 +8999,7 @@ Bibliography: ===== pmccntr -TODO We didn't manage to find a working ARM analogue to <>: link:kernel_module/pmccntr.c[] is oopsing, and even it if weren't, it likely won't give the cycle count since boot since it needs to be activate before it starts counting anything: +TODO We didn't manage to find a working ARM analogue to <>: link:packages/kernel_modules/pmccntr.c[] is oopsing, and even it if weren't, it likely won't give the cycle count since boot since it needs to be activate before it starts counting anything: * https://stackoverflow.com/questions/40454157/is-there-an-equivalent-instruction-to-rdtsc-in-arm * https://stackoverflow.com/questions/31620375/arm-cortex-a7-returning-pmccntr-0-in-kernel-mode-and-illegal-instruction-in-u/31649809#31649809 @@ -8535,13 +9007,13 @@ TODO We didn't manage to find a working ARM analogue to <>: link:kernel_m === gem5 Python scripts without rebuild -We have made a crazy setup that allows you to just `cd` into `gem5/gem5`, and edit Python scripts directly there. +We have made a crazy setup that allows you to just `cd` into `submodules/gem5`, and edit Python scripts directly there. This is not normally possible with Buildroot, since normal Buildroot packages first copy files to the output directory (`$(./getvar -a build_dir)/`), and then build there. So if you modified the Python scripts with this setup, you would still need to `./build` to copy the modified files over. -For gem5 specifically however, we have hacked up the build so that we `cd` into the `gem5/gem5` tree, and then do an link:https://www.mail-archive.com/gem5-users@gem5.org/msg15421.html[out of tree] build to `out/common/gem5`. +For gem5 specifically however, we have hacked up the build so that we `cd` into the `submodules/gem5` tree, and then do an link:https://www.mail-archive.com/gem5-users@gem5.org/msg15421.html[out of tree] build to `out/common/gem5`. Another advantage of this method is the we factor out the `arm` and `aarch64` gem5 builds which are identical and large, as well as the smaller arch generic pieces. @@ -8556,18 +9028,18 @@ The out of build tree is required, because otherwise Buildroot would copy the ou By default, we use `configs/example/fs.py` script. -The `-X-b` option enables the alternative `configs/example/arm/fs_bigLITTLE.py` script instead. +The `--gem5-biglittle` option enables the alternative `configs/example/arm/fs_bigLITTLE.py` script instead. First apply: .... -patch -d gem5/gem5 -p1 < patches/manual/gem5-biglittle.patch +patch -d "$(./getvar gem5_src_dir)" -p 1 < patches/manual/gem5-biglittle.patch .... then: .... -./run -a A -g -X-b +./run --arch aarch64 --gem5 --gem5-biglittle .... Advantages over `fs.py`: @@ -8586,18 +9058,58 @@ TODO: why is the `--dtb` required despite `fs_bigLITTLE.py` having a DTB generat Tested on: link:http://github.com/cirosantilli/linux-kernel-module-cheat/commit/18c1c823feda65f8b54cd38e261c282eee01ed9f[18c1c823feda65f8b54cd38e261c282eee01ed9f] +=== gem5 unit tests + +https://stackoverflow.com/questions/52279971/how-to-run-the-gem5-unit-tests + +Not currently exposed here. + == Buildroot -=== Custom Buildroot options +=== Rebuild Buildroot packages + +After making changes to a Buildroot package, you must explicitly request it to be rebuilt. + +For example, if you modify the kernel modules, which is a Buildroot package, you must rebuild with: + +.... +./build-buildroot -- kernel_modules-reconfigure +.... + +where `kernel_modules` is the name of our Buildroot package that contains the kernel modules. + +Since rebuilding this package is such a common case, we have a shortcut for it: + +.... +./build-buildroot --kernel-modules +.... + +The rebuild must be requested explicitly, otherwise the package does not rebuild, even if its files were modified. + +This is because Buildroot builds dozens of packages, and just reading there Makefiles and checking the stamps generates too much overhead + +So you must tell Buildroot yourself about the rebuild. + +=== Custom Buildroot configs We provide the following mechanisms: -* `./build -b data/br2`: append the Buildroot configuration file `data/br2` to a single build. Must be passed every time you run `./build`. The format is the same as link:br2/default[]. -* `./build -B 'BR2_SOME_OPTION="myval"'`: append a single option to a single build. +* `./build-buildroot --buildroot-config-fragment data/br2`: append the Buildroot configuration file `data/br2` to a single build. Must be passed every time you run `./build`. The format is the same as link:br2/default[]. +* `./build-buildroot --buildroot-config 'BR2_SOME_OPTION="myval"'`: append a single option to a single build. -You will then likely want to make those more permanent with: <> +For example, if you decide to <> after an initial build is finished, you must <> and rebuild: -==== Enable compiler optimizations +.... +./build-buildroot --buildroot-config 'BR2_OPTIMIZE_3=y' kernel_modules-dirclean kernel_modules-reconfigure +.... + +as explained at: https://buildroot.org/downloads/manual/manual.html#rebuild-pkg + +The clean is necessary because the source files didn't change, so `make` would just check the timestamps and not build anything. + +You will then likely want to make those more permanent with: <> + +==== Enable Buildroot compiler optimizations If you are benchmarking compiled programs instead of hand written assembly, remember that we configure Buildroot to disable optimizations by default with: @@ -8613,16 +9125,21 @@ You will likely want to change that to: BR2_OPTIMIZE_3=y .... -Our link:kernel_module/user[] package correctly forwards the Buildroot options to the build with `$(TARGET_CONFIGURE_OPTS)`, so you don't have to do any extra work. +Our link:packages/kernel_modules/user[] package correctly forwards the Buildroot options to the build with `$(TARGET_CONFIGURE_OPTS)`, so you don't have to do any extra work. Don't forget to do that if you are <> with your own build system. Then, you have two choices: -* if you already have a full `-O0` build, you can choose to rebuild just your package of interest to save some time as described at: <> +* if you already have a full `-O0` build, you can choose to rebuild just your package of interest to save some time as described at: <> + .... -./build -B 'BR2_OPTIMIZE_3=y' kernel_module-dirclean kernel_module-reconfigure +./build-buildroot \ + --buildroot-config 'BR2_OPTIMIZE_3=y' \ + -- \ + kernel_modules-dirclean \ + kernel_modules-reconfigure \ +; .... + However, this approach might not be representative since calls to an unoptimized libc and other libraries will have a negative performance impact. @@ -8634,7 +9151,7 @@ Kernel-wise it should be fine though due to: <> + .... mv out out~ -./build -B 'BR2_OPTIMIZE_3=y' +./build-buildroot --buildroot-config 'BR2_OPTIMIZE_3=y' .... === Find Buildroot options with make menuconfig @@ -8642,7 +9159,7 @@ mv out out~ `make menuconfig` is a convenient way to find Buildroot configurations: .... -cd "$(./getvar buildroot_out_dir)" +cd "$(./getvar buildroot_build_dir)" make menuconfig .... @@ -8715,7 +9232,7 @@ If you don't set it, the default is to use `~/.buildroot-ccache` with `5G`, whic I find it very relaxing to watch ccache at work with: .... -watch -n1 'make -C "$(./getvar buildroot_out_dir)" ccache-stats' +watch -n1 'make -C "$(./getvar buildroot_build_dir)" ccache-stats' .... or if you have it installed on host and the environment variables exported simply with: @@ -8730,8 +9247,8 @@ while a build is going on in another terminal and my cooler is humming. Especial First, see if you can't get away without actually adding a new package, for example: -* if you have a standalone C file with no dependencies besides the C standard library to be compiled with GCC, just add a new file under link:kernel_module/user[] and you are done -* if you have a dependency on a library, first check if Buildroot doesn't have a package for it already with `ls buildroot/package`. If yes, just enable that package as explained at: <> +* if you have a standalone C file with no dependencies besides the C standard library to be compiled with GCC, just add a new file under link:packages/kernel_modules/user[] and you are done +* if you have a dependency on a library, first check if Buildroot doesn't have a package for it already with `ls buildroot/package`. If yes, just enable that package as explained at: <> If none of those methods are flexible enough for you, create a new package as follows: @@ -8742,34 +9259,40 @@ If none of those methods are flexible enough for you, create a new package as fo * don't forget to rebuild with: + .... -./build -- sample_package-reconfigure -./run -F '/sample_package.out' +./build-buildroot -- sample_package-reconfigure +./run --eval-busybox '/sample_package.out' .... + -if you make any changes to that package after the initial build: <> +if you make any changes to that package after the initial build: <> === Build variants It often happens that you are comparing two versions of the build, a good and a bad one, and trying to figure out why the bad one is bad. -This section describes some techniques that can help to reduce the build time and disk usage in those situations. +Our build variants system allows you to keep multipmle built versions of all major components, so that you can easily switching between running one or the other. -==== Full builds variants +==== Buildroot build variants -The most coarse thing you can do is to keep two full checkouts of this repository, possibly with `git subtree`. - -This approach has the advantage of being simple and robust, but it wastes a lot of space and time for the full rebuild, since <> does not make compilation instantaneous due to configuration file reading. - -The next less coarse approach, is to use the `-s` option: +If you want to have multiple versions of the GCC toolchain or root filesystem, this is for you: .... -./build -s mybranch +# Build master. +./build-buildroot + +# Build another branch. +git -C "$(./getvar buildroot_src_dir)" checkout 2018.05 +./build-buildroot --buildroot-build-id 2018.05 + +# Restore master. +git -C "$(./getvar buildroot_src_dir)" checkout - + +# Run master. +./run + +# Run another branch. +./run --buildroot-build-id 2018.05 .... -which generates a full new build under `out/` named for example as `out/x86_64-mybranch`, but at least avoids copying up the source. - -TODO: only `-s` works for `./build`, e.g. if you want to `./run` afterwards you need to manually `mv` build around. This should be easy to patch however. - ==== Linux kernel build variants Since the Linux kernel is so important to us, we have created a convenient dedicated mechanism for it. @@ -8777,18 +9300,18 @@ Since the Linux kernel is so important to us, we have created a convenient dedic For example, if you want to keep two builds around, one for the latest Linux version, and the other for Linux `v4.16`: .... -./build -git -C linux checkout v4.16 -./build -L v4.16 -git -C linux checkout - +./build-buildroot +git -C "$(./getvar linux_src_dir)" checkout v4.16 +./build-buildroot --linux-build-id v4.16 +git -C "$(./getvar linux_src_dir)" checkout - ./run -./run -L v4.16 +./run --linux-build-id v4.16 .... The `-L` option should be passed to all scripts that support it, much like `-a` for the <>, e.g. to step debug: ..... -./rungdb -L v4.16 +./rungdb --linux-build-id v4.16 ..... This technique is implemented semi-hackishly by moving symlinks around inside the Buildroot build dir at build time, and selecting the right build directory at runtime. @@ -8798,12 +9321,12 @@ This technique is implemented semi-hackishly by moving symlinks around inside th Analogous to the <> but with the `-Q` option instead: .... -./build -git -C qemu checkout v2.12.0 -./build -Q v2.12.0 -q -git -C linux checkout - +./build-qemu +git -C "$(./getvar qemu_src_dir)" checkout v2.12.0 +./build-qemu --qemu-build-id v2.12.0 +git -C "$(./getvar qemu_src_dir)" checkout - ./run -./run -Q v2.12.0 +./run --qemu-build-id v2.12.0 .... ==== gem5 build variants @@ -8811,75 +9334,94 @@ git -C linux checkout - Analogous to the <> but with the `-M` option instead: .... -./build -g -git -C gem5/gem5 checkout some-branch -./build -g -M some-branch -git -C gem5/gem5 checkout - -./run -g -git -C gem5/gem5 checkout some-branch -./run -M some-branch -g +# Build master. +./build-gem5 + +# Build another branch. +git -C "$(./getvar gem5_src_dir)" checkout some-branch +./build-gem5 --gem5-build-id some-branch + +# Restore master. +git -C "$(./getvar gem5_src_dir)" checkout - + +# Run master. +./run --gem5 + +# Run another branch. +git -C "$(./getvar gem5_src_dir)" checkout some-branch +./run --gem5-build-id some-branch --gem5 .... Don't forget however that gem5 has Python scripts in its source code tree, and that those must match the source code of a given build. -Therefore, you can't forget to checkout to the sources to that of the corresponding build before running, unless you explicitly tell gem5 to use a non-default source tree with `-N`. +Therefore, you can't forget to checkout to the sources to that of the corresponding build before running, unless you explicitly tell gem5 to use a non-default source tree with `--gem5-worktree`. This becomes inevitable when you want to launch <>. ===== gem5 simultaneous runs with build variants -In order to checkout multiple gem5 builds and run them simultaneously, you also need to use the `-N` flag: +In order to checkout multiple gem5 builds and run them simultaneously, you also need to use the `--gem5-worktree` flag: .... -./build -g -git -C gem5/gem5 checkout some-branch -./build -g -M some-branch -N some-branch -git -C gem5/gem5 checkout - -./run -g -n 0 &>/dev/null & -./run -g -M some-branch -N some-branch -n 1 &>/dev/null & +# Build master. +./build-gem5 + +# Build another branch. +git -C "$(./getvar linux_src_dir)" checkout some-branch +./build-gem5 --gem5-build-id some-branch --gem5-worktree some-branch + +# Restore master. +git -C "$(./getvar linux_src_dir)" checkout - + +# Run master. +./run --gem5 --run-id 0 &>/dev/null & + +# Run another branch using the worktree for the scripts, +# without the need to check out anything. +./run --gem5 --gem5-build-id some-branch --gem5-worktree some-branch --run-id 1 &>/dev/null & .... -When `-N` is not given, the default source tree under `gem5/gem5` is used. +When `--gem5-worktree` is not given, the default source tree under `submodules/gem5` is used. -The `-N ` determines the location of the gem5 tree to be used for both: +The `--gem5-worktree ` determines the location of the gem5 tree to be used for both: * the input C files of the build at build time * the Python scripts to be used at runtime -The difference between `-M` and `-N` is that `-M` specifies the gem5 build output directory, while `-N` specifies the source input directory. +The difference between `--gem5-build-id` and `--gem5-worktree` is that `--gem5-build-id` specifies the gem5 build output directory, while `--gem5-worktree` specifies the source input directory. -If `-N ` is given, the directory used is `data/gem5/`, and: +If `--gem5-worktree ` is given, the directory used is `data/gem5/`, and: -* if that directory does not exist, create a `git worktree` at a branch `wt/` on current commit of `gem5/gem5` there. +* if that directory does not exist, create a `git worktree` at a branch `wt/` on current commit of `submodules/gem5` there. + -The `wt/` branch name prefix stands for `WorkTree`, and is done to allow us to checkout to a test `some-branch` branch under `gem5/gem5` and still use `-N some-branch`, without conflict for the worktree branch, which can only be checked out once. +The `wt/` branch name prefix stands for `WorkTree`, and is done to allow us to checkout to a test `some-branch` branch under `submodules/gem5` and still use `--gem5-worktree some-branch`, without conflict for the worktree branch, which can only be checked out once. * otherwise, leave that worktree untouched, without updating it Therefore, future builds for `worktree-id` will not automatically modify the revision of the worktree, and to do that you must manually check it out: .... git -C data/gem5/some-branch checkout some-branch-v2 -./build -g -M some-branch -N some-branch +./build-buildroot --gem5 --gem5-build-id some-branch --gem5-worktree some-branch .... -`-N` is only required if you have multiple gem5 checkouts, e.g. it would not be required for multiple builds of the same tree, e.g. a <> and a non-debug one. +`--gem5-worktree` is only required if you have multiple gem5 checkouts, e.g. it would not be required for multiple builds of the same tree, e.g. a <> and a non-debug one. ===== gem5 debug build Built and run `gem5.debug`, which has optimizations turned off unlike the default `gem5.opt`: .... -./build -a A -g -M debug -t debug -./run -a A -g -M debug -t debug +./build-gem5 --arch aarch64 --gem5-build-id debug --gem5-build-type debug +./run --arch aarch64 --gem5 --gem5-build-id debug --gem5-build-type debug .... -`-M` is optional just to prevent it from overwriting the `opt` build. +We generate a separate build folder with `--gem5-build-id` just to prevent the `opt` build from getting overwritten, so we can keep both around at the same time. A Linux kernel boot was about 14 times slower than opt at 71e927e63bda6507d5a528f22c78d65099bdf36f between the commands: .... -./run -a A -E 'm5 exit' -g -L v4.16 -./run -a A -E 'm5 exit' -g -M debug -t debug -L v4.16 +./run --arch aarch64 --eval 'm5 exit' --gem5 --linux-build-id v4.16 +./run --arch aarch64 --eval 'm5 exit' --gem5 --linux-build-id v4.16 --gem5-build-id debug --gem5-build-type debug .... Therefore the performance different is very big, making debug mode almost unusable. @@ -8889,13 +9431,13 @@ Therefore the performance different is very big, making debug mode almost unusab This hack-ish technique allows us to rebuild just one package at a time: .... -./build KERNEL_MODULE_VERSION=mybranch +./build-buildroot KERNEL_MODULE_VERSION=mybranch .... -and now you can see that a new version of `kernel_module` was built and put inside the image: +and now you can see that a new version of `kernel_modules` was built and put inside the image: .... -ls "$(./getvar build_dir)/kernel_module-mybranch" +ls "$(./getvar build_dir)/kernel_modules-mybranch" .... Unfortunately we don't have a nice runtime selection with `./run` implemented currently, you have to manually move packages around. @@ -8903,7 +9445,7 @@ Unfortunately we don't have a nice runtime selection with `./run` implemented cu TODO: is there a way to do it nicely for `*_OVERRIDE_SRCDIR` packages from link:buildroot_override[]? I tried: .... -./build -l LINUX_VERSION=mybranch +./build-buildroot -l LINUX_VERSION=mybranch .... but it fails with: @@ -8915,7 +9457,7 @@ linux/linux.mk:492: *** LINUX_SITE cannot be empty when LINUX_SOURCE is not. St and theI tried: .... -./build -l LINUX_VERSION=mybranch LINUX_SITE="$(pwd)/linux" +./build-buildroot -l LINUX_VERSION=mybranch LINUX_SITE="$(pwd)/linux" .... but it feels hackish, and the build was slower than normal, looks like the build was single threaded? @@ -8931,7 +9473,7 @@ Maybe you need to increase the filesystem size (BR2_TARGET_ROOTFS_EXT2_SIZE) The solution is to simply add: .... -./build -B 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"' +./build-buildroot --buildroot-config 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"' .... where 512Mb is "large enough". @@ -8941,7 +9483,7 @@ Note that dots cannot be used as in `1.5G`, so just use Megs as in `1500M` inste Unfortunately, TODO we don't have a perfect way to find the right value for `BR2_TARGET_ROOTFS_EXT2_SIZE`. One good heuristic is: .... -du -hsx "$(./getvar -a arm target_dir)" +du -hsx "$(./getvar --arch arm target_dir)" .... https://stackoverflow.com/questions/49211241/is-there-a-way-to-automatically-detect-the-minimum-required-br2-target-rootfs-ex @@ -8968,8 +9510,14 @@ which contributed to a large part of the slowness. Test how Buildroot deals with many files with: .... -./build -B BR2_PACKAGE_LKMC_MANY_FILES=y -- lkmc_many_files-reconfigure |& ts -i '%.s' -./build |& ts -i '%.s' +./build-buildroot \ + --buildroot-config 'BR2_PACKAGE_LKMC_MANY_FILES=y' \ + -- \ + lkmc_many_files-reconfigure \ + |& \ + ts -i '%.s' \ +; +./build-buildroot |& ts -i '%.s' .... and notice how the second build, which does not rebuilt the package at all, still gets stuck in the `RPATH` check forever without our Buildroot patch. @@ -8987,45 +9535,22 @@ We make that easy by building commands as strings, and then echoing them before So for example when you run: .... -./run -a arm +./run --arch arm .... -Stdout shows a line with the full command of type: +the very first stdout output of that script is the actual QEMU command that is being run. + +The command is also saved to a file for convenience: .... -time \ -/home/ciro/bak/git/linux-kernel-module-cheat/out/arm/buildroot/build/host-qemu-custom.default/arm-softmmu/qemu-system-arm \ --device rtl8139,netdev=net0 \ --gdb 'tcp::45457' \ --kernel '/home/ciro/bak/git/linux-kernel-module-cheat/out/arm/buildroot/build/linux-custom.default/arch/arm/boot/zImage' \ --m '256M' \ --monitor 'telnet::45454,server,nowait' \ --netdev 'user,hostfwd=tcp::45455-:45455,hostfwd=tcp::45456-:22,id=net0' \ --no-reboot \ --serial mon:stdio \ --smp '1' \ --trace 'enable=pr_manager_run,file=/home/ciro/bak/git/linux-kernel-module-cheat/out/arm/qemu/0/trace.bin' \ --virtfs 'local,path=/home/ciro/bak/git/linux-kernel-module-cheat/data/9p,mount_tag=host_scratch,security_model=mapped,id=host_scratch' \ --virtfs 'local,path=/home/ciro/bak/git/linux-kernel-module-cheat/out/arm/buildroot/build,mount_tag=host_out,security_model=mapped,id=host_out' \ --M virt,highmem=off \ --append 'root=/dev/vda console_msg_format=syslog nokaslr norandmaps panic=-1 printk.devkmsg=on printk.time=y' \ --cpu cortex-a15 \ --device virtio-gpu-pci \ --nographic \ --drive 'file=/home/ciro/bak/git/linux-kernel-module-cheat/out/arm/buildroot/images/rootfs.ext2.qcow2,format=qcow2,if=virtio,snapshot' \ -|& tee >(ts -s %.s > /home/ciro/bak/git/linux-kernel-module-cheat/out/arm/qemu/0/termout.txt) +cat "$(./getvar --arch arm run_cmd_file)" .... -and this line is also saved to a file for convenience: +which you can manually modify and execute during your experiments later: .... -cat "$(./getvar -a arm run_dir)/run.sh" -.... - -or for gem5: - -.... -cat "$(./getvar -a arm -g run_dir)/run.sh" +vim "$(./getvar --arch arm run_cmd_file)" +./"$(./getvar --arch arm run_cmd_file)" .... Next, you will also want to give the relevant images to save them time. Zip the images with: @@ -9035,7 +9560,15 @@ Next, you will also want to give the relevant images to save them time. Zip the ./zip-img .... -and then upload the `out/lkmc-*.zip` file somewhere, e.g. GitHub release assets as in https://github.com/cirosantilli/linux-kernel-module-cheat/releases/tag/test-replay-arm +Source: link:zip-img[] + +This generates a zip file: + +.... +out/lkmc-*.zip +.... + +which you can then upload somewhere, e.g. GitHub release assets as in https://github.com/cirosantilli/linux-kernel-module-cheat/releases/tag/test-replay-arm Finally, do a clone of the relevant repository out of tree and reproduce the bug there, to be 100% sure that it is an actual upstream bug, and to provide developers with the cleanest possible commands. @@ -9071,7 +9604,10 @@ We tried to automate it on Travis with link:.travis.yml[] but it hits the curren ==== Benchmark Linux kernel boot +Benchmark all: + .... +./build-all ./bench-boot cat "$(./getvar bench_boot)" .... @@ -9079,46 +9615,46 @@ cat "$(./getvar bench_boot)" Sample results at 2bddcc2891b7e5ac38c10d509bdfc1c8fe347b94: .... -cmd ./run -a x86_64 -E '/poweroff.out' -time 3.58 +cmd ./run --arch x86_64 --eval '/poweroff.out' +time 7.46 exit_status 0 -cmd ./run -a x86_64 -E '/poweroff.out' -K -time 0.89 + +cmd ./run --arch x86_64 --eval '/poweroff.out' --kvm +time 7.61 exit_status 0 -cmd ./run -a x86_64 -E '/poweroff.out' -T exec_tb -time 4.12 + +cmd ./run --arch x86_64 --eval '/poweroff.out' --trace exec_tb +time 8.04 exit_status 0 -instructions 2343768 -cmd ./run -a x86_64 -E 'm5 exit' -g -time 451.10 +instructions 1665023 + +cmd ./run --arch x86_64 --eval 'm5 exit' --gem5 +time 254.32 exit_status 0 -instructions 706187020 -cmd ./run -a arm -E '/poweroff.out' -time 1.85 +instructions 380799337 + +cmd ./run --arch arm --eval '/poweroff.out' +time 5.56 exit_status 0 -cmd ./run -a arm -E '/poweroff.out' -T exec_tb -time 1.92 + +cmd ./run --arch arm --eval '/poweroff.out' --trace exec_tb +time 5.78 exit_status 0 -instructions 681000 -cmd ./run -a arm -E 'm5 exit' -g -time 94.85 +instructions 742319 + +cmd ./run --arch aarch64 --eval '/poweroff.out' +time 4.85 exit_status 0 -instructions 139895210 -cmd ./run -a aarch64 -E '/poweroff.out' -time 1.36 + +cmd ./run --arch aarch64 --eval '/poweroff.out' --trace exec_tb +time 4.91 exit_status 0 -cmd ./run -a aarch64 -E '/poweroff.out' -T exec_tb -time 1.37 +instructions 245471 + +cmd ./run --arch aarch64 --eval 'm5 exit' --gem5 +time 68.71 exit_status 0 -instructions 178879 -cmd ./run -a aarch64 -E 'm5 exit' -g -time 72.50 -exit_status 0 -instructions 115754212 -cmd ./run -a aarch64 -E 'm5 exit' -g -- --cpu-type=HPI --caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB -time 369.13 -exit_status 0 -instructions 115774177 +instructions 120555566 .... TODO: aarch64 gem5 and QEMU use the same kernel, so why is the gem5 instruction count so much much higher? @@ -9128,7 +9664,7 @@ TODO: aarch64 gem5 and QEMU use the same kernel, so why is the gem5 instruction TODO 62f6870e4e0b384c4bd2d514116247e81b241251 takes 33 minutes to finish at 62f6870e4e0b384c4bd2d514116247e81b241251: .... -cmd ./run -a arm -E 'm5 exit' -g -- --caches --cpu-type=HPI +cmd ./run --arch arm --eval 'm5 exit' --gem5 -- --caches --cpu-type=HPI .... while aarch64 only 7 minutes. @@ -9155,7 +9691,7 @@ The build times are calculated after doing `./configure` and link:https://buildr Sample build time at 2c12b21b304178a81c9912817b782ead0286d282: 28 minutes, 15 with full ccache hits. Breakdown: 19% GCC, 13% Linux kernel, 7% uclibc, 6% host-python, 5% host-qemu, 5% host-gdb, 2% host-binutils -Single file change on `./build kernel_module-reconfigure`: 7 seconds. +Single file change on `./build-buildroot kernel_modules-reconfigure`: 7 seconds. Buildroot automatically stores build timestamps as milliseconds since Epoch. Convert to minutes: @@ -9170,32 +9706,29 @@ Or to conveniently do a clean build without affecting your current one: cat ../linux-kernel-module-cheat-regression/*/build-time.log .... -===== Find which packages are making the build slow +===== Find which packages are making the build slow and big .... -cd "$(./getvar buildroot_out_dir) -make graph-build graph-depends -xdg-open graphs/build.pie-packages.pdf -xdg-open graphs/graph-depends.pdf +./build-buildroot --skip-configure -- graph-build graph-size graph-depends +cd "$(./getvar buildroot_build_dir)/graphs" +xdg-open build.pie-packages.pdf +xdg-open graph-depends.pdf +xdg-open graph-size.pdf .... Our philosophy is: -* if something adds little to the build time, build it in by default -* otherwise, make it optional -* try to keep the toolchain (GCC, Binutils) unchanged, otherwise a full rebuild is required. -+ -So we generally just enable all toolchain options by default, even though this adds a bit of time to the build. -* if something is very valuable, we just add it by default even if it increases the Build time, notably GDB and QEMU -* runtime is sacred. -+ -We do our best to reduce the instruction and feature count to the bare minimum needed, to make the system: +* keep the root filesystem as tiny as possible to make prebuilts small. It is easy to add new packages once you have the toolchain. +* enable every feature possible on the toolchain (GCC, Binutils), because changes imply Buildroot rebuilds +* runtime is sacred. Faster systems are: + -- ** easier to understand -** run faster, specially for <> +** run faster, which is specially for <> which is slow -- + +Runtime basically just comes down to how we configure the Linux kernel, since in the root filesystem all that matters is `init=`, and that is easy to control. ++ One possibility we could play with is to build loadable modules instead of built-in modules to reduce runtime, but make it easier to get started with the modules. [[prebuilt-toolchain]] @@ -9249,7 +9782,7 @@ Sample results at gem5 2a9573f5942b5416fb0570cf5cb6cdecba733392: 10 to 12 minute Get results with: .... -./bench-all -g +./bench-all --gem5 tail -n+1 ../linux-kernel-module-cheat-regression/*/gem5-bench-build-*.txt .... @@ -9288,155 +9821,181 @@ gem5: ** https://stackoverflow.com/questions/47997565/gem5-system-requirements-for-decent-performance/48941793#48941793 ** https://github.com/gem5/gem5/issues/25 -== Conversation +== About this repo -=== kmod +=== Run command after boot -https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git - -Multi-call executable that implements: `lsmod`, `insmod`, `rmmod`, and other tools on desktop distros such as Ubuntu 16.04, where e.g.: +If you just want to run a command after boot ends without thinking much about it, just use the `--eval-busybox` option, e.g.: .... -ls -l /bin/lsmod +./run --eval-busybox 'echo hello' .... -gives: +This option passes the command to our init scripts through <>, and uses a few clever tricks along the way to make it just work. + +See <> for the gory details. + +=== Default command line arguments + +It gets annoying to retype `--arch aarch64` for every single command, or to remember `--buildroot-config` setups. + +So simplify that, do: .... -lrwxrwxrwx 1 root root 4 Jul 25 15:35 /bin/lsmod -> kmod +cp config.example data/config .... -and: +and then edit the `data/config` file to your needs. + +=== Clean the build + +You did something crazy, and nothing seems to work anymore? + +All our build outputs are stored under `out/`, so the coarsest and most effective thing you can do is: .... -dpkg -l | grep -Ei +rm -rf out .... -contains: +This implies a full rebuild for all archs however, so you might first want to explore finer grained cleans first. + +All our individual `build-*` scripts have a `--clean` option to completely nuke their builds: .... -ii kmod 22-1ubuntu5 amd64 tools for managing Linux kernel modules +./build-gem5 --clean +./build-qemu --clean +./build-buildroot --clean .... -BusyBox also implements its own version of those executables. There are some differences. - -Buildroot also has a kmod package, but we are not using it since BusyBox' version is good enough so far. - -This page will only describe features that differ from kmod to the BusyBox implementation. - -==== module-init-tools - -Name of a predecessor set of tools. - -==== kmod modprobe - -kmod's `modprobe` can also load modules under different names to avoid conflicts, e.g.: +Verify with: .... -sudo modprobe vmhgfs -o vm_hgfs +ls "$(./getvar qemu_build_dir)" +ls "$(./getvar gem5_build_dir)" +ls "$(./getvar buildroot_build_dir)" .... -=== Device tree +Note that host tools like QEMU and gem5 store all archs in a single directory to factor out build objects, so cleaning one arch will clean all of them. -<> contains a minimal runnable example. - -Good format descriptions: - -* https://www.raspberrypi.org/documentation/configuration/device-tree.md - -Minimal example +To only nuke one one Buildroot package, we can use the link:https://buildroot.org/downloads/manual/manual.html#pkg-build-steps[]`-dirclean`] Buildroot target: .... -/dts-v1/; - -/ { - a; -}; +./build-buildroot --no-all -- -dirclean .... -Check correctness with: +e.g.: .... -dtc a.dts +./build-buildroot --no-all -- kernel_modules-dirclean .... -Separate nodes are simply merged by node path, e.g.: +Verify with: .... -/dts-v1/; - -/ { - a; -}; - -/ { - b; -}; +ls "$(./getvar build_dir)" .... -then `dtc a.dts` gives: +=== Build the documentation + +You don't need to depend on GitHub: .... -/dts-v1/; - -/ { - a; - b; -}; +./build-doc +xdg-open out/README.html .... -==== Get device tree from running kernel +Source: link:build-doc[] -https://unix.stackexchange.com/questions/265890/is-it-possible-to-get-the-information-for-a-device-tree-using-sys-of-a-running/330926#330926 +=== Simultaneous runs -This is specially interesting because QEMU and gem5 are capable of generating DTBs that match the selected machine depending on dynamic command line parameters for some types of machines. +When doing long simulations sweeping across multiple system parameters, it becomes fundamental to do multiple simulations in parallel. -QEMU's `-M virt` for example, which we use by default for `aarch64`, boots just fine without the `-dtb` option: +This is specially true for gem5, which runs much slower than QEMU, and cannot use multiple host cores to speed up the simulation: link:https://github.com/cirosantilli-work/gem5-issues/issues/15[], so the only way to parallelize is to run multiple instances in parallel. + +This also has a good synergy with <>. + +First shell: .... -./run -a aarch64 +./run .... -Then, from inside the guest: +Another shell: .... -dtc -I fs -O dts /sys/firmware/devicetree/base +./run --run-id 1 .... -contains: +and now you have two QEMU instances running in parallel. + +The default run id is `0`. + +Our scripts solve two difficulties with simultaneous runs: + +* port conflicts, e.g. GDB and link:gem5-shell[] +* output directory conflicts, e.g. traces and gem5 stats overwriting one another + +Each run gets a separate output directory. For example: .... - cpus { - #address-cells = <0x1>; - #size-cells = <0x0>; - - cpu@0 { - compatible = "arm,cortex-a57"; - device_type = "cpu"; - reg = <0x0>; - }; - }; +./run --arch aarch64 --gem5 --run-id 0 &>/dev/null & +./run --arch aarch64 --gem5 --run-id 1 &>/dev/null & .... -However, if we increase the <>: +produces two separate `m5out` directories: .... -./run -a aarch64 -c 2 +echo "$(./getvar --arch aarch64 --gem5 --run-id 0 m5out_dir)" +echo "$(./getvar --arch aarch64 --gem5 --run-id 1 m5out_dir)" .... -QEMU automatically adds a second CPU to the DTB! +and the gem5 host executable stdout and stderr can be found at: -The action seems to be happening at: `hw/arm/virt.c`. +.... +less "$(./getvar --arch aarch64 --gem5 --run-id 0 termout_file)" +less "$(./getvar --arch aarch64 --gem5 --run-id 1 termout_file)" +.... -<> 2a9573f5942b5416fb0570cf5cb6cdecba733392 can also generate its own DTB. +Each line is prepended with the timestamp in seconds since the start of the program when it appeared. + +To have more semantic output directories names for later inspection, you can use a non numeric string for the run ID, and indicate the port offset explicitly: + +.... +./run --arch aarch64 --gem5 --run-id some-experiment --port-offset 1 +.... + +`--port-offset` defaults to the run ID when that is a number. + +Like <>, you will need to pass the `-n` option to anything that needs to know runtime information, e.g. <>: + +.... +./run --run-id 1 +./rungdb --run-id 1 +.... + +To run multiple gem5 checkouts, see: <>. + +Implementation note: we create multiple namespaces for two things: + +* run output directory +* ports +** QEMU allows setting all ports explicitly. ++ +If a port is not free, it just crashes. ++ +We assign a contiguous port range for each run ID. +** gem5 automatically increments ports until it finds a free one. ++ +gem5 60600f09c25255b3c8f72da7fb49100e2682093a does not seem to expose a way to set the terminal and VNC ports from `fs.py`, so we just let gem5 assign the ports itself, and use `-n` only to match what it assigned. Those ports both appear on `config.ini`. ++ +The GDB port can be assigned on `gem5.opt --remote-gdb-port`, but it does not appear on `config.ini`. === Directory structure * `data`: gitignored user created data. Deleting this might lead to loss of data. Of course, if something there becomes is important enough to you, git track it. -** `data/readfile`: see <> ** `data/9p`: see <<9p>> ** `data/gem5/`: see: <> -* link:kernel_module[]: Buildroot package that contains our kernel modules and userland C tests +* link:packages/kernel_modules[]: Buildroot package that contains our kernel modules and userland C tests * `out`: gitignored Build outputs. You won't lose data by deleting this folder since everything there can be re-generated, only time. ** `out/`: arch specific outputs *** `out//buildroot`: standard Buildroot output @@ -9464,7 +10023,17 @@ This directory has the following structure: ==== packages directory -Any directory in that subdirectory is added to `BR2_EXTERNAL` and become available to the build. +Every directory inside it is a Buildroot package. + +Those packages get automatically added to Buildroot's `BR2_EXTERNAL`, so all you need to do is to turn them on during build, e.g.: + +.... +./build-buildroot --buildroot-config BR2_SAMPLE_PACKAGE=y +.... + +If you want to add something to the root filesystem, possibly involving cross-compilation, then packages are the way to go. + +In particular, our kernel modules are stored inside a Buildroot package: link:packages/kernel_modules[]. ==== patches @@ -9501,175 +10070,7 @@ We use it for: * customized configuration files * userland module test scripts that don't need to be compiled. + -C files for example need compilation, and must go through the regular package system, e.g. through link:kernel_module/user[]. - -=== Script man pages - -These appear when you do `./some-script -h`. - -We have to keep them as separate files from the README for that to be possible. - -==== getvar - -The link:getvar[] helper script prints the value of a variable from the link:common[] file. - -For example, to get the Buildroot output directory for an ARM build, you can use: - -.... -./getvar -a arm buildroot_out_dir -.... - -This script exists mostly to factor out instructions given on the README which users are expected to copy paste into the terminal. - -Otherwise, it becomes very difficult to keep everything working across path refactors, since README snippets cannot be tested automatically. - -==== build - -.... -./build [OPTIONS] [-- EXTRA_MAKE_ARGS] -.... - -`EXTRA_MAKE_ARGS` gets appended to the end of the `make` command line. - -===== build configuration options - -[options="header"] -|=== -|Name |Argument name |Description -|`-a` |`ARCH` |Build for architecture `ARCH`. -|`-B` |`BR2_CONFIG` |Add a single Buildroot option to the current build. - Example: `-B 'BR2_TARGET_ROOTFS_EXT2_SIZE="512M"'` -|`-b` |`BR2_CONFIG_FILE` |Also use the given Buildroot configuration fragment file. - Pass multiple times to use multiple fragment files. -|`-C` |`CONFIG_SOMETHING` |Also use the given Linux kernel configuration, example: - `./build -c 'CONFIG_FORTIFY_SOURCE=y'` - Can be used multiple times for multiple configs. - These options take precedence over `-c`. -|`-c` |`KERNEL_CONFIG_FILE` |Also use the given kernel configuration fragment file. - Pass multiple times to use multiple fragment files. - These options take precedence over `-K`. -|`-f` | |Skip the Buildroot configuration. Saves a few seconds, - but requires you to know what you are doing :-) - Mnemonic: `fast`. -|`-g` | |Enable gem5 build or force its rebuild. -|`-h` | |Show this help message. -|`-L` |`VARIANT` |Linux kernel build variant. -|`-I` | |Enable initramfs for the current build. -|`-i` | |Enable initrd for the current build. -|`-K` |`KERNEL_CONFIG_FILE` |Use `KERNEL_CONFIG_FILE` as the exact Linux kernel - configuration. Ignore the default kernel config fragments, - but still add options explicitly passed with `-C` and `-c`. - on top of it. -|`-M` |`VARIANT` |gem5 build variant. -|`-p` | |Pass extra arguments to the `rootfs_post_build_script`. -|`-Q` |`VARIANT`` |QEMU build variant. -|`-S` | |Don't build QEMU with SDL support. - Graphics such as X11 won't work, only the terminal. -|`-s` | |Add a custom suffix to the build. - E.g., doing `./build -s mysuf` puts all the build output - into `out/x86_64-mysuf`. This allows keep multiple builds around - when you checkout between branches. -|`-v` | |Do a verbose build. -|=== - -===== build target options - -[options="header"] -|=== -|Name |Forces rebuild of |Extra actions -|`-g` |gem5 | -|`-k` |Kernel modules | -|`-l` |Linux kernel |Touches kernel configuration files to overcome: - https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel-custom-config-file-and-run-make-linux-reconfi -|`-q` |QEMU | -|=== - -==== run - -.... -./run [OPTIONS] [-- EXTRA_RUN_ARGS] -.... - -`EXTRA_RUN_ARGS` gets appended to the end of the emulator command line. - -[options="header"] -|=== -|Name |Argument name |Description -|`-a` |`ARCH` |Run architecture `ARCH`. -|`-c` |`NCPUS` |Emulate `NCPUS` guest CPUs. -|`-D` | |Run GDB on the emulator itself. -|`-d` | |Wait for GDB to connect before starting execution. -|`-E` |`CMDSTR` |Replace the normal init with a minimal init that just evals - with given `CMDSTR` bash command string. Example: - `-E 'insmod /hello.ko;'` -|`-e` |`CLI_OPTIONS` |Pass an extra Linux kernel command line options, - and place them before the dash separator `-`. - Only options that come before the `-`, i.e. "standard" - options, should be passed with this option. - Example: `./run -a arm -e 'init=/poweroff.out'` -|`-F` |`CMDSTR` |Much like `-f`, but base64 encodes the string. - Mnemonic: `-F` is to `-f` what `-E` is to `-e`. -|`-f` |`CLI_OPTIONS` |Pass an extra Linux kernel command line options, - add a dash `-` separator, and place the options after the dash. - Intended for custom options understood by our `init` scripts, - most of which are prefixed by `lkmc_`, e.g.: - `./run -f 'lkmc_eval="wget google.com" lkmc_lala=y'` - Mnenomic: comes after `-e`. -|`-G` | |Pass extra options to the gem5 executable. - Do not confuse with the arguments passed to config scripts, - like `fs.py`. Example: `./run -G '--debug-flags=Exec --debug' -g` -|`-g` | |Use gem5 instead of QEMU. -|`-h` | |Show this help message. -|`-I` | |Run with initramfs. -|`-i` | |Run with initrd. -|`-K` | |Use KVM. Only works if guest arch == host arch. -|`-k` | |Enable KGDB. -|`-L` |`VARIANT` |Linux kernel build variant. -|`-l` |`CHECKPOINT` |Restore the nth most recently taken gem5 checkpoint according to - directory timestamps. -|`-M` |`VARIANT` |gem5 build output variant. -|`-m` | |Set the memory size of the guest. E.g.: `-m 512M`. Default: `256M`. - The default is the minimum amount that boots all archs without extra - options added. Anything lower will lead some arch to fail to boot. - Any -|`-N` |`VARIANT` |gem5 source input variant. -|`-n` | |Run ID. -|`-R` | |Replay a QEMU run record deterministically. -|`-r` | |Record a QEMU run record for later replay with `-R`. -|`-P` | |Run the downloaded prebuilt images. -|`-Q` |`VARIANT`` |QEMU build variant. -|`-T` |`TRACE_TYPES` |Set trace events to be enabled. - If not given, gem5 tracing is completely disabled, while QEMU tracing - is enabled but uses default traces that are very rare and don't affect - performance. `./configure --enable-trace-backends=simple` seems to enable - some traces by default, e.g. `pr_manager_run`, and I don't know how to - get rid of them. -|`-U` | |Pass extra parameters to the program running on the `-u` tmux split. -|`-u` | |Create a tmUx split the window. - You must already be inside of a `tmux` session to use this option. - * on the main window, run the emulator as usual - * on the split: - ** if on QEMU and `-d` is given, GDB - ** if on gem5, the gem5 terminal -|`-V` | |Run QEMU with VNC instead of the default SDL. - Connect to it with: `vinagre localhost:5900`. -|`-X` |`EXTRA_OPTS` |Extra options that did not fit into `A-z`! - This string is parsed by `getopt` on a separate parse step with different - meanings for each flag. -|`-X-b` | |Use `fs_bigLITTLE.py` instead of `fs.py` on gem5 simulation. - Ignored by QEMU. -|`-x` | |Run in graphic mode. Mnemonic: X11. -|=== - -==== runtc - -The link:runtc[] helper script runs a Tool Chain executable built by Buildroot. - -For example, to run `readelf -h` for the `arm` architecture, use: - -.... -./runtc -a arm readelf -h -.... +C files for example need compilation, and must go through the regular package system, e.g. through link:packages/kernel_modules/user[]. === CONTRIBUTING @@ -9697,7 +10098,7 @@ Sources: Test that the Internet works: .... -./run -a x86_64 -e '- lkmc_eval="/sbin/ifup -a;wget -S google.com;poweroff;"' +./run --arch x86_64 --kernel-cli '- lkmc_eval="/sbin/ifup -a;wget -S google.com;poweroff;"' .... Source: link:rootfs_overlay/test_all.sh[]. @@ -9707,7 +10108,7 @@ Source: link:rootfs_overlay/test_all.sh[]. Shell 1: .... -./run -d +./run --debug-guest .... Shell 2: @@ -9734,33 +10135,33 @@ For example, when updating from QEMU `v2.12.0` to `v3.0.0-rc3`, the Linux kernel We then bisected it as explained at: https://stackoverflow.com/questions/4713088/how-to-use-git-bisect/22592593#22592593 with the link:qemu-bisect-boot[] script: .... -cd qemu +root_dir="$(pwd)" +cd "$(./getvar qemu_src_dir)" git bisect start # Check that our test script fails on v3.0.0-rc3 as expected, and mark it as bad. -../qemu-bisect-boot +"${root_dir}/qemu-bisect-boot" # Should output 1. echo #? git bisect bad # Same for the good end. git checkout v2.12.0 -../qemu-bisect-boot +"${root_dir}/qemu-bisect-boot" # Should output 0. echo #? git bisect good # This leaves us at the offending commit. -git bisect run ../qemu-bisect-boot +git bisect run ../biset-qemu-linux-boot # Clean up after the bisection. git bisect reset -cd .. git submodule update -rm -rf "$(./getvar -a arm build_dir)/host-qemu-custom.bisect" +"${root_dir}/build-qemu" --clean --qemu-build-id bisect .... -An example of Linux kernel commit bisection on gem5 boots can be found at: link:linux-bisect-boot-gem5[]. +An example of Linux kernel commit bisection on gem5 boots can be found at: link:bisect-linux-boot-gem5[]. ==== Update a forked submodule @@ -9772,7 +10173,7 @@ This example is based on the Linux kernel, for which we used to have patches, bu # Last point before out patches. last_mainline_revision=v4.15 next_mainline_revision=v4.16 -cd linux +cd "$(./getvar linux_src_dir)" # Create a branch before the rebase in case things go wrong. git checkout -b "lkmc-${last_mainline_revision}" @@ -9806,18 +10207,18 @@ hello cpp Sources: -* link:kernel_module/user/hello.c[] -* link:kernel_module/user/hello_cpp.c[] +* link:packages/kernel_modules/user/hello.c[] +* link:packages/kernel_modules/user/hello_cpp.c[] ===== rand_check.out Print out several parameters that normally change randomly from boot to boot: .... -./run -F '/rand_check.out;/poweroff.out' +./run --eval-busybox '/rand_check.out;/poweroff.out' .... -Source: link:kernel_module/user/rand_check.c[] +Source: link:packages/kernel_modules/user/rand_check.c[] This can be used to check the determinism of: @@ -9826,55 +10227,15 @@ This can be used to check the determinism of: ==== Releases -This is not yet super stable, but one day maybe: +This is not yet super stable, but one day maybe this script will automatically do a release: .... -./do-release +./release .... -Source: link:do-release[]. +Source: link:release[]. -This should in particular enable <>. - -=== About - -This project is for people who want to learn and modify low level system components: - -* Linux kernel and Linux kernel modules -* full systems emulators like QEMU and gem5 -* C standard libraries. This could also be put on a submodule if people show interest. -* Buildroot. We use and therefore document, a large part of its feature set. - -Philosophy: - -* automate as much as possible to make things more reproducible -* do everything from source to make things understandable and hackable - -This project should be called "Linux kernel playground", like: https://github.com/Fuzion24/AndroidKernelExploitationPlayground maybe I'll rename it some day. Would semi conflict with: http://copr-fe.cloud.fedoraproject.org/coprs/jwboyer/kernel-playground/ though. - -==== Fairy tale - -____ -Once upon a time, there was a boy called Linus. - -Linus made a super fun toy, and since he was not very humble, decided to call it Linux. - -Linux was an awesome toy, but it had one big problem: it was very difficult to learn how to play with it! - -As a result, only some weird kids who were very bored ended up playing with Linux, and everyone thought those kids were very cool, in their own weird way. - -One day, a mysterious new kid called Ciro tried to play with Linux, and like many before him, got very frustrated, and gave up. - -A few years later, Ciro had grown up a bit, and by chance came across a very cool toy made by the boy Petazzoni and his gang: it was called Buildroot. - -Ciro noticed that if you used Buildroot together with Linux, Linux suddenly became very fun to play with! - -So Ciro decided to explain to as many kids as possible how to use Buildroot to play with Linux. - -And so everyone was happy. Except some of the old weird kernel hackers who wanted to keep their mystique, but so be it. - -THE END -____ +This should in particular enable to easily update <>. === Bibliography diff --git a/bench-all b/bench-all index 8610fdf4..845c22fe 100755 --- a/bench-all +++ b/bench-all @@ -1,8 +1,9 @@ #!/usr/bin/env bash -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common" -set -x -bench_build=false -bench_buildroot_baseline=false +set -eux +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +bench_all=false +bench_buildroot_build=false +bench_buildroot_baseline_build=false bench_gem5_build=false bench_linux_boot=false default_arch=x86_64 @@ -10,19 +11,16 @@ update_repo=false while getopts Aa:Bbglu OPT; do case "$OPT" in A) - bench_build=true - bench_buildroot_baseline=true - bench_gem5_build=true - bench_linux_boot=true + bench_all=true ;; a) default_arch="$OPTARG" ;; b) - bench_build=true + bench_buildroot_build=true ;; B) - bench_buildroot_baseline=true + bench_buildroot_baseline_build=true ;; g) bench_gem5_build=true @@ -40,9 +38,24 @@ while getopts Aa:Bbglu OPT; do done shift "$(($OPTIND - 1))" comment="${1:-}" +if \ + ! "$bench_buildroot_build" && \ + ! "$bench_buildroot_baseline_build" && \ + ! "$bench_gem5_build" && \ + ! "$bench_linux_boot" \ +; then + bench_all=true +fi +if "$bench_all"; then + bench_buildroot_build=true + bench_buildroot_baseline_build=true + bench_gem5_build=true + bench_linux_boot=true +fi +getvar="${root_dir}/getvar" # Create output directory. -benchmark_repo="${common_root_dir}/../linux-kernel-module-cheat-regression" +benchmark_repo="${root_dir}/../linux-kernel-module-cheat-regression" mkdir -p "$benchmark_repo" last_dir="$(ls "$benchmark_repo" | grep -E '^[0-9]' | tail -n 1)" if [ -n "$last_dir" ]; then @@ -51,62 +64,61 @@ else seq_id=0 fi seq_id="$(printf '%0.4d' "$seq_id")" -dir_basename="${seq_id}_${common_sha}" +dir_basename="${seq_id}_$("$getvar" sha)" new_dir="${benchmark_repo}/${dir_basename}" mkdir "$new_dir" -if "$bench_build"; then - common_arch="$default_arch" - common_suffix=bench - common_setup - rm -rf "$common_out_arch_dir" - ./build -a "$common_arch" -B 'BR2_CCACHE=n' -s "$common_suffix" - cp "${common_build_dir}/build-time.log" "${new_dir}/build-time-${common_arch}.log" - rm -rf "$common_out_arch_dir" +do_bench_buildroot_build() ( + arch="$default_arch" + build_id=bench + if [ "${1:-}" = baseline ]; then + baseline=--baseline + baseline_suffix=-baseline + else + baseline= + baseline_suffix= + fi + common_build_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" build_dir)" + common_images_dir="$("$getvar" --arch "$arch" --buildroot-build-id "$build_id" images_dir)" + "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --clean + "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --no-all -- source + "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" + cp "${common_build_dir}/build-time.log" "${new_dir}/buildroot-build-time-${baseline_suffix}${arch}.log" + wc -c "${images_dir}/"* > "${new_dir}/buildroot-image-size-${baseline_suffix}${arch}.log" + "${root_dir}/build-buildroot" --arch "$arch" $baseline --buildroot-build-id "$build_id" --clean +) + +if "$bench_buildroot_build"; then + do_bench_buildroot_build fi -if "$bench_buildroot_baseline"; then - cd "${common_root_dir}/buildroot" - git clean -xdf - make "qemu_${default_arch}_defconfig" - printf ' -BR2_CCACHE=y -BR2_TARGET_ROOTFS_CPIO=y -BR2_TARGET_ROOTFS_EXT2=n -' >>.config - make olddefconfig - make source - time env -u LD_LIBRARY_PATH make BR2_JLEVEL="$(nproc)" - cp output/build/build-time.log "${new_dir}/baseline-build-time-${default_arch}.log" - wc -c output/images/* > "${new_dir}/baseline-image-size-${default_arch}.log" - git clean -xdf +if "$bench_buildroot_baseline_build"; then + do_bench_buildroot_build baseline fi if "$bench_gem5_build"; then - arches='x86_64 arm' - for common_arch in $arches; do - common_setup - cd "${common_gem5_src_dir}" - git clean -xdf - results_file="${common_gem5_out_dir}/bench-build.txt" - gem5_outdir="${common_out_dir}/bench_build" - rm -fr "$results_file" "${gem5_outdir}" - # TODO understand better: --foreground required otherwise we cannot - # kill the build with Ctrl+C if something goes wrong, can be minimized to: - # bash -c "eval 'timeout 5 sleep 3'" - common_bench_cmd "timeout --foreground 900 ../build -a '$common_arch' -o '${gem5_outdir}'" "$results_file" - cp "$results_file" "${new_dir}/gem5-bench-build-${common_arch}.txt" - cd "${common_root_dir}/gem5/gem5" - git clean -xdf - rm -fr "${gem5_outdir}" - done + common_arch="$default_arch" + gem5_build_id=bench-build + common_gem5_build_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_out_dir)" + common_gem5_src_dir="$("$getvar" --arch "$common_arch" --gem5-build-id "$gem5_build_id" gem5_src_dir)" + results_file="${common_gem5_build_dir}/bench-build.txt" + git -C "${common_gem5_src_dir}" clean -xdf + rm -f "$results_file" + "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id" + # TODO understand better: --foreground required otherwise we cannot + # kill the build with Ctrl+C if something goes wrong, can be minimized to: + # bash -c "eval 'timeout 5 sleep 3'" + "${root_dir}/bench-cmd" "timeout --foreground 900 ./build-gem5 --arch '$common_arch' --gem5-build-id '$gem5_build_id'" "$results_file" + cp "$results_file" "${new_dir}/gem5-bench-build-${common_arch}.txt" + git -C "${common_gem5_src_dir}" clean -xdf + "${root_dir}/build-gem5" --arch "$common_arch" --clean --gem5-build-id "$gem5_build_id" fi if "$bench_linux_boot"; then - cd "${common_root_dir}" - ./build-all - ./bench-boot - cp "$common_bench_boot" "$new_dir" + cd "${root_dir}" + "${root_dir}/build-all" + "${root_dir}/bench-boot" -t 3 + cp "$(${root_dir}/getvar bench_boot)" "$new_dir" fi if "$update_repo"; then diff --git a/bench-boot b/bench-boot index fd5212bd..3c04ac46 100755 --- a/bench-boot +++ b/bench-boot @@ -1,5 +1,6 @@ #!/usr/bin/env bash -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common" +set -eu +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" test_size=1 OPTIND=1 while getopts t: OPT; do @@ -16,56 +17,83 @@ while getopts t: OPT; do esac done shift "$(($OPTIND - 1))" -extra_args="$*" - +if [ $# -gt 1 ]; then + extra_args=" $*" +else + extra_args= +fi +getvar="${root_dir}/getvar" +common_bench_boot="$("$getvar" bench_boot)" caches='--caches --l2cache --l1d_size=1024kB --l1i_size=1024kB --l2_size=1024kB --l3_size=1024kB' + bench() ( - common_bench_cmd "./run -a ${1} ${extra_args}" "$common_bench_boot" - echo >> "$common_bench_boot" + "${root_dir}/bench-cmd" "./run --arch ${1}${extra_args}" "$common_bench_boot" ) + gem5_insts() ( - printf "instructions $(./gem5-stat -a "$1" sim_insts)\n" >> "$common_bench_boot" + printf "instructions $(./gem5-stat --arch "$1" sim_insts)\n" >> "$common_bench_boot" ) + qemu_insts() ( common_arch="$1" - ./qemu-trace2txt -a "$common_arch" - common_setup - printf "instructions $(wc -l "${common_trace_txt_file}" | cut -d' ' -f1)\n" >> "$common_bench_boot" + ./qemu-trace2txt --arch "$common_arch" + common_qemu_trace_txt_file="$("$getvar" --arch "$common_arch" qemu_trace_txt_file)" + printf "instructions $(wc -l "${common_qemu_trace_txt_file}" | cut -d' ' -f1)\n" >> "$common_bench_boot" ) + +newline() ( + echo >> "$common_bench_boot" +) + rm -f "${common_bench_boot}" arch=x86_64 -bench "$arch -E '/poweroff.out'" -bench "$arch -E '/poweroff.out' -K" +bench "${arch} --eval '/poweroff.out'" +newline +bench "${arch} --eval '/poweroff.out' --kvm" +newline if [ "$test_size" -ge 2 ]; then - bench "$arch -E '/poweroff.out' -T exec_tb" + bench "${arch} --eval '/poweroff.out' --trace exec_tb" qemu_insts "$arch" -fi -if [ "$test_size" -ge 2 ]; then - bench "$arch -E 'm5 exit' -g" + newline + bench "$arch --eval 'm5 exit' --gem5" gem5_insts "$arch" + newline fi -#bench "$arch -E 'm5 exit' -g -- --cpu-type=DerivO3CPU ${caches}" +#bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=DerivO3CPU ${caches}" #gem5_insts "$arch" +#newline arch=arm -bench "$arch -E '/poweroff.out'" +bench "$arch --eval '/poweroff.out'" +newline if [ "$test_size" -ge 2 ]; then - bench "$arch -E '/poweroff.out' -T exec_tb" + bench "$arch --eval '/poweroff.out' --trace exec_tb" qemu_insts "$arch" + newline + #bench "$arch --eval 'm5 exit' --gem5" + #gem5_insts "$arch" + #newline fi -#bench "$arch -E 'm5 exit' -g" -#gem5_insts "$arch" -#bench "$arch -E 'm5 exit' -g -- --cpu-type=HPI ${caches}" -#gem5_insts "$arch" +#if [ "$test_size" -ge 3 ]; then +# bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}" +# gem5_insts "$arch" +# newline +#fi arch=aarch64 -bench "$arch -E '/poweroff.out'" +bench "$arch --eval '/poweroff.out'" +newline if [ "$test_size" -ge 2 ]; then - bench "$arch -E '/poweroff.out' -T exec_tb" + bench "$arch --eval '/poweroff.out' --trace exec_tb" qemu_insts "$arch" + newline + bench "$arch --eval 'm5 exit' --gem5" + gem5_insts "$arch" + newline +fi +if [ "$test_size" -ge 3 ]; then + bench "$arch --eval 'm5 exit' --gem5 -- --cpu-type=HPI ${caches}" + gem5_insts "$arch" + newline fi -#bench "$arch -E 'm5 exit' -g" -#gem5_insts "$arch" -#bench "$arch -E 'm5 exit' -g -- --cpu-type=HPI ${caches}" -#gem5_insts "$arch" diff --git a/bench-cmd b/bench-cmd new file mode 100755 index 00000000..5b1dbe22 --- /dev/null +++ b/bench-cmd @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Benchmark a command as a string and output results +# to a file with format: +# +# cmd +# time