From 762bb78d89d1359c01a7ecee37e3f8132a48c1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ciro=20Santilli=20=E5=85=AD=E5=9B=9B=E4=BA=8B=E4=BB=B6=20?= =?UTF-8?q?=E6=B3=95=E8=BD=AE=E5=8A=9F?= Date: Sat, 15 Sep 2018 20:29:09 +0100 Subject: [PATCH] investigate squashfs to overcome BR2_TARGET_ROOTFS_EXT2_SIZE but fail Generate qcow2 when running QEMU if it is not present. --- README.adoc | 24 +++++++++++++++++++----- build-buildroot | 19 ++++++------------- buildroot_config/default | 5 ++++- common.py | 16 ++++++++++++++-- kernel_config/default | 7 +++++-- run | 31 +++++++++++++++---------------- 6 files changed, 63 insertions(+), 39 deletions(-) diff --git a/README.adoc b/README.adoc index fa7df50b..9a5f79aa 100644 --- a/README.adoc +++ b/README.adoc @@ -222,7 +222,7 @@ We use `./build-buildroot` because the kernel modules go inside the root filesys The reboot after rebuild is annoying. We don't have a perfect solution for it yet, but there are some ideas cooking at: <>. -Using <> can speed boot up however if your host and guest have the same arch, e.g. on an `x86_64` host: +Using <> can <> the boot however if your host and guest have the same arch, e.g. on an `x86_64` host: .... ./run --kvm @@ -7103,6 +7103,7 @@ But TODO we didn't get it working yet: * https://stackoverflow.com/questions/41119656/how-can-i-overlayfs-the-root-filesystem-on-linux * https://unix.stackexchange.com/questions/316018/how-to-use-overlayfs-to-protect-the-root-filesystem +* https://unix.stackexchange.com/questions/420646/mount-root-as-overlayfs Test with the script: @@ -7129,11 +7130,13 @@ ls /overlay/proc A less good alternative is to set `LD_LIBRARY_PATH` on the 9p mount and run executables directly from the mount. -Even mor awesome than `chroot` be to `pivot_root`, but I couldn't get that working either: +Even more awesome than `chroot` would be to `pivot_root`, but I couldn't get that working either: * https://stackoverflow.com/questions/28015688/pivot-root-device-or-resource-busy * https://unix.stackexchange.com/questions/179788/pivot-root-device-or-resource-busy +Here is a more basic working example of OverlayFS usage: https://askubuntu.com/questions/109413/how-do-i-use-overlayfs/1075564#1075564 + === Guest host networking First ensure that networking is enabled before trying out anything in this section: <> @@ -9298,11 +9301,22 @@ Unfortunately, TODO we don't have a perfect way to find the right value for `BR2 du -hsx "$(./getvar --arch arm target_dir)" .... -https://stackoverflow.com/questions/49211241/is-there-a-way-to-automatically-detect-the-minimum-required-br2-target-rootfs-ex +Some promising ways to overcome this problem include: -libguestfs is very promising link:https://serverfault.com/questions/246835/convert-directory-to-qemu-kvm-virtual-disk-image/916697#916697[], in particular link:http://libguestfs.org/guestfish.1.html#vfs-minimum-size[`vfs-minimum-size`]. +* <> +TODO benchmark: would gem5 suffer a considerable disk read performance hit due to decompressing SquashFS? +* libguestfs: link:https://serverfault.com/questions/246835/convert-directory-to-qemu-kvm-virtual-disk-image/916697#916697[], in particular link:http://libguestfs.org/guestfish.1.html#vfs-minimum-size[`vfs-minimum-size`] +* use methods described at: <> instead of putting builds on the root filesystem -One way to overcome this problem is to mount benchmarks from host instead of adding them to the root filesystem, e.g. with: <<9p>>. +Bibliography: https://stackoverflow.com/questions/49211241/is-there-a-way-to-automatically-detect-the-minimum-required-br2-target-rootfs-ex + +==== SquashFS + +link:https://en.wikipedia.org/wiki/SquashFS[SquashFS] creation with `mksquashfs` does not take fixed sizes, and I have successfully booted from it, but it is readonly, which is unacceptable. + +But then we could mount link:https://wiki.debian.org/ramfs[ramfs] on top of it with <> to make it writable, but my attempts failed exactly as mentioned at <>. + +This is the exact unanswered question: https://unix.stackexchange.com/questions/343484/mounting-squashfs-image-with-read-write-overlay-for-rootfs [[rpath]] === Buildroot rebuild is slow when the root filesystem is large diff --git a/build-buildroot b/build-buildroot index 915a378b..8f44d9d2 100755 --- a/build-buildroot +++ b/build-buildroot @@ -97,7 +97,7 @@ https://stackoverflow.com/questions/49260466/why-when-i-change-br2-linux-kernel- That target builds the root filesystem and all its dependencies.''' ) kernel_module_group.add_argument( - '--no-kernel-modules', default=defaults['kernel_modules'], action='store_true', + '--no-kernel-modules', default=defaults['no_kernel_modules'], action='store_true', help="Don't build the kernel modules package" ) parser.add_argument( @@ -293,18 +293,11 @@ def main(args, extra_args=None): cwd=common.buildroot_src_dir, ) == 0 - # Create the qcow2 from ext2. This is optional, because gem5 - # does not need the qcow2. - if os.path.exists(common.qemu_img_executable) and os.path.exists(common.ext2_file): - assert common.run_cmd([ - common.qemu_img_executable, - '-T', 'pr_manager_run,file=/dev/null', - 'convert', - '-f', 'raw', - '-O', 'qcow2', - common.ext2_file, - common.qcow2_file, - ]) == 0 + # Create the qcow2 from ext2. + # Skip if qemu is not present, because gem5 does not need the qcow2. + # so we don't force a QEMU build for gem5. + if not args.no_all and os.path.exists(common.qemu_img_executable): + common.raw_to_qcow2() return 0 diff --git a/buildroot_config/default b/buildroot_config/default index 795712b0..4764505a 100644 --- a/buildroot_config/default +++ b/buildroot_config/default @@ -16,8 +16,11 @@ BR2_TOOLCHAIN_BUILDROOT_WCHAR=y # Rootfs BR2_TARGET_ROOTFS_CPIO=n BR2_TARGET_ROOTFS_EXT2=y -BR2_TARGET_ROOTFS_INITRAMFS=n BR2_TARGET_ROOTFS_EXT2_SIZE="512M" +BR2_TARGET_ROOTFS_SQUASHFS=n +BR2_TARGET_ROOTFS_INITRAMFS=n +# TODO can you boot with those as root filesystem? +BR2_TARGET_ROOTFS_TAR=n # Host GDB BR2_GDB_VERSION="7.11.1" diff --git a/common.py b/common.py index eecf2a71..8f73d288 100644 --- a/common.py +++ b/common.py @@ -219,6 +219,18 @@ def print_time(ellapsed_seconds): minutes, seconds = divmod(rem, 60) print("time {:02}:{:02}:{:02}".format(int(hours), int(minutes), int(seconds))) +def raw_to_qcow2(): + global this + assert this.run_cmd([ + this.qemu_img_executable, + '-T', 'pr_manager_run,file=/dev/null', + 'convert', + '-f', 'raw', + '-O', 'qcow2', + this.rootfs_raw_file, + this.qcow2_file, + ]) == 0 + def resolve_args(defaults, args, extra_args): if extra_args is None: extra_args = {} @@ -331,8 +343,8 @@ def setup(parser, **extra_args): this.host_dir = os.path.join(this.buildroot_build_dir, 'host') this.host_bin_dir = os.path.join(this.host_dir, 'usr', 'bin') this.images_dir = os.path.join(this.buildroot_build_dir, 'images') - this.ext2_file = os.path.join(this.images_dir, 'rootfs.ext2') - this.qcow2_file = os.path.join(this.images_dir, 'rootfs.ext2.qcow2') + this.rootfs_raw_file = os.path.join(this.images_dir, 'rootfs.squashfs') + this.qcow2_file = this.rootfs_raw_file + '.qcow2' this.staging_dir = os.path.join(this.buildroot_build_dir, 'staging') this.target_dir = os.path.join(this.buildroot_build_dir, 'target') this.run_dir_base = os.path.join(this.out_dir, 'run') diff --git a/kernel_config/default b/kernel_config/default index 0486eb91..bd1dd04c 100644 --- a/kernel_config/default +++ b/kernel_config/default @@ -1,11 +1,14 @@ # Misc. CONFIG_BLK_DEV_INITRD=y -CONFIG_DEBUG_FS=y CONFIG_DYNAMIC_DEBUG=y CONFIG_MODULE_SRCVERSION_ALL=y -CONFIG_OVERLAY_FS=y CONFIG_STRICT_DEVMEM=n +# Filesystems. +CONFIG_DEBUG_FS=y +CONFIG_OVERLAY_FS=y +CONFIG_SQUASHFS=y + # GDB debugging. CONFIG_DEBUG_INFO=y CONFIG_DEBUG_KERNEL=y diff --git a/run b/run index 6d5e6eff..8cc97036 100755 --- a/run +++ b/run @@ -125,7 +125,7 @@ def main(args, extra_args=None): os.path.join(common.gem5_src_dir, 'configs', 'example', 'arm', 'fs_bigLITTLE.py'), '--big-cpus', '2', '--cpu-type', 'atomic', - '--disk', common.ext2_file, + '--disk', common.rootfs_raw_file, '--dtb', os.path.join(common.gem5_system_dir, 'arm', 'dt', 'armv8_gem5_v1_big_little_2_2.dtb'), '--kernel', common.vmlinux, '--little-cpus', '2' @@ -138,7 +138,7 @@ def main(args, extra_args=None): extra_emulator_args.extend(['-r', str(sorted(cpt_dirs).index(cpt_dir) + 1)]) cmd += [ common.gem5_fs_file, - '--disk-image', common.ext2_file, + '--disk-image', common.rootfs_raw_file, '--kernel', common.vmlinux, '--mem-size', memory, '--num-cpus', str(args.cpus), @@ -159,6 +159,19 @@ def main(args, extra_args=None): else: extra_emulator_args.extend(extra_qemu_args) os.makedirs(common.run_dir, exist_ok=True) + if args.prebuilt: + common.mkdir() + qemu_executable = "qemu-system-{}".format(args.arch) + else: + qemu_executable = common.qemu_executable + if not os.path.exists(qemu_executable): + raise Exception('QEMU executable does not exist, did you forget to build or install it?\n' \ + 'Tried to use: ' + qemu_executable) + if not os.path.exists(common.qcow2_file): + if not os.path.exists(common.rootfs_raw_file): + raise Exception('Root filesystem not found. Did you build it?\n' \ + 'Tried to use: ' + common.rootfs_raw_file) + common.raw_to_qcow2() if args.debug_vm: serial_monitor = [] else: @@ -167,11 +180,6 @@ def main(args, extra_args=None): extra_emulator_args.append('-enable-kvm') if args.kgdb: extra_emulator_args.extend(['-serial', 'tcp::{},server,nowait'.format(common.gdb_port)]) - if args.prebuilt: - common.mkdir() - qemu_executable = "qemu-system-{}".format(args.arch) - else: - qemu_executable = common.qemu_executable cmd = ( debug_vm + [ @@ -212,12 +220,6 @@ def main(args, extra_args=None): '-drive', 'file={},format=qcow2,if={}{}{}'.format(common.qcow2_file, driveif, snapshot, rrid) ]) - if not os.path.exists(common.qcow2_file): - raise Exception( - 'Cannot find the qcow2 root filesystem. You must build QEMU\n' - 'before building the root filesystem? That is needed because the qcow2\n' + - 'is created with qemu-img. Tried to use: ' + qemu_executable - ) if rr: extra_emulator_args.extend([ '-drive', 'driver=blkreplay,if=none,image=img-direct,id=img-blkreplay', @@ -257,9 +259,6 @@ def main(args, extra_args=None): ] + virtio_gpu_pci ) - if not os.path.exists(qemu_executable): - raise Exception('QEMU executable does not exist, did you forget to build or install it?\n' + - 'Tried to use: ' + qemu_executable) if args.tmux: if args.gem5: subprocess.Popen([os.path.join(common.root_dir, 'tmu'),