Ubuntu 16.04: Debug Ubuntu 16.04 kernel with QEMU GDB-stub

Download kernel debug symbol from repository for debug symbol.

$ sudo apt install -y linux-image-$(uname -r)-dbgsym
$ file /usr/lib/debug/boot/vmlinux-4.4.0-31-generic
/usr/lib/debug/boot/vmlinux-4.4.0-31-generic: ELF 64-bit LSB
executable, x86-64, version 1 (SYSV), statically linked,
BuildID[sha1]=c66bb1efd9220f7386509bee83bcc80a1aa21744, not stripped

Download kernel source code from repository for source code.

$ mkdir ~/linux
$ cd ~/linux
$ apt source linux
$ ls
linux-4.4.0  linux_4.4.0-31.50.diff.gz  linux_4.4.0-31.50.dsc  linux_4.4.0.orig.tar.gz
$ cd ..

Move kernel to path referenced by debug symbol.

$ sudo mkdir -p /build/linux-dcxD3m/
$ sudo mv linux/linux-4.4.0 /build/linux-dcxD3m/

Create root filesystem with livecd-rootfs. Following script takes about 2 hours for creating root filesystem which has xubuntu-desktop package.

#!/bin/sh -ex

PASSWORD="password"
PACKAGE="xubuntu-desktop linux-modules-$(uname -r)"
SIZE=4096 # rootfs size: 1MByte x 4096 = 4GByte
hostname=`hostname`
TIMEZONE="Asia/Tokyo" # Please change your timezone.

# Intall packages for creating rootfs.
sudo apt install -y livecd-rootfs systemd-container xorriso

# Create work directory
mkdir rootfs
cd rootfs

# Prepare live-build
cp -a /usr/share/livecd-rootfs/live-build/auto .
cp -a /usr/share/livecd-rootfs/live-build/ubuntu-core .
PROJECT=ubuntu-core lb config
sed -i 's/precise/xenial/g' config/bootstrap

# Create filesystem
dd if=/dev/zero of=rootfs.ext4 bs=1M count=${SIZE}
mkfs.ext4 rootfs.ext4
mkdir chroot
sudo mount -t ext4 -o loop,rw rootfs.ext4 chroot

# Run live-build and create minimul rootfs
sudo PROJECT=ubuntu-core lb build

# Additional rootfs configuration
sudo cp /etc/apt/sources.list chroot/etc/apt/
sudo cp /etc/default/keyboard chroot/etc/default/
sudo su -c "echo ${hostname} > chroot/etc/hostname"
yes "${PASSWORD}" | sudo systemd-nspawn -D chroot passwd

# Install expect.
sudo apt install -y expect

# Run command on systemd_nspawn via expect.
systemd_nspawn_via_expect()
{
    expect -c "
set timeout -1
spawn sudo systemd-nspawn -b -D chroot
expect \"${hostname} login:\"
send -- \"root\n\"
expect \"Password:\"
send -- \"${PASSWORD}\n\"
expect \"root@${hostname}:~#\"
send -- \"${1} || (apt install -f && ${1})\n\"
expect \"root@${hostname}:~#\"
send -- \"poweroff\n\"
expect \"${PS1}\"
"
    # BUG: The systemd-nspawn by root user cannot be poweroffed.
    sleep 10
    while machinectl status --no-pager chroot > /dev/null 2>&1; do
        echo 3 | sudo tee /proc/sys/vm/drop_caches
        sync
        sudo machinectl terminate chroot
        sleep 10
    done
}

# Update apt database.
systemd_nspawn_via_expect "apt update -y"
systemd_nspawn_via_expect "apt install -y apt-utils"

# Install console-setup which needs interactive setting.
expect -c "
set timeout -1
spawn sudo systemd-nspawn -D chroot apt install -y console-setup
expect \"Encoding to use on the console:\"
send -- \"27\n\"
expect \"Character set to support:\"
send -- \"20\n\"
expect \"${PS1}\"
"

# Install tzdata which needs interactive setting.
systemd_nspawn_via_expect "ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime"
systemd_nspawn_via_expect "echo ${TIMEZONE} > /etc/timezone"
systemd_nspawn_via_expect "DEBIAN_FRONTEND=noninteractive apt install -y tzdata"
systemd_nspawn_via_expect "dpkg-reconfigure --frontend noninteractive tzdata"

# Install packages.
[ -n "${PACKAGE}" ] && systemd_nspawn_via_expect "apt install -y ${PACKAGE}"

# Unmount filesystem
sudo umount chroot

Install QEMU and GDB.

$ sudo apt install -y qemu gdb64

Only root can read Ubuntu kernel. Copy kernel to working directory so that QEMU can access.

$ sudo cp /boot/vmlinuz-4.4.0-31-generic .
$ sudo chown hiroom2:hiroom2 vmlinuz-4.4.0-31-generic

Create xfs formatted file.

$ dd if=/dev/zero of=xfs.img bs=1M count=4K
$ sudo apt install -y xfsprogs
$ mkfs.xfs xfs.img

Create GDB script file.

$ cat <<EOF > gdb.x
set architecture i386:x86-64
target remote localhost:10000
symbol-file /usr/lib/debug/boot/vmlinux-4.4.0-31-generic
c
EOF

Run QEMU. When debugging 64bit kernel, GDB-stub cannot debug initialize code before start_kernel (This may be because GDB-stub does not support that the instruction will be switch from 32bit to 64bit when kernel initialization). So -S option cannot be used.

$ qemu-system-x86_64 \
  --kernel vmlinuz-4.4.0-31-generic \
  --initrd /boot/initrd.img-4.4.0-31-generic \
  --hda rootfs/rootfs.ext4 \
  --hdb xfs.img \
  --append "console=tty1,115200 console=ttyS0,115200 root=/dev/sda rw" \
  --nographic \
  -m 512M \
  --gdb tcp::10000

When start_kernel is running (The string by printk is displayed to console), run GDB.

$ gdb64 -x gdb.x

Load xfs module on Ubuntu 16.04 on QEMU. Get load address by reading /proc/modules.

# modprobe xfs
# grep "^xfs" /proc/modules
xfs 970752 0 - Live 0xffffffffc05c4000

Add xfs module symbol file on GDB. Add breakpoint at xfs_fs_mount which will be called by mount command.

(gdb) add-symbol-file /usr/lib/debug/lib/modules/4.4.0-31-generic/kernel/fs/xfs/xfs.ko 0xffffffffc05c4000
(gdb) b xfs_fs_mount
(gdb) c

Run mount command on Ubuntu 16.04 on QEMU.

# mount -t xfs /dev/sdb /mnt

GDB break at xfs_fs_mount.

Breakpoint 1, xfs_fs_mount (fs_type=0xffffffffc0673980, flags=0,
    dev_name=0xffff88001a207010 "/dev/sdb", data=0x0 <irq_stack_union>)
    at /build/linux-dcxD3m/linux-4.4.0/fs/xfs/xfs_super.c:1611
warning: Source file is more recent than executable.
1611    {
(gdb) l
1606    xfs_fs_mount(
1607            struct file_system_type *fs_type,
1608            int                     flags,
1609            const char              *dev_name,
1610            void                    *data)
1611    {
1612            return mount_bdev(fs_type, flags, dev_name, data, xfs_fs_fill_super);
1613    }
1614
1615    static long
(gdb)