From 78904cbf2e55dd008273cf75ee5671edba7b1050 Mon Sep 17 00:00:00 2001 From: Holger Pandel Date: Mon, 18 Nov 2019 21:50:48 +0100 Subject: [PATCH] add QCOW2 build mechanism --- Dockerfile | 2 +- README.md | 98 ++++++++++++++++++++++++- build-docker.sh | 8 +++ build.sh | 72 +++++++++++++++++-- depends | 2 + export-image/04-finalise/01-run.sh | 13 +++- export-noobs/00-release/00-run.sh | 6 +- export-noobs/prerun.sh | 12 ++-- imagetool.sh | 112 +++++++++++++++++++++++++++++ scripts/qcow2_handling | 96 +++++++++++++++++++++++++ stage0/prerun.sh | 2 +- 11 files changed, 408 insertions(+), 15 deletions(-) create mode 100755 imagetool.sh create mode 100644 scripts/qcow2_handling diff --git a/Dockerfile b/Dockerfile index 706a5fb..967b618 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get -y update && \ apt-get -y install \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ - bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod\ + bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod qemu-utils kpartx\ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/README.md b/README.md index e131ca4..58c0636 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ To install the required dependencies for pi-gen you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ -dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl +dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl qemu-utils kpartx ``` The file `depends` contains a list of tools needed. The format of this @@ -36,6 +36,37 @@ The following environment variables are supported: but you should use something else for a customized version. Export files in stages may add suffixes to `IMG_NAME`. +* `USE_QCOW2`(Default: `1` ) + + Instead of using traditional way of building the rootfs of every stage in + single subdirectories and copying over the previous one to the next one, + qcow2 based virtual disks with backing images are used in every stage. + This speeds up the build process and reduces overall space consumption + significantly. + + Additional optional parameters regarding qcow2 build: + + * `BASE_QCOW2_SIZE` (Default: 12G) + + Size of the virtual qcow2 disk. + Note: it will not actually use that much of space at once but defines the + maximum size of the virtual disk. If you change the build process by adding + a lot of bigger packages or additional build stages, it can be necessary to + increase the value because the virtual disk can run out of space like a normal + hard drive would. + + * `NBD_DEV` (Default: auto eval) + + The virtual qcow2 disk has to be attached to a special network block device (NBD) + so that it can be accessed like any other disk. Normally a free device node will be + evaluated automatically. If you want to predefine a device node set this parameter + in your `config` file like so: + `ǸBD_DEV=/dev/nbd1`. + + **CAUTION:** Although the qcow2 build mechanism will run fine inside Docker, it can happen + that the network block device is not disconnected correctly after the Docker process has + ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong) + * `APT_PROXY` (Default: unset) If you require the use of an apt proxy, set it here. This proxy setting @@ -324,6 +355,71 @@ follows: * Once you're happy with the image you can remove the SKIP_IMAGES files and export your image to test +# Regarding Qcow2 image building + +### Get infos about the image in use + +If you issue the two commands shown in the example below in a second command shell while a build +is running you can find out, which network block device is currently being used and which qcow2 image +is bound to it. + +Example: + +```bash +root@build-machine:~/$ lsblk | grep nbd +nbd1 43:32 0 10G 0 disk +├─nbd1p1 43:33 0 10G 0 part +└─nbd1p1 253:0 0 10G 0 part + +root@build-machine:~/$ ps xa | grep qemu-nbd + 2392 pts/6 S+ 0:00 grep --color=auto qemu-nbd +31294 ? Ssl 0:12 qemu-nbd --discard=unmap -c /dev/nbd1 image-stage4.qcow2 +``` + +Here you can see, that the qcow2 image `image-stage4.qcow2` is currently connected to `/dev/nbd1` with +the associated partition map `/dev/mapper/nbd1p1`. Don't worry that `lsblk` shows two entries. It is totally fine, because the device map is accessible via `/dev/mapper/nbd1p1` and also via `/dev/dm-0`. This is all part of the device mapper functionality of the kernel. See `dmsetup` for further information. + +### Mount a qcow2 image + +If you want to examine the content of a a single stage, you can simply mount the qcow2 image found in the `WORK_DIR` directory with the tool `./imagetool.sh`. + +See `./imagetool.sh -h` for further details on how to use it. + +### Disconnect an image if something went wrong + +It can happen, that your build stops in case of an error. Normally `./build.sh` should handle image disconnection appropriately, but in rare cases, especially during a Docker build, this may not work as expected. If that happens, starting a new build will fail and you may have to disconnect the image and/or device yourself. + +A typical message indicating that there are some orphaned device mapper entries is this: + +``` +Failed to set NBD socket +Disconnect client, due to: Unexpected end-of-file before all bytes were read +``` + +If that happens go through the following steps: + +1. First, check if the image is somehow mounted to a directory entry and umount it as you would any other block device, like i.e. a hard disk or USB stick. + +2. Second, to disconnect an image from `qemu-nbd`, the QEMU Disk Network Block Device Server, issue the following command (be sure to change the device name to the one actually used): + + ```bash + sudo qemu-nbd -d /dev/nbd1 + ``` + + Note: if you use Docker build, normally no active `qemu-nbd` process exists anymore as it will be terminated when the Docker container stops. + +3. To disconnect a device partition map from the network block device, execute: + + ```bash + sudo kpartx -d /dev/nbd1 + or + sudo ./imagetool.sh --cleanup + ``` + + Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well. + +Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. + # Troubleshooting ## `64 Bit Systems` diff --git a/build-docker.sh b/build-docker.sh index 79b68d4..50674ce 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,4 +1,5 @@ #!/bin/bash -eu + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" BUILD_OPTS="$*" @@ -77,6 +78,9 @@ ${DOCKER} build -t pi-gen "${DIR}" if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ @@ -88,6 +92,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then else trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ @@ -96,6 +103,7 @@ else rsync -av work/*/build.log deploy/" & wait "$!" fi + echo "copying results from deploy/" ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy . ls -lah deploy diff --git a/build.sh b/build.sh index a730e44..fce52d2 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,7 @@ #!/bin/bash -e + +#set -x + # shellcheck disable=SC2119 run_sub_stage() { @@ -22,6 +25,11 @@ EOF on_chroot << EOF apt-get install --no-install-recommends -y $PACKAGES EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages-nr" fi @@ -32,6 +40,11 @@ EOF on_chroot << EOF apt-get install -y $PACKAGES EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages" fi @@ -82,17 +95,27 @@ EOF run_stage(){ log "Begin ${STAGE_DIR}" STAGE="$(basename "${STAGE_DIR}")" + pushd "${STAGE_DIR}" > /dev/null - unmount "${WORK_DIR}/${STAGE}" + STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs + + if [ "${USE_QCOW2}" = "1" ]; then + if [ ! -f SKIP ]; then + load_qimage + fi + else + unmount "${WORK_DIR}/${STAGE}" + fi + if [ ! -f SKIP_IMAGES ]; then if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" fi fi if [ ! -f SKIP ]; then - if [ "${CLEAN}" = "1" ]; then + if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then if [ -d "${ROOTFS_DIR}" ]; then rm -rf "${ROOTFS_DIR}" fi @@ -109,7 +132,13 @@ run_stage(){ fi done fi - unmount "${WORK_DIR}/${STAGE}" + + if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage + else + unmount "${WORK_DIR}/${STAGE}" + fi + PREV_STAGE="${STAGE}" PREV_STAGE_DIR="${STAGE_DIR}" PREV_ROOTFS_DIR="${ROOTFS_DIR}" @@ -141,6 +170,15 @@ do esac done +term() { + if [ "${USE_QCOW2}" = "1" ]; then + log "Unloading image" + unload_qimage + fi +} + +trap term EXIT INT TERM + export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} @@ -208,6 +246,10 @@ source "${SCRIPT_DIR}/common" # shellcheck source=scripts/dependencies_check source "${SCRIPT_DIR}/dependencies_check" +export USE_QCOW2="${USE_QCOW2:-1}" +export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} +source "${SCRIPT_DIR}/qcow2_handling" + dependencies_check "${BASE_DIR}/depends" #check username is valid @@ -237,13 +279,29 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - run_stage + QIMAGE="image-$(basename "${EXPORT_DIR}").qcow2" + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + mount_qimage "${WORK_DIR}/${QIMAGE}" "${EXPORT_ROOTFS_DIR}" + echo "Mounting image ${WORK_DIR}/${QIMAGE} to export rootfs ${EXPORT_ROOTFS_DIR}" + run_stage + unload_qimage + USE_QCOW2=1 + else + run_stage + fi if [ "${USE_QEMU}" != "1" ]; then if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_NOOBS" STAGE_DIR="${BASE_DIR}/export-noobs" - run_stage + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + run_stage + USE_QCOW2=1 + else + run_stage + fi fi fi done @@ -255,4 +313,8 @@ if [ -x postrun.sh ]; then log "End postrun.sh" fi +if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage +fi + log "End ${BASE_DIR}" diff --git a/depends b/depends index 6238eb1..25b4701 100644 --- a/depends +++ b/depends @@ -16,3 +16,5 @@ xxd file git lsmod:kmod +qemu-nbd:qemu-utils +kpartx diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index 0864639..0b1fdc9 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -95,7 +95,16 @@ if [ "${DEPLOY_ZIP}" == "1" ]; then "$(basename "${IMG_FILE}")" popd > /dev/null else - cp "$IMG_FILE" "$DEPLOY_DIR" + if [ "${USE_QCOW2}" = "1" ]; then + mv "$IMG_FILE" "$DEPLOY_DIR/" + else + cp "$IMG_FILE" "$DEPLOY_DIR" + fi +fi + +if [ "${USE_QCOW2}" = "1" ]; then + mv "$INFO_FILE" "$DEPLOY_DIR/" +else + cp "$INFO_FILE" "$DEPLOY_DIR" fi -cp "$INFO_FILE" "$DEPLOY_DIR" diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index 1d0b12f..e313505 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -39,4 +39,8 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|NOOBS_DESCRIPTION|${NOOBS_DESCRIPTION}|" sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|" -cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" +if [ "${USE_QCOW2}" = "1" ]; then + mv "${NOOBS_DIR}" "${DEPLOY_DIR}/" +else + cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" +fi diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 54e0c59..ae88deb 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -1,11 +1,15 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}" -unmount_image "${IMG_FILE}" - mkdir -p "${STAGE_WORK_DIR}" -cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/" + +if [ "${USE_QCOW2}" = "1" ]; then + IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" +else + cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/" + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +fi +unmount_image "${IMG_FILE}" rm -rf "${NOOBS_DIR}" diff --git a/imagetool.sh b/imagetool.sh new file mode 100755 index 0000000..b0ebb00 --- /dev/null +++ b/imagetool.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +if [ "$(id -u)" != "0" ]; then + echo "Please run as root" 1>&2 + exit 1 +fi + +progname=$(basename $0) + +function usage() +{ + cat << HEREDOC + +Usage: + Mount Image : $progname [--mount] [--image-name ] [--mount-point ] + Umount Image: $progname [--umount] [--mount-point ] + Cleanup NBD : $progname [--cleanup] + + arguments: + -h, --help show this help message and exit + -c, --cleanup cleanup orphaned device mappings + -m, --mount mount image + -u, --umount umount image + -i, --image-name path to qcow2 image + -p, --mount-point mount point for image + + This tool will use /dev/nbd1 as default for mounting an image. If you want to use another device, execute like this: + NBD_DEV=/dev/nbd2 ./$progname --mount --image --mount-point + +HEREDOC +} + +MOUNT=0 +UMOUNT=0 +IMAGE="" +MOUNTPOINT="" + +nbd_cleanup() { + DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)" + if [ ! -z "${DEVS}" ]; then + for d in $DEVS; do + if [ ! -z "${d}" ]; then + QDEV="$(ps xa | grep $d | grep -v grep)" + if [ -z "${QDEV}" ]; then + kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d" + fi + fi + done + fi +} + +# As long as there is at least one more argument, keep looping +while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + -h|--help) + usage + exit + ;; + -c|--cleanup) + nbd_cleanup + ;; + -m|--mount) + MOUNT=1 + ;; + -u|--umount) + UMOUNT=1 + ;; + -i|--image-name) + shift + IMAGE="$1" + ;; + -p|--mount-point) + shift + MOUNTPOINT="$1" + ;; + *) + echo "Unknown option '$key'" + usage + exit + ;; + esac + # Shift after checking all the cases to get the next option + shift +done + +if [ "${MOUNT}" = "1" ] && [ "${UMOUNT}" = "1" ]; then + usage + echo "Concurrent mount options not possible." + exit +fi + +if [ "${MOUNT}" = "1" ] && ([ -z "${IMAGE}" ] || [ -z "${MOUNTPOINT}" ]); then + usage + echo "Can not mount image. Image path and/or mount point missing." + exit +fi + +if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then + usage + echo "Can not umount. Mount point parameter missing." + exit +fi + +export NBD_DEV="${NBD_DEV:-/dev/nbd1}" +source scripts/qcow2_handling + +if [ "${MOUNT}" = "1" ]; then + mount_qimage "${MOUNTPOINT}" "${IMAGE}" +elif [ "${UMOUNT}" = "1" ]; then + umount_qimage "${MOUNTPOINT}" +fi diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling new file mode 100644 index 0000000..0d8e4e7 --- /dev/null +++ b/scripts/qcow2_handling @@ -0,0 +1,96 @@ +# QCOW2 Routines + +export CURRENT_IMAGE +export CURRENT_MOUNTPOINT + +# set in build.sh +# should be fairly enough for the beginning +# overwrite here by uncommenting following lines +# BASE_QCOW2_SIZE=12G + +init_nbd() { + modprobe nbd max_part=16 + if [ -z "${NBD_DEV}" ]; then + for x in /sys/class/block/nbd* ; do + S=`cat $x/size` + if [ "$S" == "0" ] ; then + NBD_DEV=/dev/$(basename $x) + MAP_DEV=/dev/mapper/$(basename $x)p1 + break + fi + done + fi +} + +# mount qcow2 image: mount_image +mount_qimage() { + init_nbd + qemu-nbd --discard=unmap -c $NBD_DEV "$1" + kpartx -a $NBD_DEV + mount $MAP_DEV "$2" + CURRENT_IMAGE="$1" + CURRENT_MOUNTPOINT="$2" +} +export -f mount_qimage + +# umount qcow2 image: umount_image +umount_qimage() { + while mount | grep -q "$1"; do + local LOCS + LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) + for loc in $LOCS; do + echo "$loc" + umount "$loc" + done + done + qemu-nbd -d $NBD_DEV + kpartx -d $NBD_DEV +} +export -f umount_qimage + +# create base image / backing image / mount image +load_qimage() { + if [ -z "${CURRENT_MOUNTPOINT}" ]; then + if [ ! -d "${ROOTFS_DIR}" ]; then mkdir -p "${ROOTFS_DIR}"; fi + + if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; fi + + if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + pushd ${WORK_DIR} > /dev/null + init_nbd + if [ -z "${PREV_STAGE}" ]; then + qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + echo 'type=83' | sfdisk $NBD_DEV + kpartx -a $NBD_DEV + mkfs.ext4 $MAP_DEV + else + if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then exit 1; fi + qemu-img create -f qcow2 \ + -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ + ${WORK_DIR}/image-${STAGE}.qcow2 + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + kpartx -a $NBD_DEV + fi + mount $MAP_DEV "${ROOTFS_DIR}" + CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 + CURRENT_MOUNTPOINT=${ROOTFS_DIR} + popd > /dev/null + else + mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" + fi + echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" + fi +} +export -f load_qimage + +# umount current image and refresh mount point env var +unload_qimage() { + if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then + fstrim -v "${CURRENT_MOUNTPOINT}" || true + umount_qimage "${CURRENT_MOUNTPOINT}" + CURRENT_IMAGE="" + CURRENT_MOUNTPOINT="" + fi +} +export -f unload_qimage diff --git a/stage0/prerun.sh b/stage0/prerun.sh index 9ce3e02..b727cc7 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -if [ ! -d "${ROOTFS_DIR}" ]; then +if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then bootstrap buster "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/ fi