add QCOW2 build mechanism

Dockerfile

@@ -6,7 +6,7 @@ RUN apt-get -y update && \
     apt-get -y install \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
-        bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod\
+        bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod qemu-utils kpartx\
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

README.md

@@ -14,7 +14,7 @@ To install the required dependencies for pi-gen you should run:
 
 ```bash
 apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \
-dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl
+dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl qemu-utils kpartx
 ```
 
 The file `depends` contains a list of tools needed.  The format of this
@@ -36,6 +36,37 @@ The following environment variables are supported:
    but you should use something else for a customized version.  Export files
    in stages may add suffixes to `IMG_NAME`.
 
+* `USE_QCOW2`(Default: `1` )
+
+    Instead of using traditional way of building the rootfs of every stage in
+    single subdirectories and copying over the previous one to the next one,
+    qcow2 based virtual disks with backing images are used in every stage.
+    This speeds up the build process and reduces overall space consumption
+    significantly.
+
+    <u>Additional optional parameters regarding qcow2 build:</u>
+
+    * `BASE_QCOW2_SIZE` (Default: 12G)
+
+        Size of the virtual qcow2 disk.
+        Note: it will not actually use that much of space at once but defines the
+        maximum size of the virtual disk. If you change the build process by adding
+        a lot of bigger packages or additional build stages, it can be necessary to
+        increase the value because the virtual disk can run out of space like a normal
+        hard drive would.
+
+    * `NBD_DEV` (Default: auto eval)
+
+        The virtual qcow2 disk has to be attached to a special network block device (NBD)
+        so that it can be accessed like any other disk. Normally a free device node will be
+        evaluated automatically. If you want to predefine a device node set this parameter
+        in your `config` file like so:
+        `ǸBD_DEV=/dev/nbd1`.
+
+    **CAUTION:**  Although the qcow2 build mechanism will run fine inside Docker, it can happen
+    that the network block device is not disconnected correctly after the Docker process has
+    ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong)
+
  * `APT_PROXY` (Default: unset)
 
    If you require the use of an apt proxy, set it here.  This proxy setting
@@ -324,6 +355,71 @@ follows:
  * Once you're happy with the image you can remove the SKIP_IMAGES files and
    export your image to test
 
+# Regarding Qcow2 image building
+
+### Get infos about the image in use
+
+If you issue the two commands shown in the example below in a second command shell while a build
+is running you can find out, which network block device is currently being used and which qcow2 image
+is bound to it.
+
+Example:
+
+```bash
+root@build-machine:~/$ lsblk | grep nbd
+nbd1      43:32   0    10G  0 disk 
+├─nbd1p1  43:33   0    10G  0 part 
+└─nbd1p1 253:0    0    10G  0 part
+
+root@build-machine:~/$ ps xa | grep qemu-nbd
+ 2392 pts/6    S+     0:00 grep --color=auto qemu-nbd
+31294 ?        Ssl    0:12 qemu-nbd --discard=unmap -c /dev/nbd1 image-stage4.qcow2
+```
+
+Here you can see, that the qcow2 image `image-stage4.qcow2` is currently connected to `/dev/nbd1` with
+the associated partition map `/dev/mapper/nbd1p1`. Don't worry that `lsblk` shows two entries. It is totally fine, because the device map is accessible via `/dev/mapper/nbd1p1` and also via `/dev/dm-0`. This is all part of the device mapper functionality of the kernel. See `dmsetup` for further information.
+
+### Mount a qcow2 image
+
+If you want to examine the content of a a single stage, you can simply mount the qcow2 image found in the `WORK_DIR` directory with the tool `./imagetool.sh`.
+
+See `./imagetool.sh -h` for further details on how to use it.
+
+### Disconnect an image if something went wrong
+
+It can happen, that your build stops in case of an error. Normally `./build.sh` should handle image disconnection appropriately, but in rare cases, especially during a Docker build, this may not work as expected. If that happens, starting a new build will fail and you may have to disconnect the image and/or device yourself.
+
+A typical message indicating that there are some orphaned device mapper entries is this:
+
+```
+Failed to set NBD socket 
+Disconnect client, due to: Unexpected end-of-file before all bytes were read
+```
+
+If that happens go through the following steps:
+
+1. First, check if the image is somehow mounted to a directory entry and umount it as you would any other block device, like i.e. a hard disk or USB stick.
+
+2. Second, to disconnect an image from `qemu-nbd`, the QEMU Disk Network Block Device Server, issue the following command (be sure to change the device name to the one actually used):
+
+   ```bash
+   sudo qemu-nbd -d /dev/nbd1
+   ```
+
+   Note: if you use Docker build, normally no active `qemu-nbd` process exists anymore as it will be terminated when the Docker container stops.
+
+3. To disconnect a device partition map from the network block device, execute:
+
+   ```bash
+   sudo kpartx -d /dev/nbd1
+   or
+   sudo ./imagetool.sh --cleanup
+   ```
+   
+   Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well.
+
+Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. 
+
 # Troubleshooting
 
 ## `64 Bit Systems`

build-docker.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -eu
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 BUILD_OPTS="$*"
@@ -77,6 +78,9 @@ ${DOCKER} build -t pi-gen "${DIR}"
 if [ "${CONTAINER_EXISTS}" != "" ]; then
 	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
 	time ${DOCKER} run --rm --privileged \
+		--cap-add=ALL \
+		-v /dev:/dev \
+		-v /lib/modules:/lib/modules \
 		--volume "${CONFIG_FILE}":/config:ro \
 		-e "GIT_HASH=${GIT_HASH}" \
 		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
@@ -88,6 +92,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then
 else
 	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
 	time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
+		--cap-add=ALL \
+		-v /dev:/dev \
+		-v /lib/modules:/lib/modules \
 		--volume "${CONFIG_FILE}":/config:ro \
 		-e "GIT_HASH=${GIT_HASH}" \
 		pi-gen \
@@ -96,6 +103,7 @@ else
 	rsync -av work/*/build.log deploy/" &
 	wait "$!"
 fi
+
 echo "copying results from deploy/"
 ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy .
 ls -lah deploy

build.sh

@@ -1,4 +1,7 @@
 #!/bin/bash -e
+
+#set -x
+
 # shellcheck disable=SC2119
 run_sub_stage()
 {
@@ -22,6 +25,11 @@ EOF
 				on_chroot << EOF
 apt-get install --no-install-recommends -y $PACKAGES
 EOF
+			    if [ "${USE_QCOW2}" = "1" ]; then
+				on_chroot << EOF
+apt-get clean
+EOF
+			    fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
 		fi
@@ -32,6 +40,11 @@ EOF
 				on_chroot << EOF
 apt-get install -y $PACKAGES
 EOF
+			    if [ "${USE_QCOW2}" = "1" ]; then
+				on_chroot << EOF
+apt-get clean
+EOF
+			    fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages"
 		fi
@@ -82,17 +95,27 @@ EOF
 run_stage(){
 	log "Begin ${STAGE_DIR}"
 	STAGE="$(basename "${STAGE_DIR}")"
+
 	pushd "${STAGE_DIR}" > /dev/null
-	unmount "${WORK_DIR}/${STAGE}"
+
 	STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
 	ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs
+
+	if [ "${USE_QCOW2}" = "1" ]; then 
+	    if [ ! -f SKIP ]; then
+		load_qimage
+	    fi
+	else
+	    unmount "${WORK_DIR}/${STAGE}"
+	fi
+	
 	if [ ! -f SKIP_IMAGES ]; then
 		if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
 			EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
 		fi
 	fi
 	if [ ! -f SKIP ]; then
-		if [ "${CLEAN}" = "1" ]; then
+		if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then
 			if [ -d "${ROOTFS_DIR}" ]; then
 				rm -rf "${ROOTFS_DIR}"
 			fi
@@ -109,7 +132,13 @@ run_stage(){
 			fi
 		done
 	fi
+
+	if [ "${USE_QCOW2}" = "1" ]; then 
+		unload_qimage
+	else
     	    unmount "${WORK_DIR}/${STAGE}"
+	fi
+
 	PREV_STAGE="${STAGE}"
 	PREV_STAGE_DIR="${STAGE_DIR}"
 	PREV_ROOTFS_DIR="${ROOTFS_DIR}"
@@ -141,6 +170,15 @@ do
 	esac
 done
 
+term() {
+    if [ "${USE_QCOW2}" = "1" ]; then
+	log "Unloading image"
+	unload_qimage
+    fi
+}
+
+trap term EXIT INT TERM
+
 export PI_GEN=${PI_GEN:-pi-gen}
 export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
 
@@ -208,6 +246,10 @@ source "${SCRIPT_DIR}/common"
 # shellcheck source=scripts/dependencies_check
 source "${SCRIPT_DIR}/dependencies_check"
 
+export USE_QCOW2="${USE_QCOW2:-1}"
+export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
+source "${SCRIPT_DIR}/qcow2_handling"
+
 dependencies_check "${BASE_DIR}/depends"
 
 #check username is valid
@@ -237,13 +279,29 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 	# shellcheck source=/dev/null
 	source "${EXPORT_DIR}/EXPORT_IMAGE"
 	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
+        QIMAGE="image-$(basename "${EXPORT_DIR}").qcow2"
+	if [ "${USE_QCOW2}" = "1" ]; then
+	    USE_QCOW2=0
+	    mount_qimage "${WORK_DIR}/${QIMAGE}" "${EXPORT_ROOTFS_DIR}"
+	    echo "Mounting image ${WORK_DIR}/${QIMAGE} to export rootfs ${EXPORT_ROOTFS_DIR}"
 	    run_stage
+	    unload_qimage
+	    USE_QCOW2=1
+	else
+	    run_stage
+	fi 
 	if [ "${USE_QEMU}" != "1" ]; then
 		if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
 			# shellcheck source=/dev/null
 			source "${EXPORT_DIR}/EXPORT_NOOBS"
 			STAGE_DIR="${BASE_DIR}/export-noobs"
+			if [ "${USE_QCOW2}" = "1" ]; then
+			    USE_QCOW2=0
 			    run_stage
+			    USE_QCOW2=1
+			else
+			    run_stage
+			fi
 		fi
 	fi
 done
@@ -255,4 +313,8 @@ if [ -x postrun.sh ]; then
 	log "End postrun.sh"
 fi
 
+if [ "${USE_QCOW2}" = "1" ]; then
+    unload_qimage
+fi
+
 log "End ${BASE_DIR}"

depends

@@ -16,3 +16,5 @@ xxd
 file
 git
 lsmod:kmod
+qemu-nbd:qemu-utils
+kpartx

export-image/04-finalise/01-run.sh

@@ -94,8 +94,17 @@ if [ "${DEPLOY_ZIP}" == "1" ]; then
 	zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \
 		"$(basename "${IMG_FILE}")"
 	popd > /dev/null
+else
+	if [ "${USE_QCOW2}" = "1" ]; then
+	    mv "$IMG_FILE" "$DEPLOY_DIR/"
 	else
 	    cp "$IMG_FILE" "$DEPLOY_DIR"
 	fi
+fi
 
+if [ "${USE_QCOW2}" = "1" ]; then
+    mv "$INFO_FILE" "$DEPLOY_DIR/"
+else
     cp "$INFO_FILE" "$DEPLOY_DIR"
+fi
+

export-noobs/00-release/00-run.sh

@@ -39,4 +39,8 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|NOOBS_DESCRIPTION|${NOOBS_DESCRIPTION}|"
 
 sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|"
 
+if [ "${USE_QCOW2}" = "1" ]; then
+    mv "${NOOBS_DIR}" "${DEPLOY_DIR}/"
+else
     cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/"
+fi

export-noobs/prerun.sh

@@ -1,11 +1,15 @@
 #!/bin/bash -e
 
-IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}"
-unmount_image "${IMG_FILE}"
-
 mkdir -p "${STAGE_WORK_DIR}"
+
+if [ "${USE_QCOW2}" = "1" ]; then
+    IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img"
+else
     cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/"
+    IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+fi
+unmount_image "${IMG_FILE}"
 
 rm -rf "${NOOBS_DIR}"
 

imagetool.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+if [ "$(id -u)" != "0" ]; then
+        echo "Please run as root" 1>&2
+        exit 1
+fi
+
+progname=$(basename $0)
+
+function usage()
+{
+   cat << HEREDOC
+
+Usage:
+    Mount Image : $progname [--mount] [--image-name <path to qcow2 image>] [--mount-point <mount point>]
+    Umount Image: $progname [--umount] [--mount-point <mount point>]
+    Cleanup NBD : $progname [--cleanup]
+
+   arguments:
+     -h, --help           show this help message and exit
+     -c, --cleanup        cleanup orphaned device mappings
+     -m, --mount          mount image
+     -u, --umount         umount image
+     -i, --image-name     path to qcow2 image
+     -p, --mount-point    mount point for image
+
+   This tool will use /dev/nbd1 as default for mounting an image. If you want to use another device, execute like this:
+   NBD_DEV=/dev/nbd2 ./$progname --mount --image <your image> --mount-point <your path>
+
+HEREDOC
+}
+
+MOUNT=0
+UMOUNT=0
+IMAGE=""
+MOUNTPOINT=""
+
+nbd_cleanup() {
+    DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)"
+    if [ ! -z "${DEVS}" ]; then
+        for d in $DEVS; do
+            if [ ! -z "${d}" ]; then
+                QDEV="$(ps xa | grep $d | grep -v grep)"
+                if [ -z "${QDEV}" ]; then
+                    kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d"
+                fi
+            fi
+        done
+    fi
+}
+
+# As long as there is at least one more argument, keep looping
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+	-h|--help)
+	usage
+	exit
+	;;
+        -c|--cleanup)
+        nbd_cleanup
+        ;;
+        -m|--mount)
+        MOUNT=1
+        ;;
+        -u|--umount)
+        UMOUNT=1
+        ;;
+        -i|--image-name)
+        shift
+        IMAGE="$1"
+        ;;
+        -p|--mount-point)
+        shift
+        MOUNTPOINT="$1"
+        ;;
+        *)
+        echo "Unknown option '$key'"
+	usage
+	exit
+        ;;
+    esac
+    # Shift after checking all the cases to get the next option
+    shift
+done
+
+if [ "${MOUNT}" = "1" ] && [ "${UMOUNT}" = "1" ]; then
+    usage
+    echo "Concurrent mount options not possible."
+    exit
+fi
+
+if [ "${MOUNT}" = "1" ] && ([ -z "${IMAGE}"  ] || [ -z "${MOUNTPOINT}"  ]); then
+    usage
+    echo "Can not mount image. Image path and/or mount point missing."
+    exit
+fi
+
+if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then
+    usage
+    echo "Can not umount. Mount point parameter missing."
+    exit
+fi
+
+export NBD_DEV="${NBD_DEV:-/dev/nbd1}"
+source scripts/qcow2_handling
+
+if [ "${MOUNT}" = "1" ]; then
+    mount_qimage "${MOUNTPOINT}" "${IMAGE}"
+elif [ "${UMOUNT}" = "1" ]; then
+    umount_qimage "${MOUNTPOINT}"
+fi

scripts/qcow2_handling

@@ -0,0 +1,96 @@
+# QCOW2 Routines
+
+export CURRENT_IMAGE
+export CURRENT_MOUNTPOINT
+
+# set in build.sh
+# should be fairly enough for the beginning
+# overwrite here by uncommenting following lines
+# BASE_QCOW2_SIZE=12G
+
+init_nbd() {
+    modprobe nbd max_part=16
+    if [ -z "${NBD_DEV}" ]; then
+	for x in /sys/class/block/nbd* ; do
+	    S=`cat $x/size`
+	    if [ "$S" == "0" ] ; then
+		NBD_DEV=/dev/$(basename $x)
+		MAP_DEV=/dev/mapper/$(basename $x)p1
+		break
+	    fi
+	done
+    fi
+}
+
+# mount qcow2 image: mount_image <image file> <mountpoint>
+mount_qimage() {
+    init_nbd
+    qemu-nbd --discard=unmap -c $NBD_DEV "$1"
+    kpartx -a $NBD_DEV
+    mount $MAP_DEV "$2"
+    CURRENT_IMAGE="$1"
+    CURRENT_MOUNTPOINT="$2"
+}
+export -f mount_qimage
+
+# umount qcow2 image: umount_image <current mountpoint>
+umount_qimage() {
+    while mount | grep -q "$1"; do
+            local LOCS
+            LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r)
+            for loc in $LOCS; do
+		    echo "$loc"
+                    umount "$loc"
+            done
+    done
+    qemu-nbd -d $NBD_DEV
+    kpartx -d $NBD_DEV
+}
+export -f umount_qimage
+
+# create base image / backing image / mount image
+load_qimage() {
+    if [ -z "${CURRENT_MOUNTPOINT}" ]; then
+        if [ ! -d "${ROOTFS_DIR}" ]; then mkdir -p "${ROOTFS_DIR}"; fi
+
+        if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; fi
+
+        if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then
+            pushd ${WORK_DIR} > /dev/null
+	    init_nbd
+            if [ -z "${PREV_STAGE}" ]; then
+                qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE
+                qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
+		echo 'type=83' | sfdisk $NBD_DEV
+                kpartx -a $NBD_DEV
+                mkfs.ext4 $MAP_DEV
+    	    else
+		if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then exit 1; fi
+                qemu-img create -f qcow2 \
+                    -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \
+                    ${WORK_DIR}/image-${STAGE}.qcow2
+                qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
+                kpartx -a $NBD_DEV
+            fi
+            mount $MAP_DEV "${ROOTFS_DIR}"
+	    CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2
+	    CURRENT_MOUNTPOINT=${ROOTFS_DIR}
+	    popd > /dev/null
+	else
+	    mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}"
+	fi
+	echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})"
+    fi
+}
+export -f load_qimage
+
+# umount current image and refresh mount point env var
+unload_qimage() {
+    if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then
+        fstrim -v "${CURRENT_MOUNTPOINT}" || true
+	umount_qimage "${CURRENT_MOUNTPOINT}"
+        CURRENT_IMAGE=""
+        CURRENT_MOUNTPOINT=""
+    fi
+}
+export -f unload_qimage

stage0/prerun.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e
 
-if [ ! -d "${ROOTFS_DIR}" ]; then
+if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then
 	bootstrap buster "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/
 fi