diff --git a/Dockerfile b/Dockerfile
index 706a5fb..967b618 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,7 +6,7 @@ RUN apt-get -y update && \
apt-get -y install \
git vim parted \
quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
- bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod\
+ bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod qemu-utils kpartx\
&& rm -rf /var/lib/apt/lists/*
COPY . /pi-gen/
diff --git a/README.md b/README.md
index e131ca4..58c0636 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ To install the required dependencies for pi-gen you should run:
```bash
apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \
-dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl
+dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl qemu-utils kpartx
```
The file `depends` contains a list of tools needed. The format of this
@@ -36,6 +36,37 @@ The following environment variables are supported:
but you should use something else for a customized version. Export files
in stages may add suffixes to `IMG_NAME`.
+* `USE_QCOW2`(Default: `1` )
+
+ Instead of using traditional way of building the rootfs of every stage in
+ single subdirectories and copying over the previous one to the next one,
+ qcow2 based virtual disks with backing images are used in every stage.
+ This speeds up the build process and reduces overall space consumption
+ significantly.
+
+ Additional optional parameters regarding qcow2 build:
+
+ * `BASE_QCOW2_SIZE` (Default: 12G)
+
+ Size of the virtual qcow2 disk.
+ Note: it will not actually use that much of space at once but defines the
+ maximum size of the virtual disk. If you change the build process by adding
+ a lot of bigger packages or additional build stages, it can be necessary to
+ increase the value because the virtual disk can run out of space like a normal
+ hard drive would.
+
+ * `NBD_DEV` (Default: auto eval)
+
+ The virtual qcow2 disk has to be attached to a special network block device (NBD)
+ so that it can be accessed like any other disk. Normally a free device node will be
+ evaluated automatically. If you want to predefine a device node set this parameter
+ in your `config` file like so:
+ `ǸBD_DEV=/dev/nbd1`.
+
+ **CAUTION:** Although the qcow2 build mechanism will run fine inside Docker, it can happen
+ that the network block device is not disconnected correctly after the Docker process has
+ ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong)
+
* `APT_PROXY` (Default: unset)
If you require the use of an apt proxy, set it here. This proxy setting
@@ -324,6 +355,71 @@ follows:
* Once you're happy with the image you can remove the SKIP_IMAGES files and
export your image to test
+# Regarding Qcow2 image building
+
+### Get infos about the image in use
+
+If you issue the two commands shown in the example below in a second command shell while a build
+is running you can find out, which network block device is currently being used and which qcow2 image
+is bound to it.
+
+Example:
+
+```bash
+root@build-machine:~/$ lsblk | grep nbd
+nbd1 43:32 0 10G 0 disk
+├─nbd1p1 43:33 0 10G 0 part
+└─nbd1p1 253:0 0 10G 0 part
+
+root@build-machine:~/$ ps xa | grep qemu-nbd
+ 2392 pts/6 S+ 0:00 grep --color=auto qemu-nbd
+31294 ? Ssl 0:12 qemu-nbd --discard=unmap -c /dev/nbd1 image-stage4.qcow2
+```
+
+Here you can see, that the qcow2 image `image-stage4.qcow2` is currently connected to `/dev/nbd1` with
+the associated partition map `/dev/mapper/nbd1p1`. Don't worry that `lsblk` shows two entries. It is totally fine, because the device map is accessible via `/dev/mapper/nbd1p1` and also via `/dev/dm-0`. This is all part of the device mapper functionality of the kernel. See `dmsetup` for further information.
+
+### Mount a qcow2 image
+
+If you want to examine the content of a a single stage, you can simply mount the qcow2 image found in the `WORK_DIR` directory with the tool `./imagetool.sh`.
+
+See `./imagetool.sh -h` for further details on how to use it.
+
+### Disconnect an image if something went wrong
+
+It can happen, that your build stops in case of an error. Normally `./build.sh` should handle image disconnection appropriately, but in rare cases, especially during a Docker build, this may not work as expected. If that happens, starting a new build will fail and you may have to disconnect the image and/or device yourself.
+
+A typical message indicating that there are some orphaned device mapper entries is this:
+
+```
+Failed to set NBD socket
+Disconnect client, due to: Unexpected end-of-file before all bytes were read
+```
+
+If that happens go through the following steps:
+
+1. First, check if the image is somehow mounted to a directory entry and umount it as you would any other block device, like i.e. a hard disk or USB stick.
+
+2. Second, to disconnect an image from `qemu-nbd`, the QEMU Disk Network Block Device Server, issue the following command (be sure to change the device name to the one actually used):
+
+ ```bash
+ sudo qemu-nbd -d /dev/nbd1
+ ```
+
+ Note: if you use Docker build, normally no active `qemu-nbd` process exists anymore as it will be terminated when the Docker container stops.
+
+3. To disconnect a device partition map from the network block device, execute:
+
+ ```bash
+ sudo kpartx -d /dev/nbd1
+ or
+ sudo ./imagetool.sh --cleanup
+ ```
+
+ Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well.
+
+Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again.
+
# Troubleshooting
## `64 Bit Systems`
diff --git a/build-docker.sh b/build-docker.sh
index 79b68d4..50674ce 100755
--- a/build-docker.sh
+++ b/build-docker.sh
@@ -1,4 +1,5 @@
#!/bin/bash -eu
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
BUILD_OPTS="$*"
@@ -77,6 +78,9 @@ ${DOCKER} build -t pi-gen "${DIR}"
if [ "${CONTAINER_EXISTS}" != "" ]; then
trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
time ${DOCKER} run --rm --privileged \
+ --cap-add=ALL \
+ -v /dev:/dev \
+ -v /lib/modules:/lib/modules \
--volume "${CONFIG_FILE}":/config:ro \
-e "GIT_HASH=${GIT_HASH}" \
--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
@@ -88,6 +92,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then
else
trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
+ --cap-add=ALL \
+ -v /dev:/dev \
+ -v /lib/modules:/lib/modules \
--volume "${CONFIG_FILE}":/config:ro \
-e "GIT_HASH=${GIT_HASH}" \
pi-gen \
@@ -96,6 +103,7 @@ else
rsync -av work/*/build.log deploy/" &
wait "$!"
fi
+
echo "copying results from deploy/"
${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy .
ls -lah deploy
diff --git a/build.sh b/build.sh
index a730e44..fce52d2 100755
--- a/build.sh
+++ b/build.sh
@@ -1,4 +1,7 @@
#!/bin/bash -e
+
+#set -x
+
# shellcheck disable=SC2119
run_sub_stage()
{
@@ -22,6 +25,11 @@ EOF
on_chroot << EOF
apt-get install --no-install-recommends -y $PACKAGES
EOF
+ if [ "${USE_QCOW2}" = "1" ]; then
+ on_chroot << EOF
+apt-get clean
+EOF
+ fi
fi
log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
fi
@@ -32,6 +40,11 @@ EOF
on_chroot << EOF
apt-get install -y $PACKAGES
EOF
+ if [ "${USE_QCOW2}" = "1" ]; then
+ on_chroot << EOF
+apt-get clean
+EOF
+ fi
fi
log "End ${SUB_STAGE_DIR}/${i}-packages"
fi
@@ -82,17 +95,27 @@ EOF
run_stage(){
log "Begin ${STAGE_DIR}"
STAGE="$(basename "${STAGE_DIR}")"
+
pushd "${STAGE_DIR}" > /dev/null
- unmount "${WORK_DIR}/${STAGE}"
+
STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs
+
+ if [ "${USE_QCOW2}" = "1" ]; then
+ if [ ! -f SKIP ]; then
+ load_qimage
+ fi
+ else
+ unmount "${WORK_DIR}/${STAGE}"
+ fi
+
if [ ! -f SKIP_IMAGES ]; then
if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
fi
fi
if [ ! -f SKIP ]; then
- if [ "${CLEAN}" = "1" ]; then
+ if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then
if [ -d "${ROOTFS_DIR}" ]; then
rm -rf "${ROOTFS_DIR}"
fi
@@ -109,7 +132,13 @@ run_stage(){
fi
done
fi
- unmount "${WORK_DIR}/${STAGE}"
+
+ if [ "${USE_QCOW2}" = "1" ]; then
+ unload_qimage
+ else
+ unmount "${WORK_DIR}/${STAGE}"
+ fi
+
PREV_STAGE="${STAGE}"
PREV_STAGE_DIR="${STAGE_DIR}"
PREV_ROOTFS_DIR="${ROOTFS_DIR}"
@@ -141,6 +170,15 @@ do
esac
done
+term() {
+ if [ "${USE_QCOW2}" = "1" ]; then
+ log "Unloading image"
+ unload_qimage
+ fi
+}
+
+trap term EXIT INT TERM
+
export PI_GEN=${PI_GEN:-pi-gen}
export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
@@ -208,6 +246,10 @@ source "${SCRIPT_DIR}/common"
# shellcheck source=scripts/dependencies_check
source "${SCRIPT_DIR}/dependencies_check"
+export USE_QCOW2="${USE_QCOW2:-1}"
+export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
+source "${SCRIPT_DIR}/qcow2_handling"
+
dependencies_check "${BASE_DIR}/depends"
#check username is valid
@@ -237,13 +279,29 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
# shellcheck source=/dev/null
source "${EXPORT_DIR}/EXPORT_IMAGE"
EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
- run_stage
+ QIMAGE="image-$(basename "${EXPORT_DIR}").qcow2"
+ if [ "${USE_QCOW2}" = "1" ]; then
+ USE_QCOW2=0
+ mount_qimage "${WORK_DIR}/${QIMAGE}" "${EXPORT_ROOTFS_DIR}"
+ echo "Mounting image ${WORK_DIR}/${QIMAGE} to export rootfs ${EXPORT_ROOTFS_DIR}"
+ run_stage
+ unload_qimage
+ USE_QCOW2=1
+ else
+ run_stage
+ fi
if [ "${USE_QEMU}" != "1" ]; then
if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
# shellcheck source=/dev/null
source "${EXPORT_DIR}/EXPORT_NOOBS"
STAGE_DIR="${BASE_DIR}/export-noobs"
- run_stage
+ if [ "${USE_QCOW2}" = "1" ]; then
+ USE_QCOW2=0
+ run_stage
+ USE_QCOW2=1
+ else
+ run_stage
+ fi
fi
fi
done
@@ -255,4 +313,8 @@ if [ -x postrun.sh ]; then
log "End postrun.sh"
fi
+if [ "${USE_QCOW2}" = "1" ]; then
+ unload_qimage
+fi
+
log "End ${BASE_DIR}"
diff --git a/depends b/depends
index 6238eb1..25b4701 100644
--- a/depends
+++ b/depends
@@ -16,3 +16,5 @@ xxd
file
git
lsmod:kmod
+qemu-nbd:qemu-utils
+kpartx
diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh
index 0864639..0b1fdc9 100755
--- a/export-image/04-finalise/01-run.sh
+++ b/export-image/04-finalise/01-run.sh
@@ -95,7 +95,16 @@ if [ "${DEPLOY_ZIP}" == "1" ]; then
"$(basename "${IMG_FILE}")"
popd > /dev/null
else
- cp "$IMG_FILE" "$DEPLOY_DIR"
+ if [ "${USE_QCOW2}" = "1" ]; then
+ mv "$IMG_FILE" "$DEPLOY_DIR/"
+ else
+ cp "$IMG_FILE" "$DEPLOY_DIR"
+ fi
+fi
+
+if [ "${USE_QCOW2}" = "1" ]; then
+ mv "$INFO_FILE" "$DEPLOY_DIR/"
+else
+ cp "$INFO_FILE" "$DEPLOY_DIR"
fi
-cp "$INFO_FILE" "$DEPLOY_DIR"
diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh
index 1d0b12f..e313505 100755
--- a/export-noobs/00-release/00-run.sh
+++ b/export-noobs/00-release/00-run.sh
@@ -39,4 +39,8 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|NOOBS_DESCRIPTION|${NOOBS_DESCRIPTION}|"
sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|"
-cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/"
+if [ "${USE_QCOW2}" = "1" ]; then
+ mv "${NOOBS_DIR}" "${DEPLOY_DIR}/"
+else
+ cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/"
+fi
diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh
index 54e0c59..ae88deb 100755
--- a/export-noobs/prerun.sh
+++ b/export-noobs/prerun.sh
@@ -1,11 +1,15 @@
#!/bin/bash -e
-IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}"
-unmount_image "${IMG_FILE}"
-
mkdir -p "${STAGE_WORK_DIR}"
-cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/"
+
+if [ "${USE_QCOW2}" = "1" ]; then
+ IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img"
+else
+ cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/"
+ IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+fi
+unmount_image "${IMG_FILE}"
rm -rf "${NOOBS_DIR}"
diff --git a/imagetool.sh b/imagetool.sh
new file mode 100755
index 0000000..b0ebb00
--- /dev/null
+++ b/imagetool.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+if [ "$(id -u)" != "0" ]; then
+ echo "Please run as root" 1>&2
+ exit 1
+fi
+
+progname=$(basename $0)
+
+function usage()
+{
+ cat << HEREDOC
+
+Usage:
+ Mount Image : $progname [--mount] [--image-name ] [--mount-point ]
+ Umount Image: $progname [--umount] [--mount-point ]
+ Cleanup NBD : $progname [--cleanup]
+
+ arguments:
+ -h, --help show this help message and exit
+ -c, --cleanup cleanup orphaned device mappings
+ -m, --mount mount image
+ -u, --umount umount image
+ -i, --image-name path to qcow2 image
+ -p, --mount-point mount point for image
+
+ This tool will use /dev/nbd1 as default for mounting an image. If you want to use another device, execute like this:
+ NBD_DEV=/dev/nbd2 ./$progname --mount --image --mount-point
+
+HEREDOC
+}
+
+MOUNT=0
+UMOUNT=0
+IMAGE=""
+MOUNTPOINT=""
+
+nbd_cleanup() {
+ DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)"
+ if [ ! -z "${DEVS}" ]; then
+ for d in $DEVS; do
+ if [ ! -z "${d}" ]; then
+ QDEV="$(ps xa | grep $d | grep -v grep)"
+ if [ -z "${QDEV}" ]; then
+ kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d"
+ fi
+ fi
+ done
+ fi
+}
+
+# As long as there is at least one more argument, keep looping
+while [[ $# -gt 0 ]]; do
+ key="$1"
+ case "$key" in
+ -h|--help)
+ usage
+ exit
+ ;;
+ -c|--cleanup)
+ nbd_cleanup
+ ;;
+ -m|--mount)
+ MOUNT=1
+ ;;
+ -u|--umount)
+ UMOUNT=1
+ ;;
+ -i|--image-name)
+ shift
+ IMAGE="$1"
+ ;;
+ -p|--mount-point)
+ shift
+ MOUNTPOINT="$1"
+ ;;
+ *)
+ echo "Unknown option '$key'"
+ usage
+ exit
+ ;;
+ esac
+ # Shift after checking all the cases to get the next option
+ shift
+done
+
+if [ "${MOUNT}" = "1" ] && [ "${UMOUNT}" = "1" ]; then
+ usage
+ echo "Concurrent mount options not possible."
+ exit
+fi
+
+if [ "${MOUNT}" = "1" ] && ([ -z "${IMAGE}" ] || [ -z "${MOUNTPOINT}" ]); then
+ usage
+ echo "Can not mount image. Image path and/or mount point missing."
+ exit
+fi
+
+if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then
+ usage
+ echo "Can not umount. Mount point parameter missing."
+ exit
+fi
+
+export NBD_DEV="${NBD_DEV:-/dev/nbd1}"
+source scripts/qcow2_handling
+
+if [ "${MOUNT}" = "1" ]; then
+ mount_qimage "${MOUNTPOINT}" "${IMAGE}"
+elif [ "${UMOUNT}" = "1" ]; then
+ umount_qimage "${MOUNTPOINT}"
+fi
diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling
new file mode 100644
index 0000000..0d8e4e7
--- /dev/null
+++ b/scripts/qcow2_handling
@@ -0,0 +1,96 @@
+# QCOW2 Routines
+
+export CURRENT_IMAGE
+export CURRENT_MOUNTPOINT
+
+# set in build.sh
+# should be fairly enough for the beginning
+# overwrite here by uncommenting following lines
+# BASE_QCOW2_SIZE=12G
+
+init_nbd() {
+ modprobe nbd max_part=16
+ if [ -z "${NBD_DEV}" ]; then
+ for x in /sys/class/block/nbd* ; do
+ S=`cat $x/size`
+ if [ "$S" == "0" ] ; then
+ NBD_DEV=/dev/$(basename $x)
+ MAP_DEV=/dev/mapper/$(basename $x)p1
+ break
+ fi
+ done
+ fi
+}
+
+# mount qcow2 image: mount_image
+mount_qimage() {
+ init_nbd
+ qemu-nbd --discard=unmap -c $NBD_DEV "$1"
+ kpartx -a $NBD_DEV
+ mount $MAP_DEV "$2"
+ CURRENT_IMAGE="$1"
+ CURRENT_MOUNTPOINT="$2"
+}
+export -f mount_qimage
+
+# umount qcow2 image: umount_image
+umount_qimage() {
+ while mount | grep -q "$1"; do
+ local LOCS
+ LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r)
+ for loc in $LOCS; do
+ echo "$loc"
+ umount "$loc"
+ done
+ done
+ qemu-nbd -d $NBD_DEV
+ kpartx -d $NBD_DEV
+}
+export -f umount_qimage
+
+# create base image / backing image / mount image
+load_qimage() {
+ if [ -z "${CURRENT_MOUNTPOINT}" ]; then
+ if [ ! -d "${ROOTFS_DIR}" ]; then mkdir -p "${ROOTFS_DIR}"; fi
+
+ if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; fi
+
+ if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then
+ pushd ${WORK_DIR} > /dev/null
+ init_nbd
+ if [ -z "${PREV_STAGE}" ]; then
+ qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE
+ qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
+ echo 'type=83' | sfdisk $NBD_DEV
+ kpartx -a $NBD_DEV
+ mkfs.ext4 $MAP_DEV
+ else
+ if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then exit 1; fi
+ qemu-img create -f qcow2 \
+ -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \
+ ${WORK_DIR}/image-${STAGE}.qcow2
+ qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
+ kpartx -a $NBD_DEV
+ fi
+ mount $MAP_DEV "${ROOTFS_DIR}"
+ CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2
+ CURRENT_MOUNTPOINT=${ROOTFS_DIR}
+ popd > /dev/null
+ else
+ mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}"
+ fi
+ echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})"
+ fi
+}
+export -f load_qimage
+
+# umount current image and refresh mount point env var
+unload_qimage() {
+ if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then
+ fstrim -v "${CURRENT_MOUNTPOINT}" || true
+ umount_qimage "${CURRENT_MOUNTPOINT}"
+ CURRENT_IMAGE=""
+ CURRENT_MOUNTPOINT=""
+ fi
+}
+export -f unload_qimage
diff --git a/stage0/prerun.sh b/stage0/prerun.sh
index 9ce3e02..b727cc7 100755
--- a/stage0/prerun.sh
+++ b/stage0/prerun.sh
@@ -1,5 +1,5 @@
#!/bin/bash -e
-if [ ! -d "${ROOTFS_DIR}" ]; then
+if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then
bootstrap buster "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/
fi