diff --git a/Dockerfile b/Dockerfile index 2e53149..9110925 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get -y update && \ git vim parted \ quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \ bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\ - binfmt-support ca-certificates \ + binfmt-support ca-certificates qemu-utils kpartx \ && rm -rf /var/lib/apt/lists/* COPY . /pi-gen/ diff --git a/README.md b/README.md index 29d8166..388a61f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ To install the required dependencies for pi-gen you should run: ```bash apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \ -dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc +dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc \ +qemu-utils kpartx ``` The file `depends` contains a list of tools needed. The format of this @@ -36,7 +37,30 @@ The following environment variables are supported: but you should use something else for a customized version. Export files in stages may add suffixes to `IMG_NAME`. - * `RELEASE` (Default: buster) +* `USE_QCOW2`(Default: `1` ) + + Instead of using traditional way of building the rootfs of every stage in + single subdirectories and copying over the previous one to the next one, + qcow2 based virtual disks with backing images are used in every stage. + This speeds up the build process and reduces overall space consumption + significantly. + + Additional optional parameters regarding qcow2 build: + + * `BASE_QCOW2_SIZE` (Default: 12G) + + Size of the virtual qcow2 disk. + Note: it will not actually use that much of space at once but defines the + maximum size of the virtual disk. If you change the build process by adding + a lot of bigger packages or additional build stages, it can be necessary to + increase the value because the virtual disk can run out of space like a normal + hard drive would. + + **CAUTION:** Although the qcow2 build mechanism will run fine inside Docker, it can happen + that the network block device is not disconnected correctly after the Docker process has + ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong) + +* `RELEASE` (Default: buster) The release version to build images against. Valid values are jessie, stretch buster, bullseye, and testing. @@ -328,6 +352,71 @@ follows: * Once you're happy with the image you can remove the SKIP_IMAGES files and export your image to test +# Regarding Qcow2 image building + +### Get infos about the image in use + +If you issue the two commands shown in the example below in a second command shell while a build +is running you can find out, which network block device is currently being used and which qcow2 image +is bound to it. + +Example: + +```bash +root@build-machine:~/$ lsblk | grep nbd +nbd1 43:32 0 10G 0 disk +├─nbd1p1 43:33 0 10G 0 part +└─nbd1p1 253:0 0 10G 0 part + +root@build-machine:~/$ ps xa | grep qemu-nbd + 2392 pts/6 S+ 0:00 grep --color=auto qemu-nbd +31294 ? Ssl 0:12 qemu-nbd --discard=unmap -c /dev/nbd1 image-stage4.qcow2 +``` + +Here you can see, that the qcow2 image `image-stage4.qcow2` is currently connected to `/dev/nbd1` with +the associated partition map `/dev/mapper/nbd1p1`. Don't worry that `lsblk` shows two entries. It is totally fine, because the device map is accessible via `/dev/mapper/nbd1p1` and also via `/dev/dm-0`. This is all part of the device mapper functionality of the kernel. See `dmsetup` for further information. + +### Mount a qcow2 image + +If you want to examine the content of a a single stage, you can simply mount the qcow2 image found in the `WORK_DIR` directory with the tool `./imagetool.sh`. + +See `./imagetool.sh -h` for further details on how to use it. + +### Disconnect an image if something went wrong + +It can happen, that your build stops in case of an error. Normally `./build.sh` should handle image disconnection appropriately, but in rare cases, especially during a Docker build, this may not work as expected. If that happens, starting a new build will fail and you may have to disconnect the image and/or device yourself. + +A typical message indicating that there are some orphaned device mapper entries is this: + +``` +Failed to set NBD socket +Disconnect client, due to: Unexpected end-of-file before all bytes were read +``` + +If that happens go through the following steps: + +1. First, check if the image is somehow mounted to a directory entry and umount it as you would any other block device, like i.e. a hard disk or USB stick. + +2. Second, to disconnect an image from `qemu-nbd`, the QEMU Disk Network Block Device Server, issue the following command (be sure to change the device name to the one actually used): + + ```bash + sudo qemu-nbd -d /dev/nbd1 + ``` + + Note: if you use Docker build, normally no active `qemu-nbd` process exists anymore as it will be terminated when the Docker container stops. + +3. To disconnect a device partition map from the network block device, execute: + + ```bash + sudo kpartx -d /dev/nbd1 + or + sudo ./imagetool.sh --cleanup + ``` + + Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well. + +Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. + # Troubleshooting ## `64 Bit Systems` diff --git a/build-docker.sh b/build-docker.sh index b6a9ea3..aba71e6 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,4 +1,5 @@ #!/bin/bash -eu + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" BUILD_OPTS="$*" @@ -77,6 +78,9 @@ ${DOCKER} build -t pi-gen "${DIR}" if [ "${CONTAINER_EXISTS}" != "" ]; then trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM time ${DOCKER} run --rm --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ @@ -88,6 +92,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then else trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \ + --cap-add=ALL \ + -v /dev:/dev \ + -v /lib/modules:/lib/modules \ --volume "${CONFIG_FILE}":/config:ro \ -e "GIT_HASH=${GIT_HASH}" \ pi-gen \ @@ -96,6 +103,7 @@ else rsync -av work/*/build.log deploy/" & wait "$!" fi + echo "copying results from deploy/" ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy . ls -lah deploy diff --git a/build.sh b/build.sh index a8247ab..c00555a 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,5 @@ #!/bin/bash -e + # shellcheck disable=SC2119 run_sub_stage() { @@ -13,7 +14,7 @@ $(cat "${i}-debconf") SELEOF EOF - log "End ${SUB_STAGE_DIR}/${i}-debconf" + log "End ${SUB_STAGE_DIR}/${i}-debconf" fi if [ -f "${i}-packages-nr" ]; then log "Begin ${SUB_STAGE_DIR}/${i}-packages-nr" @@ -22,6 +23,11 @@ EOF on_chroot << EOF apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages-nr" fi @@ -32,6 +38,11 @@ EOF on_chroot << EOF apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF +apt-get clean +EOF + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages" fi @@ -82,17 +93,30 @@ EOF run_stage(){ log "Begin ${STAGE_DIR}" STAGE="$(basename "${STAGE_DIR}")" + pushd "${STAGE_DIR}" > /dev/null - unmount "${WORK_DIR}/${STAGE}" + STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs + + if [ "${USE_QCOW2}" = "1" ]; then + if [ ! -f SKIP ]; then + load_qimage + fi + else + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi + fi + if [ ! -f SKIP_IMAGES ]; then if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" fi fi if [ ! -f SKIP ]; then - if [ "${CLEAN}" = "1" ]; then + if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then if [ -d "${ROOTFS_DIR}" ]; then rm -rf "${ROOTFS_DIR}" fi @@ -103,13 +127,21 @@ run_stage(){ log "End ${STAGE_DIR}/prerun.sh" fi for SUB_STAGE_DIR in "${STAGE_DIR}"/*; do - if [ -d "${SUB_STAGE_DIR}" ] && - [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then + if [ -d "${SUB_STAGE_DIR}" ] && [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then run_sub_stage fi done fi - unmount "${WORK_DIR}/${STAGE}" + + if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage + else + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi + fi + PREV_STAGE="${STAGE}" PREV_STAGE_DIR="${STAGE_DIR}" PREV_ROOTFS_DIR="${ROOTFS_DIR}" @@ -143,6 +175,15 @@ do esac done +term() { + if [ "${USE_QCOW2}" = "1" ]; then + log "Unloading image" + unload_qimage + fi +} + +trap term EXIT INT TERM + export PI_GEN=${PI_GEN:-pi-gen} export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen} @@ -208,6 +249,18 @@ source "${SCRIPT_DIR}/common" # shellcheck source=scripts/dependencies_check source "${SCRIPT_DIR}/dependencies_check" +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" +export USE_QCOW2="${USE_QCOW2:-1}" +export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} +source "${SCRIPT_DIR}/qcow2_handling" +if [ "${USE_QCOW2}" = "1" ]; then + NO_PRERUN_QCOW2=1 +else + NO_PRERUN_QCOW2=0 +fi + +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" + dependencies_check "${BASE_DIR}/depends" #check username is valid @@ -242,22 +295,98 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - run_stage + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}" + echo "------------------------------------------------------------------------" + echo "Running export stage for ${EXPORT_NAME}" + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true + EXPORT_STAGE=$(basename "${EXPORT_DIR}") + for s in $STAGE_LIST; do + TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}") + done + FIRST_STAGE=${TMP_LIST%% *} + FIRST_IMAGE="image-${FIRST_STAGE}.qcow2" + + pushd "${WORK_DIR}" > /dev/null + echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}" + cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2" + + ARR=($TMP_LIST) + # rebase stage images to new export base + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="${EXPORT_NAME}" + continue + fi + echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + + # commit current export stage into base export image + echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2" + qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2 + + # rebase stage images back to original first stage for easy re-run + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="image-${CURR_STAGE}" + continue + fi + echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + popd > /dev/null + + mkdir -p "${WORK_DIR}/export-image/rootfs" + mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/" + echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs" + mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs" + + CLEAN=0 + run_stage + CLEAN=1 + USE_QCOW2=1 + + else + run_stage + fi if [ "${USE_QEMU}" != "1" ]; then if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_NOOBS" STAGE_DIR="${BASE_DIR}/export-noobs" - run_stage + if [ "${USE_QCOW2}" = "1" ]; then + USE_QCOW2=0 + run_stage + USE_QCOW2=1 + else + run_stage + fi fi fi done -if [ -x ${BASE_DIR}/postrun.sh ]; then +if [ -x postrun.sh ]; then log "Begin postrun.sh" cd "${BASE_DIR}" ./postrun.sh log "End postrun.sh" fi +if [ "${USE_QCOW2}" = "1" ]; then + unload_qimage +fi + log "End ${BASE_DIR}" diff --git a/depends b/depends index a86bc82..cb029fd 100644 --- a/depends +++ b/depends @@ -17,3 +17,5 @@ file git lsmod:kmod bc +qemu-nbd:qemu-utils +kpartx diff --git a/export-image/03-set-partuuid/00-run.sh b/export-image/03-set-partuuid/00-run.sh index 1538c07..16e1b15 100755 --- a/export-image/03-set-partuuid/00-run.sh +++ b/export-image/03-set-partuuid/00-run.sh @@ -1,13 +1,18 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then -IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -BOOT_PARTUUID="${IMGID}-01" -ROOT_PARTUUID="${IMGID}-02" + IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" -sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + BOOT_PARTUUID="${IMGID}-01" + ROOT_PARTUUID="${IMGID}-02" + + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" + +fi -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index 0864639..9afa115 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -77,25 +77,30 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" dpkg -l --root "$ROOTFS_DIR" } >> "$INFO_FILE" -ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" - -unmount "${ROOTFS_DIR}" -zerofree "${ROOT_DEV}" - -unmount_image "${IMG_FILE}" - mkdir -p "${DEPLOY_DIR}" rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +mv "$INFO_FILE" "$DEPLOY_DIR/" + +if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" + + unmount "${ROOTFS_DIR}" + zerofree "${ROOT_DEV}" + + unmount_image "${IMG_FILE}" +else + unload_qimage + make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" +fi + if [ "${DEPLOY_ZIP}" == "1" ]; then pushd "${STAGE_WORK_DIR}" > /dev/null zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \ "$(basename "${IMG_FILE}")" popd > /dev/null else - cp "$IMG_FILE" "$DEPLOY_DIR" + mv "$IMG_FILE" "$DEPLOY_DIR/" fi - -cp "$INFO_FILE" "$DEPLOY_DIR" diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 8bbc566..efd175a 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -1,61 +1,63 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -unmount_image "${IMG_FILE}" + unmount_image "${IMG_FILE}" -rm -f "${IMG_FILE}" + rm -f "${IMG_FILE}" -rm -rf "${ROOTFS_DIR}" -mkdir -p "${ROOTFS_DIR}" + rm -rf "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}" -BOOT_SIZE="$((256 * 1024 * 1024))" -ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + BOOT_SIZE="$((256 * 1024 * 1024))" + ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) -# All partition sizes and starts will be aligned to this size -ALIGN="$((4 * 1024 * 1024))" -# Add this much space to the calculated file size. This allows for -# some overhead (since actual space usage is usually rounded up to the -# filesystem block size) and gives some free space on the resulting -# image. -ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" + # All partition sizes and starts will be aligned to this size + ALIGN="$((4 * 1024 * 1024))" + # Add this much space to the calculated file size. This allows for + # some overhead (since actual space usage is usually rounded up to the + # filesystem block size) and gives some free space on the resulting + # image. + ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" -BOOT_PART_START=$((ALIGN)) -BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) -ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) -ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) -IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) + BOOT_PART_START=$((ALIGN)) + BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) + ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) + ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) + IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) -truncate -s "${IMG_SIZE}" "${IMG_FILE}" + truncate -s "${IMG_SIZE}" "${IMG_FILE}" -parted --script "${IMG_FILE}" mklabel msdos -parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" -parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" mklabel msdos + parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" -PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) -BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) -BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) + PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) + BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) + BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) -ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) -ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) + ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) + ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) -BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") -ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") -echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" -echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") + ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") + echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" + echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" -ROOT_FEATURES="^huge_file" -for FEATURE in metadata_csum 64bit; do + ROOT_FEATURES="^huge_file" + for FEATURE in metadata_csum 64bit; do if grep -q "$FEATURE" /etc/mke2fs.conf; then - ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" + ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi -done -mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null -mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + done + mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null + mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null -mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 -mkdir -p "${ROOTFS_DIR}/boot" -mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 + mkdir -p "${ROOTFS_DIR}/boot" + mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat -rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" -rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" + rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" + rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" +fi diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index 18a3a5d..216096b 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -40,4 +40,8 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|RELEASE|${RELEASE}|" sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|" -cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" +if [ "${USE_QCOW2}" = "1" ]; then + mv "${NOOBS_DIR}" "${DEPLOY_DIR}/" +else + cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" +fi diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 54e0c59..b7f2b4a 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -1,11 +1,15 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}" -unmount_image "${IMG_FILE}" - mkdir -p "${STAGE_WORK_DIR}" -cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/" + +if [ "${DEPLOY_ZIP}" == "1" ]; then + IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" +else + IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +fi + +unmount_image "${IMG_FILE}" rm -rf "${NOOBS_DIR}" @@ -33,4 +37,8 @@ bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs/boot" -cpf - umount "${STAGE_WORK_DIR}/rootfs/boot" bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs" --one-file-system -cpf - . | xz -T0 > "${NOOBS_DIR}/root.tar.xz" +if [ "${USE_QCOW2}" = "1" ]; then + rm "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service" +fi + unmount_image "${IMG_FILE}" diff --git a/imagetool.sh b/imagetool.sh new file mode 100755 index 0000000..adc1ab7 --- /dev/null +++ b/imagetool.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +if [ "$(id -u)" != "0" ]; then + echo "Please run as root" 1>&2 + exit 1 +fi + +progname=$(basename $0) + +function usage() +{ + cat << HEREDOC + +Usage: + Mount Image : $progname [--mount] [--image-name ] [--mount-point ] + Umount Image: $progname [--umount] [--mount-point ] + Cleanup NBD : $progname [--cleanup] + + arguments: + -h, --help show this help message and exit + -c, --cleanup cleanup orphaned device mappings + -m, --mount mount image + -u, --umount umount image + -i, --image-name path to qcow2 image + -p, --mount-point mount point for image + + This tool will use /dev/nbd1 as default for mounting an image. If you want to use another device, execute like this: + NBD_DEV=/dev/nbd2 ./$progname --mount --image --mount-point + +HEREDOC +} + +MOUNT=0 +UMOUNT=0 +IMAGE="" +MOUNTPOINT="" + +nbd_cleanup() { + DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)" + if [ ! -z "${DEVS}" ]; then + for d in $DEVS; do + if [ ! -z "${d}" ]; then + QDEV="$(ps xa | grep $d | grep -v grep)" + if [ -z "${QDEV}" ]; then + kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d" + fi + fi + done + fi +} + +# As long as there is at least one more argument, keep looping +while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + -h|--help) + usage + exit + ;; + -c|--cleanup) + nbd_cleanup + ;; + -m|--mount) + MOUNT=1 + ;; + -u|--umount) + UMOUNT=1 + ;; + -i|--image-name) + shift + IMAGE="$1" + ;; + -p|--mount-point) + shift + MOUNTPOINT="$1" + ;; + *) + echo "Unknown option '$key'" + usage + exit + ;; + esac + # Shift after checking all the cases to get the next option + shift +done + +if [ "${MOUNT}" = "1" ] && [ "${UMOUNT}" = "1" ]; then + usage + echo "Concurrent mount options not possible." + exit +fi + +if [ "${MOUNT}" = "1" ] && ([ -z "${IMAGE}" ] || [ -z "${MOUNTPOINT}" ]); then + usage + echo "Can not mount image. Image path and/or mount point missing." + exit +fi + +if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then + usage + echo "Can not umount. Mount point parameter missing." + exit +fi + +export NBD_DEV="${NBD_DEV:-/dev/nbd1}" +source scripts/qcow2_handling + +if [ "${MOUNT}" = "1" ]; then + mount_qimage "${MOUNTPOINT}" "${IMAGE}" +elif [ "${UMOUNT}" = "1" ]; then + umount_qimage "${MOUNTPOINT}" +fi diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling new file mode 100644 index 0000000..f722937 --- /dev/null +++ b/scripts/qcow2_handling @@ -0,0 +1,256 @@ +#!/bin/bash + +# QCOW2 Routines + +export CURRENT_IMAGE +export CURRENT_MOUNTPOINT + +export NBD_DEV +export MAP_BOOT_DEV +export MAP_ROOT_DEV + +# set in build.sh +# should be fairly enough for the beginning +# overwrite here by uncommenting following lines +# BASE_QCOW2_SIZE=12G + +# find and initialize free block device nodes +init_nbd() { + modprobe nbd max_part=16 + if [ -z "${NBD_DEV}" ]; then + for x in /sys/class/block/nbd* ; do + S=`cat $x/size` + if [ "$S" == "0" ] ; then + NBD_DEV=/dev/$(basename $x) + MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1 + MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2 + break + fi + done + fi +} +export -f init_nbd + +# connect image to block device +connect_blkdev() { + init_nbd + qemu-nbd --discard=unmap -c $NBD_DEV "$1" + sync + kpartx -a $NBD_DEV + sync + CURRENT_IMAGE="$1" +} +export -f connect_blkdev + +# disconnect image from block device +disconnect_blkdev() { + kpartx -d $NBD_DEV + qemu-nbd -d $NBD_DEV + NBD_DEV= + MAP_BOOT_DEV= + MAP_ROOT_DEV= + CURRENT_IMAGE= +} +export -f disconnect_blkdev + +# mount qcow2 image: mount_image +mount_qimage() { + connect_blkdev "$1" + mount -v -t ext4 $MAP_ROOT_DEV "$2" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "$2/boot" + CURRENT_MOUNTPOINT="$2" +} +export -f mount_qimage + +# umount qcow2 image: umount_image +umount_qimage() { + sync + #umount "$1/boot" + while mount | grep -q "$1"; do + local LOCS + LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) + for loc in $LOCS; do + echo "$loc" + while mountpoint -q "$loc" && ! umount "$loc"; do + sleep 0.1 + done + done + done + CURRENT_MOUNTPOINT= + disconnect_blkdev +} +export -f umount_qimage + +# create base image / backing image / mount image +load_qimage() { + if [ -z "${CURRENT_MOUNTPOINT}" ]; then + if [ ! -d "${ROOTFS_DIR}" ]; then + mkdir -p "${ROOTFS_DIR}"; + fi + + if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; + fi + + if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + pushd ${WORK_DIR} > /dev/null + init_nbd + if [ -z "${PREV_STAGE}" ]; then + echo "Creating base image: image-${STAGE}.qcow2" + # -o preallocation=falloc + qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + sfdisk $NBD_DEV << EOF +4MiB,250MiB,c,* +254MiB,,83; +EOF + sync + kpartx -a $NBD_DEV + mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV + mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV + sync + else + if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then + exit 1; + fi + echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2" + qemu-img create -f qcow2 \ + -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ + ${WORK_DIR}/image-${STAGE}.qcow2 + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + kpartx -a $NBD_DEV + fi + + mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot" + CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 + CURRENT_MOUNTPOINT=${ROOTFS_DIR} + popd > /dev/null + else + mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" + fi + echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" + fi +} +export -f load_qimage + +# umount current image and refresh mount point env var +unload_qimage() { + if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then + fstrim -v "${CURRENT_MOUNTPOINT}" || true + umount_qimage "${CURRENT_MOUNTPOINT}" + fi +} +export -f unload_qimage + +# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer +# helper function for make_bootable_image, do not call directly +function resize_qcow2() { + if [ -z "$CALL_FROM_MBI" ]; then + echo "resize_qcow2: cannot be called directly, use make_bootable_image instead" + return 1 + fi + + # ROOT_MARGIN=$((800*1024*1024)) + ROOT_MARGIN=$((1*1024*1024)) + PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print` + PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '` + PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '` + + e2fsck -y -f $MAP_ROOT_DEV || true + + DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '` + BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }') + BLOCK_SIZE=${BLOCK_SIZE// /} + + let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE + resize2fs -p $MAP_ROOT_DEV $DATA_SIZE + sleep 1 + + let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE + let PART_NEW_END=$PART_START+$PART_NEW_SIZE + ACT1=`parted -s "$NBD_DEV" rm 2` + ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END` + NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '` +} +export -f resize_qcow2 + +# create raw img from qcow2: make_bootable_image +function make_bootable_image() { + + EXPORT_QCOW2="$1" + EXPORT_IMAGE="$2" + + echo "Connect block device to source qcow2" + connect_blkdev "${EXPORT_QCOW2}" + + echo "Resize fs and partition" + CALL_FROM_MBI=1 + resize_qcow2 + sync + CALL_FROM_MBI= + + echo "Disconnect block device" + disconnect_blkdev + + if [ -z "$NEW_IMG_SIZE" ]; then + echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit." + exit 1 + fi + + echo "Shrinking qcow2 image" + qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE + sync + + echo "Convert qcow2 to raw image" + qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}" + sync + + echo "Get PARTUUIDs from image" + IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")" + + BOOT_PARTUUID="${IMGID}-01" + echo "Boot: $BOOT_PARTUUID" + ROOT_PARTUUID="${IMGID}-02" + echo "Root: $ROOT_PARTUUID" + + echo "Mount image" + MOUNTROOT=${WORK_DIR}/tmpimage + mkdir -p $MOUNTROOT + + MOUNTPT=$MOUNTROOT + PARTITION=2 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + MOUNTPT=$MOUNTROOT/boot + PARTITION=1 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + if [ ! -d "${MOUNTROOT}/root" ]; then + echo "Image damaged or not mounted. Exit." + exit 1 + fi + + echo "Setup PARTUUIDs" + if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then + echo "Set UUIDs to make it bootable" + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt" + fi + + echo "Umount image" + sync + umount "${MOUNTROOT}/boot" || exit 1 + umount "${MOUNTROOT}" || exit 1 + + echo "Remove qcow2 export image" + rm -f "${EXPORT_QCOW2}" +} +export -f make_bootable_image diff --git a/stage0/prerun.sh b/stage0/prerun.sh index 3b25783..d4dd0a1 100755 --- a/stage0/prerun.sh +++ b/stage0/prerun.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -if [ ! -d "${ROOTFS_DIR}" ]; then +if [ ! -d "${ROOTFS_DIR}" ] || [ "${USE_QCOW2}" = "1" ]; then bootstrap ${RELEASE} "${ROOTFS_DIR}" http://raspbian.raspberrypi.org/raspbian/ fi