diff --git a/build.sh b/build.sh index 3b02fce..b09e496 100755 --- a/build.sh +++ b/build.sh @@ -1,7 +1,5 @@ #!/bin/bash -e -#set -x - # shellcheck disable=SC2119 run_sub_stage() { @@ -106,7 +104,10 @@ run_stage(){ load_qimage fi else - unmount "${WORK_DIR}/${STAGE}" + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi fi if [ ! -f SKIP_IMAGES ]; then @@ -136,7 +137,10 @@ run_stage(){ if [ "${USE_QCOW2}" = "1" ]; then unload_qimage else - unmount "${WORK_DIR}/${STAGE}" + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi fi PREV_STAGE="${STAGE}" @@ -245,9 +249,17 @@ source "${SCRIPT_DIR}/common" # shellcheck source=scripts/dependencies_check source "${SCRIPT_DIR}/dependencies_check" +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" export USE_QCOW2="${USE_QCOW2:-1}" export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} source "${SCRIPT_DIR}/qcow2_handling" +if [ "${USE_QCOW2}" = "1" ]; then + NO_PRERUN_QCOW2=1 +else + NO_PRERUN_QCOW2=0 +fi + +export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" dependencies_check "${BASE_DIR}/depends" @@ -278,14 +290,70 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do # shellcheck source=/dev/null source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - QIMAGE="image-$(basename "${EXPORT_DIR}").qcow2" if [ "${USE_QCOW2}" = "1" ]; then USE_QCOW2=0 - mount_qimage "${WORK_DIR}/${QIMAGE}" "${EXPORT_ROOTFS_DIR}" - echo "Mounting image ${WORK_DIR}/${QIMAGE} to export rootfs ${EXPORT_ROOTFS_DIR}" + EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}" + echo "------------------------------------------------------------------------" + echo "Running export stage for ${EXPORT_NAME}" + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true + EXPORT_STAGE=$(basename "${EXPORT_DIR}") + for s in $STAGE_LIST; do + TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}") + done + FIRST_STAGE=${TMP_LIST%% *} + FIRST_IMAGE="image-${FIRST_STAGE}.qcow2" + + pushd "${WORK_DIR}" > /dev/null + echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}" + cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2" + + ARR=($TMP_LIST) + # rebase stage images to new export base + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="${EXPORT_NAME}" + continue + fi + echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + + # commit current export stage into base export image + echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2" + qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2 + + # rebase stage images back to original first stage for easy re-run + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="image-${CURR_STAGE}" + continue + fi + echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + popd > /dev/null + + mkdir -p "${WORK_DIR}/export-image/rootfs" + mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/" + echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs" + mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs" + + CLEAN=0 run_stage - unload_qimage + CLEAN=1 USE_QCOW2=1 + else run_stage fi @@ -305,7 +373,7 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do fi done -if [ -x ${BASE_DIR}/postrun.sh ]; then +if [ -x postrun.sh ]; then log "Begin postrun.sh" cd "${BASE_DIR}" ./postrun.sh diff --git a/export-image/03-set-partuuid/00-run.sh b/export-image/03-set-partuuid/00-run.sh index 1538c07..cf5c0db 100755 --- a/export-image/03-set-partuuid/00-run.sh +++ b/export-image/03-set-partuuid/00-run.sh @@ -1,13 +1,18 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then -IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -BOOT_PARTUUID="${IMGID}-01" -ROOT_PARTUUID="${IMGID}-02" + IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" -sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + BOOT_PARTUUID="${IMGID}-01" + ROOT_PARTUUID="${IMGID}-02" + + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" + +fi -sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index 0b1fdc9..625bbd1 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -77,34 +77,30 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE" dpkg -l --root "$ROOTFS_DIR" } >> "$INFO_FILE" -ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" - -unmount "${ROOTFS_DIR}" -zerofree "${ROOT_DEV}" - -unmount_image "${IMG_FILE}" - mkdir -p "${DEPLOY_DIR}" rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +mv "$INFO_FILE" "$DEPLOY_DIR/" + +if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" + + unmount "${ROOTFS_DIR}" + zerofree "${ROOT_DEV}" + + unmount_image "${IMG_FILE}" +else + unload_qimage + make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" +fi + if [ "${DEPLOY_ZIP}" == "1" ]; then pushd "${STAGE_WORK_DIR}" > /dev/null zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \ "$(basename "${IMG_FILE}")" popd > /dev/null else - if [ "${USE_QCOW2}" = "1" ]; then - mv "$IMG_FILE" "$DEPLOY_DIR/" - else - cp "$IMG_FILE" "$DEPLOY_DIR" - fi + mv "$IMG_FILE" "$DEPLOY_DIR/" fi - -if [ "${USE_QCOW2}" = "1" ]; then - mv "$INFO_FILE" "$DEPLOY_DIR/" -else - cp "$INFO_FILE" "$DEPLOY_DIR" -fi - diff --git a/export-image/prerun.sh b/export-image/prerun.sh index cecde32..a8ba8d8 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -1,61 +1,63 @@ #!/bin/bash -e -IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" +if [ "${NO_PRERUN_QCOW2}" = "0" ]; then + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" -unmount_image "${IMG_FILE}" + unmount_image "${IMG_FILE}" -rm -f "${IMG_FILE}" + rm -f "${IMG_FILE}" -rm -rf "${ROOTFS_DIR}" -mkdir -p "${ROOTFS_DIR}" + rm -rf "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}" -BOOT_SIZE="$((256 * 1024 * 1024))" -ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + BOOT_SIZE="$((256 * 1024 * 1024))" + ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) -# All partition sizes and starts will be aligned to this size -ALIGN="$((4 * 1024 * 1024))" -# Add this much space to the calculated file size. This allows for -# some overhead (since actual space usage is usually rounded up to the -# filesystem block size) and gives some free space on the resulting -# image. -ROOT_MARGIN=$((800*1024*1024)) + # All partition sizes and starts will be aligned to this size + ALIGN="$((4 * 1024 * 1024))" + # Add this much space to the calculated file size. This allows for + # some overhead (since actual space usage is usually rounded up to the + # filesystem block size) and gives some free space on the resulting + # image. + ROOT_MARGIN=$((800*1024*1024)) -BOOT_PART_START=$((ALIGN)) -BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) -ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) -ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) -IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) + BOOT_PART_START=$((ALIGN)) + BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) + ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) + ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) + IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) -truncate -s "${IMG_SIZE}" "${IMG_FILE}" + truncate -s "${IMG_SIZE}" "${IMG_FILE}" -parted --script "${IMG_FILE}" mklabel msdos -parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" -parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" mklabel msdos + parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" -PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) -BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) -BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) + PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) + BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) + BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) -ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) -ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) + ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) + ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) -BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") -ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") -echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" -echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") + ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") + echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" + echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" -ROOT_FEATURES="^huge_file" -for FEATURE in metadata_csum 64bit; do + ROOT_FEATURES="^huge_file" + for FEATURE in metadata_csum 64bit; do if grep -q "$FEATURE" /etc/mke2fs.conf; then ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi -done -mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null -mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + done + mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null + mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null -mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 -mkdir -p "${ROOTFS_DIR}/boot" -mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 + mkdir -p "${ROOTFS_DIR}/boot" + mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat -rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" -rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" + rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" + rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" +fi diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index ae88deb..905bfb2 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -3,12 +3,12 @@ NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}" mkdir -p "${STAGE_WORK_DIR}" -if [ "${USE_QCOW2}" = "1" ]; then +if [ "${DEPLOY_ZIP}" == "1" ]; then IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" else - cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/" - IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" fi + unmount_image "${IMG_FILE}" rm -rf "${NOOBS_DIR}" @@ -37,4 +37,8 @@ bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs/boot" -cpf - umount "${STAGE_WORK_DIR}/rootfs/boot" bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs" --one-file-system -cpf - . | xz -T0 > "${NOOBS_DIR}/root.tar.xz" +if [ "${USE_QCOW2}" = "1" ]; then + rm "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service" +fi + unmount_image "${IMG_FILE}" diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling index a04c742..8f6d69e 100644 --- a/scripts/qcow2_handling +++ b/scripts/qcow2_handling @@ -1,37 +1,62 @@ +#!/bin/bash + # QCOW2 Routines export CURRENT_IMAGE export CURRENT_MOUNTPOINT export NBD_DEV -export MAP_DEV +export MAP_BOOT_DEV +export MAP_ROOT_DEV # set in build.sh # should be fairly enough for the beginning # overwrite here by uncommenting following lines # BASE_QCOW2_SIZE=12G +# find and initialize free block device nodes init_nbd() { modprobe nbd max_part=16 if [ -z "${NBD_DEV}" ]; then for x in /sys/class/block/nbd* ; do S=`cat $x/size` if [ "$S" == "0" ] ; then - NBD_DEV=/dev/$(basename $x) - MAP_DEV=/dev/mapper/$(basename $x)p1 - break + NBD_DEV=/dev/$(basename $x) + MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1 + MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2 + break fi done fi } +export -f init_nbd -# mount qcow2 image: mount_image -mount_qimage() { +# connect image to block device +connect_blkdev() { init_nbd qemu-nbd --discard=unmap -c $NBD_DEV "$1" kpartx -a $NBD_DEV - mount $MAP_DEV "$2" CURRENT_IMAGE="$1" +} +export -f connect_blkdev + +# disconnect image from block device +disconnect_blkdev() { + kpartx -d $NBD_DEV + qemu-nbd -d $NBD_DEV + NBD_DEV= + MAP_BOOT_DEV= + MAP_ROOT_DEV= + CURRENT_IMAGE= +} +export -f disconnect_blkdev + +# mount qcow2 image: mount_image +mount_qimage() { + connect_blkdev "$1" + mount -v -t ext4 $MAP_ROOT_DEV "$2" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "$2/boot" CURRENT_MOUNTPOINT="$2" } export -f mount_qimage @@ -39,16 +64,19 @@ export -f mount_qimage # umount qcow2 image: umount_image umount_qimage() { sync + #umount "$1/boot" while mount | grep -q "$1"; do local LOCS LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) for loc in $LOCS; do - echo "$loc" - umount "$loc" + echo "$loc" + while mountpoint -q "$loc" && ! umount "$loc"; do + sleep 0.1 + done done done - kpartx -d $NBD_DEV - qemu-nbd -d $NBD_DEV + CURRENT_MOUNTPOINT= + disconnect_blkdev } export -f umount_qimage @@ -61,29 +89,38 @@ load_qimage() { if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then pushd ${WORK_DIR} > /dev/null - init_nbd + init_nbd if [ -z "${PREV_STAGE}" ]; then + echo "Creating base image: image-${STAGE}.qcow2" + # -o preallocation=falloc qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 - echo 'type=83' | sfdisk $NBD_DEV + sfdisk $NBD_DEV << EOF +,250MiB,b +,,83; +EOF kpartx -a $NBD_DEV - mkfs.ext4 $MAP_DEV + mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV + mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV else - if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then exit 1; fi + if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then exit 1; fi + echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2" qemu-img create -f qcow2 \ -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ ${WORK_DIR}/image-${STAGE}.qcow2 qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 kpartx -a $NBD_DEV fi - mount $MAP_DEV "${ROOTFS_DIR}" - CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 - CURRENT_MOUNTPOINT=${ROOTFS_DIR} - popd > /dev/null - else - mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" - fi - echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" + mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot" + CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 + CURRENT_MOUNTPOINT=${ROOTFS_DIR} + popd > /dev/null + else + mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" + fi + echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" fi } export -f load_qimage @@ -92,9 +129,105 @@ export -f load_qimage unload_qimage() { if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then fstrim -v "${CURRENT_MOUNTPOINT}" || true - umount_qimage "${CURRENT_MOUNTPOINT}" - CURRENT_IMAGE="" - CURRENT_MOUNTPOINT="" + umount_qimage "${CURRENT_MOUNTPOINT}" fi } export -f unload_qimage + +# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer +# helper function for make_bootable_image, do not call directly +function resize_qcow2() { + if [ -z "$CALL_FROM_MBI" ]; then echo "resize_qcow2: cannot be called directly, use make_bootable_image instead"; return 1; fi + + ROOT_MARGIN=$((800*1024*1024)) + PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print` + PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '` + PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '` + + e2fsck -y -f $MAP_ROOT_DEV || true + + DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '` + BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }') + BLOCK_SIZE=${BLOCK_SIZE// /} + + let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE + resize2fs -p $MAP_ROOT_DEV $DATA_SIZE + sleep 1 + + let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE + let PART_NEW_END=$PART_START+$PART_NEW_SIZE + ACT1=`parted -s "$NBD_DEV" rm 2` + ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END` + NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '` +} +export -f resize_qcow2 + +# create raw img from qcow2: make_bootable_image +function make_bootable_image() { + + EXPORT_QCOW2="$1" + EXPORT_IMAGE="$2" + + echo "Connect block device to source qcow2" + connect_blkdev "${EXPORT_QCOW2}" + + echo "Resize fs and partition" + CALL_FROM_MBI=1 + resize_qcow2 + CALL_FROM_MBI= + + echo "Disconnect block device" + disconnect_blkdev + + if [ -z "$NEW_IMG_SIZE" ]; then + echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit." + exit 1 + fi + + echo "Shrinking qcow2 image" + qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE + + echo "Convert qcow2 to raw image" + qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}" + + echo "Get PARTUUIDs from image" + IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")" + + BOOT_PARTUUID="${IMGID}-01" + echo "Boot: $BOOT_PARTUUID" + ROOT_PARTUUID="${IMGID}-02" + echo "Root: $ROOT_PARTUUID" + + echo "Mount image" + MOUNTROOT=${WORK_DIR}/tmpimage + mkdir -p $MOUNTROOT + + MOUNTPT=$MOUNTROOT + PARTITION=2 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + MOUNTPT=$MOUNTROOT/boot + PARTITION=1 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + + if [ ! -d "${MOUNTROOT}/root" ]; then + echo "Image damaged or not mounted. Exit." + exit 1 + fi + + echo "Setup PARTUUIDs" + if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then + echo "Set UUIDs to make it bootable" + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt" + fi + + echo "Umount image" + umount "${MOUNTROOT}/boot" || exit 1 + umount "${MOUNTROOT}" || exit 1 + + echo "Remove qcow2 export image" + rm -f "${EXPORT_QCOW2}" +} +export -f make_bootable_image