Export stage completely reworked, uses qemu-img commit now

pull/349/head
Holger Pandel 2019-12-02 17:19:05 +01:00
parent 53de9f344c
commit 4ea3675b52
6 changed files with 314 additions and 106 deletions

View File

@ -1,7 +1,5 @@
#!/bin/bash -e
#set -x
# shellcheck disable=SC2119
run_sub_stage()
{
@ -106,8 +104,11 @@ run_stage(){
load_qimage
fi
else
# make sure we are not umounting during export-image stage
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
unmount "${WORK_DIR}/${STAGE}"
fi
fi
if [ ! -f SKIP_IMAGES ]; then
if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
@ -136,8 +137,11 @@ run_stage(){
if [ "${USE_QCOW2}" = "1" ]; then
unload_qimage
else
# make sure we are not umounting during export-image stage
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
unmount "${WORK_DIR}/${STAGE}"
fi
fi
PREV_STAGE="${STAGE}"
PREV_STAGE_DIR="${STAGE_DIR}"
@ -245,9 +249,17 @@ source "${SCRIPT_DIR}/common"
# shellcheck source=scripts/dependencies_check
source "${SCRIPT_DIR}/dependencies_check"
export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
export USE_QCOW2="${USE_QCOW2:-1}"
export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
source "${SCRIPT_DIR}/qcow2_handling"
if [ "${USE_QCOW2}" = "1" ]; then
NO_PRERUN_QCOW2=1
else
NO_PRERUN_QCOW2=0
fi
export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
dependencies_check "${BASE_DIR}/depends"
@ -278,14 +290,70 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
# shellcheck source=/dev/null
source "${EXPORT_DIR}/EXPORT_IMAGE"
EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
QIMAGE="image-$(basename "${EXPORT_DIR}").qcow2"
if [ "${USE_QCOW2}" = "1" ]; then
USE_QCOW2=0
mount_qimage "${WORK_DIR}/${QIMAGE}" "${EXPORT_ROOTFS_DIR}"
echo "Mounting image ${WORK_DIR}/${QIMAGE} to export rootfs ${EXPORT_ROOTFS_DIR}"
EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}"
echo "------------------------------------------------------------------------"
echo "Running export stage for ${EXPORT_NAME}"
rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true
rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true
rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true
rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true
EXPORT_STAGE=$(basename "${EXPORT_DIR}")
for s in $STAGE_LIST; do
TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}")
done
FIRST_STAGE=${TMP_LIST%% *}
FIRST_IMAGE="image-${FIRST_STAGE}.qcow2"
pushd "${WORK_DIR}" > /dev/null
echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}"
cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2"
ARR=($TMP_LIST)
# rebase stage images to new export base
for CURR_STAGE in "${ARR[@]}"; do
if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
PREV_IMG="${EXPORT_NAME}"
continue
fi
echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
break
fi
PREV_IMG="image-${CURR_STAGE}"
done
# commit current export stage into base export image
echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2"
qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2
# rebase stage images back to original first stage for easy re-run
for CURR_STAGE in "${ARR[@]}"; do
if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
PREV_IMG="image-${CURR_STAGE}"
continue
fi
echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
break
fi
PREV_IMG="image-${CURR_STAGE}"
done
popd > /dev/null
mkdir -p "${WORK_DIR}/export-image/rootfs"
mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/"
echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs"
mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs"
CLEAN=0
run_stage
unload_qimage
CLEAN=1
USE_QCOW2=1
else
run_stage
fi
@ -305,7 +373,7 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
fi
done
if [ -x ${BASE_DIR}/postrun.sh ]; then
if [ -x postrun.sh ]; then
log "Begin postrun.sh"
cd "${BASE_DIR}"
./postrun.sh

View File

@ -1,5 +1,7 @@
#!/bin/bash -e
if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
@ -11,3 +13,6 @@ sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"
fi

View File

@ -77,17 +77,24 @@ cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
dpkg -l --root "$ROOTFS_DIR"
} >> "$INFO_FILE"
mkdir -p "${DEPLOY_DIR}"
rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip"
rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
mv "$INFO_FILE" "$DEPLOY_DIR/"
if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')"
unmount "${ROOTFS_DIR}"
zerofree "${ROOT_DEV}"
unmount_image "${IMG_FILE}"
mkdir -p "${DEPLOY_DIR}"
rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip"
rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
else
unload_qimage
make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE"
fi
if [ "${DEPLOY_ZIP}" == "1" ]; then
pushd "${STAGE_WORK_DIR}" > /dev/null
@ -95,16 +102,5 @@ if [ "${DEPLOY_ZIP}" == "1" ]; then
"$(basename "${IMG_FILE}")"
popd > /dev/null
else
if [ "${USE_QCOW2}" = "1" ]; then
mv "$IMG_FILE" "$DEPLOY_DIR/"
else
cp "$IMG_FILE" "$DEPLOY_DIR"
fi
fi
if [ "${USE_QCOW2}" = "1" ]; then
mv "$INFO_FILE" "$DEPLOY_DIR/"
else
cp "$INFO_FILE" "$DEPLOY_DIR"
fi

View File

@ -1,5 +1,6 @@
#!/bin/bash -e
if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
unmount_image "${IMG_FILE}"
@ -59,3 +60,4 @@ mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
fi

View File

@ -3,12 +3,12 @@
NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}"
mkdir -p "${STAGE_WORK_DIR}"
if [ "${USE_QCOW2}" = "1" ]; then
if [ "${DEPLOY_ZIP}" == "1" ]; then
IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img"
else
cp "${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" "${STAGE_WORK_DIR}/"
IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
fi
unmount_image "${IMG_FILE}"
rm -rf "${NOOBS_DIR}"
@ -37,4 +37,8 @@ bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs/boot" -cpf -
umount "${STAGE_WORK_DIR}/rootfs/boot"
bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs" --one-file-system -cpf - . | xz -T0 > "${NOOBS_DIR}/root.tar.xz"
if [ "${USE_QCOW2}" = "1" ]; then
rm "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service"
fi
unmount_image "${IMG_FILE}"

View File

@ -1,16 +1,20 @@
#!/bin/bash
# QCOW2 Routines
export CURRENT_IMAGE
export CURRENT_MOUNTPOINT
export NBD_DEV
export MAP_DEV
export MAP_BOOT_DEV
export MAP_ROOT_DEV
# set in build.sh
# should be fairly enough for the beginning
# overwrite here by uncommenting following lines
# BASE_QCOW2_SIZE=12G
# find and initialize free block device nodes
init_nbd() {
modprobe nbd max_part=16
if [ -z "${NBD_DEV}" ]; then
@ -18,20 +22,41 @@ init_nbd() {
S=`cat $x/size`
if [ "$S" == "0" ] ; then
NBD_DEV=/dev/$(basename $x)
MAP_DEV=/dev/mapper/$(basename $x)p1
MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1
MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2
break
fi
done
fi
}
export -f init_nbd
# mount qcow2 image: mount_image <image file> <mountpoint>
mount_qimage() {
# connect image to block device
connect_blkdev() {
init_nbd
qemu-nbd --discard=unmap -c $NBD_DEV "$1"
kpartx -a $NBD_DEV
mount $MAP_DEV "$2"
CURRENT_IMAGE="$1"
}
export -f connect_blkdev
# disconnect image from block device
disconnect_blkdev() {
kpartx -d $NBD_DEV
qemu-nbd -d $NBD_DEV
NBD_DEV=
MAP_BOOT_DEV=
MAP_ROOT_DEV=
CURRENT_IMAGE=
}
export -f disconnect_blkdev
# mount qcow2 image: mount_image <image file> <mountpoint>
mount_qimage() {
connect_blkdev "$1"
mount -v -t ext4 $MAP_ROOT_DEV "$2"
mkdir -p "${ROOTFS_DIR}/boot"
mount -v -t vfat $MAP_BOOT_DEV "$2/boot"
CURRENT_MOUNTPOINT="$2"
}
export -f mount_qimage
@ -39,16 +64,19 @@ export -f mount_qimage
# umount qcow2 image: umount_image <current mountpoint>
umount_qimage() {
sync
#umount "$1/boot"
while mount | grep -q "$1"; do
local LOCS
LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r)
for loc in $LOCS; do
echo "$loc"
umount "$loc"
while mountpoint -q "$loc" && ! umount "$loc"; do
sleep 0.1
done
done
kpartx -d $NBD_DEV
qemu-nbd -d $NBD_DEV
done
CURRENT_MOUNTPOINT=
disconnect_blkdev
}
export -f umount_qimage
@ -63,20 +91,29 @@ load_qimage() {
pushd ${WORK_DIR} > /dev/null
init_nbd
if [ -z "${PREV_STAGE}" ]; then
echo "Creating base image: image-${STAGE}.qcow2"
# -o preallocation=falloc
qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE
qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
echo 'type=83' | sfdisk $NBD_DEV
sfdisk $NBD_DEV << EOF
,250MiB,b
,,83;
EOF
kpartx -a $NBD_DEV
mkfs.ext4 $MAP_DEV
mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV
mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV
else
if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then exit 1; fi
echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2"
qemu-img create -f qcow2 \
-o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \
${WORK_DIR}/image-${STAGE}.qcow2
qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2
kpartx -a $NBD_DEV
fi
mount $MAP_DEV "${ROOTFS_DIR}"
mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}"
mkdir -p "${ROOTFS_DIR}/boot"
mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot"
CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2
CURRENT_MOUNTPOINT=${ROOTFS_DIR}
popd > /dev/null
@ -93,8 +130,104 @@ unload_qimage() {
if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then
fstrim -v "${CURRENT_MOUNTPOINT}" || true
umount_qimage "${CURRENT_MOUNTPOINT}"
CURRENT_IMAGE=""
CURRENT_MOUNTPOINT=""
fi
}
export -f unload_qimage
# based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer
# helper function for make_bootable_image, do not call directly
function resize_qcow2() {
if [ -z "$CALL_FROM_MBI" ]; then echo "resize_qcow2: cannot be called directly, use make_bootable_image instead"; return 1; fi
ROOT_MARGIN=$((800*1024*1024))
PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print`
PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '`
PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '`
e2fsck -y -f $MAP_ROOT_DEV || true
DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '`
BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }')
BLOCK_SIZE=${BLOCK_SIZE// /}
let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE
resize2fs -p $MAP_ROOT_DEV $DATA_SIZE
sleep 1
let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE
let PART_NEW_END=$PART_START+$PART_NEW_SIZE
ACT1=`parted -s "$NBD_DEV" rm 2`
ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END`
NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '`
}
export -f resize_qcow2
# create raw img from qcow2: make_bootable_image <in.qcow2> <out.img>
function make_bootable_image() {
EXPORT_QCOW2="$1"
EXPORT_IMAGE="$2"
echo "Connect block device to source qcow2"
connect_blkdev "${EXPORT_QCOW2}"
echo "Resize fs and partition"
CALL_FROM_MBI=1
resize_qcow2
CALL_FROM_MBI=
echo "Disconnect block device"
disconnect_blkdev
if [ -z "$NEW_IMG_SIZE" ]; then
echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit."
exit 1
fi
echo "Shrinking qcow2 image"
qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE
echo "Convert qcow2 to raw image"
qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}"
echo "Get PARTUUIDs from image"
IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")"
BOOT_PARTUUID="${IMGID}-01"
echo "Boot: $BOOT_PARTUUID"
ROOT_PARTUUID="${IMGID}-02"
echo "Root: $ROOT_PARTUUID"
echo "Mount image"
MOUNTROOT=${WORK_DIR}/tmpimage
mkdir -p $MOUNTROOT
MOUNTPT=$MOUNTROOT
PARTITION=2
mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
MOUNTPT=$MOUNTROOT/boot
PARTITION=1
mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
if [ ! -d "${MOUNTROOT}/root" ]; then
echo "Image damaged or not mounted. Exit."
exit 1
fi
echo "Setup PARTUUIDs"
if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then
echo "Set UUIDs to make it bootable"
sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt"
fi
echo "Umount image"
umount "${MOUNTROOT}/boot" || exit 1
umount "${MOUNTROOT}" || exit 1
echo "Remove qcow2 export image"
rm -f "${EXPORT_QCOW2}"
}
export -f make_bootable_image