diff --git a/build.sh b/build.sh index 3409ba4..c00555a 100755 --- a/build.sh +++ b/build.sh @@ -14,7 +14,7 @@ $(cat "${i}-debconf") SELEOF EOF - log "End ${SUB_STAGE_DIR}/${i}-debconf" + log "End ${SUB_STAGE_DIR}/${i}-debconf" fi if [ -f "${i}-packages-nr" ]; then log "Begin ${SUB_STAGE_DIR}/${i}-packages-nr" @@ -23,11 +23,11 @@ EOF on_chroot << EOF apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES EOF - if [ "${USE_QCOW2}" = "1" ]; then - on_chroot << EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF apt-get clean EOF - fi + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages-nr" fi @@ -38,11 +38,11 @@ EOF on_chroot << EOF apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES EOF - if [ "${USE_QCOW2}" = "1" ]; then - on_chroot << EOF + if [ "${USE_QCOW2}" = "1" ]; then + on_chroot << EOF apt-get clean EOF - fi + fi fi log "End ${SUB_STAGE_DIR}/${i}-packages" fi @@ -100,14 +100,14 @@ run_stage(){ ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs if [ "${USE_QCOW2}" = "1" ]; then - if [ ! -f SKIP ]; then - load_qimage - fi + if [ ! -f SKIP ]; then + load_qimage + fi else - # make sure we are not umounting during export-image stage - if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then - unmount "${WORK_DIR}/${STAGE}" - fi + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi fi if [ ! -f SKIP_IMAGES ]; then @@ -127,8 +127,7 @@ run_stage(){ log "End ${STAGE_DIR}/prerun.sh" fi for SUB_STAGE_DIR in "${STAGE_DIR}"/*; do - if [ -d "${SUB_STAGE_DIR}" ] && - [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then + if [ -d "${SUB_STAGE_DIR}" ] && [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then run_sub_stage fi done @@ -137,10 +136,10 @@ run_stage(){ if [ "${USE_QCOW2}" = "1" ]; then unload_qimage else - # make sure we are not umounting during export-image stage - if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then - unmount "${WORK_DIR}/${STAGE}" - fi + # make sure we are not umounting during export-image stage + if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then + unmount "${WORK_DIR}/${STAGE}" + fi fi PREV_STAGE="${STAGE}" @@ -177,10 +176,10 @@ do done term() { - if [ "${USE_QCOW2}" = "1" ]; then - log "Unloading image" - unload_qimage - fi + if [ "${USE_QCOW2}" = "1" ]; then + log "Unloading image" + unload_qimage + fi } trap term EXIT INT TERM @@ -255,9 +254,9 @@ export USE_QCOW2="${USE_QCOW2:-1}" export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G} source "${SCRIPT_DIR}/qcow2_handling" if [ "${USE_QCOW2}" = "1" ]; then - NO_PRERUN_QCOW2=1 + NO_PRERUN_QCOW2=1 else - NO_PRERUN_QCOW2=0 + NO_PRERUN_QCOW2=0 fi export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}" @@ -297,71 +296,71 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do source "${EXPORT_DIR}/EXPORT_IMAGE" EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs if [ "${USE_QCOW2}" = "1" ]; then - USE_QCOW2=0 - EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}" - echo "------------------------------------------------------------------------" - echo "Running export stage for ${EXPORT_NAME}" - rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true - rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true - rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true - rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true - EXPORT_STAGE=$(basename "${EXPORT_DIR}") - for s in $STAGE_LIST; do - TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}") - done - FIRST_STAGE=${TMP_LIST%% *} - FIRST_IMAGE="image-${FIRST_STAGE}.qcow2" + USE_QCOW2=0 + EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}" + echo "------------------------------------------------------------------------" + echo "Running export stage for ${EXPORT_NAME}" + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true + rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true + EXPORT_STAGE=$(basename "${EXPORT_DIR}") + for s in $STAGE_LIST; do + TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}") + done + FIRST_STAGE=${TMP_LIST%% *} + FIRST_IMAGE="image-${FIRST_STAGE}.qcow2" - pushd "${WORK_DIR}" > /dev/null - echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}" - cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2" + pushd "${WORK_DIR}" > /dev/null + echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}" + cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2" - ARR=($TMP_LIST) - # rebase stage images to new export base - for CURR_STAGE in "${ARR[@]}"; do - if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then - PREV_IMG="${EXPORT_NAME}" - continue - fi + ARR=($TMP_LIST) + # rebase stage images to new export base + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="${EXPORT_NAME}" + continue + fi echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" - qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 - if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then - break - fi - PREV_IMG="image-${CURR_STAGE}" - done + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done - # commit current export stage into base export image - echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2" - qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2 + # commit current export stage into base export image + echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2" + qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2 - # rebase stage images back to original first stage for easy re-run - for CURR_STAGE in "${ARR[@]}"; do - if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then - PREV_IMG="image-${CURR_STAGE}" - continue - fi + # rebase stage images back to original first stage for easy re-run + for CURR_STAGE in "${ARR[@]}"; do + if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then + PREV_IMG="image-${CURR_STAGE}" + continue + fi echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2" - qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 - if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then - break - fi - PREV_IMG="image-${CURR_STAGE}" - done - popd > /dev/null + qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2 + if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then + break + fi + PREV_IMG="image-${CURR_STAGE}" + done + popd > /dev/null - mkdir -p "${WORK_DIR}/export-image/rootfs" - mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/" - echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs" - mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs" + mkdir -p "${WORK_DIR}/export-image/rootfs" + mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/" + echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs" + mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs" - CLEAN=0 - run_stage - CLEAN=1 - USE_QCOW2=1 + CLEAN=0 + run_stage + CLEAN=1 + USE_QCOW2=1 else - run_stage + run_stage fi if [ "${USE_QEMU}" != "1" ]; then if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then @@ -369,11 +368,11 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do source "${EXPORT_DIR}/EXPORT_NOOBS" STAGE_DIR="${BASE_DIR}/export-noobs" if [ "${USE_QCOW2}" = "1" ]; then - USE_QCOW2=0 - run_stage - USE_QCOW2=1 + USE_QCOW2=0 + run_stage + USE_QCOW2=1 else - run_stage + run_stage fi fi fi @@ -387,7 +386,7 @@ if [ -x postrun.sh ]; then fi if [ "${USE_QCOW2}" = "1" ]; then - unload_qimage + unload_qimage fi log "End ${BASE_DIR}" diff --git a/export-image/03-set-partuuid/00-run.sh b/export-image/03-set-partuuid/00-run.sh index cf5c0db..16e1b15 100755 --- a/export-image/03-set-partuuid/00-run.sh +++ b/export-image/03-set-partuuid/00-run.sh @@ -2,17 +2,17 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then - IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" - IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" + IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" - BOOT_PARTUUID="${IMGID}-01" - ROOT_PARTUUID="${IMGID}-02" + BOOT_PARTUUID="${IMGID}-01" + ROOT_PARTUUID="${IMGID}-02" - sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" fi diff --git a/export-image/04-finalise/01-run.sh b/export-image/04-finalise/01-run.sh index 625bbd1..9afa115 100755 --- a/export-image/04-finalise/01-run.sh +++ b/export-image/04-finalise/01-run.sh @@ -85,15 +85,15 @@ rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" mv "$INFO_FILE" "$DEPLOY_DIR/" if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then - ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" + ROOT_DEV="$(mount | grep "${ROOTFS_DIR} " | cut -f1 -d' ')" - unmount "${ROOTFS_DIR}" - zerofree "${ROOT_DEV}" + unmount "${ROOTFS_DIR}" + zerofree "${ROOT_DEV}" - unmount_image "${IMG_FILE}" + unmount_image "${IMG_FILE}" else - unload_qimage - make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" + unload_qimage + make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE" fi if [ "${DEPLOY_ZIP}" == "1" ]; then diff --git a/export-image/prerun.sh b/export-image/prerun.sh index 4d5f651..efd175a 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -1,63 +1,63 @@ #!/bin/bash -e if [ "${NO_PRERUN_QCOW2}" = "0" ]; then - IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" - unmount_image "${IMG_FILE}" + unmount_image "${IMG_FILE}" - rm -f "${IMG_FILE}" + rm -f "${IMG_FILE}" - rm -rf "${ROOTFS_DIR}" - mkdir -p "${ROOTFS_DIR}" + rm -rf "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}" - BOOT_SIZE="$((256 * 1024 * 1024))" - ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + BOOT_SIZE="$((256 * 1024 * 1024))" + ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) - # All partition sizes and starts will be aligned to this size - ALIGN="$((4 * 1024 * 1024))" - # Add this much space to the calculated file size. This allows for - # some overhead (since actual space usage is usually rounded up to the - # filesystem block size) and gives some free space on the resulting - # image. - ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" + # All partition sizes and starts will be aligned to this size + ALIGN="$((4 * 1024 * 1024))" + # Add this much space to the calculated file size. This allows for + # some overhead (since actual space usage is usually rounded up to the + # filesystem block size) and gives some free space on the resulting + # image. + ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)" - BOOT_PART_START=$((ALIGN)) - BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) - ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) - ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) - IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) + BOOT_PART_START=$((ALIGN)) + BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) + ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) + ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) + IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE)) - truncate -s "${IMG_SIZE}" "${IMG_FILE}" + truncate -s "${IMG_SIZE}" "${IMG_FILE}" - parted --script "${IMG_FILE}" mklabel msdos - parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" - parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" mklabel msdos + parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" + parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" - PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) - BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) - BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) + PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) + BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) + BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) - ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) - ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) + ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) + ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) - BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") - ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") - echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" - echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") + ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") + echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" + echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" - ROOT_FEATURES="^huge_file" - for FEATURE in metadata_csum 64bit; do + ROOT_FEATURES="^huge_file" + for FEATURE in metadata_csum 64bit; do if grep -q "$FEATURE" /etc/mke2fs.conf; then - ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" + ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" fi - done - mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null - mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + done + mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null + mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null - mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 - mkdir -p "${ROOTFS_DIR}/boot" - mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 + mkdir -p "${ROOTFS_DIR}/boot" + mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat - rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" - rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" + rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" + rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" fi diff --git a/export-noobs/00-release/00-run.sh b/export-noobs/00-release/00-run.sh index 8d54eb7..216096b 100755 --- a/export-noobs/00-release/00-run.sh +++ b/export-noobs/00-release/00-run.sh @@ -41,7 +41,7 @@ sed "${NOOBS_DIR}/os.json" -i -e "s|RELEASE|${RELEASE}|" sed "${NOOBS_DIR}/release_notes.txt" -i -e "s|UNRELEASED|${IMG_DATE}|" if [ "${USE_QCOW2}" = "1" ]; then - mv "${NOOBS_DIR}" "${DEPLOY_DIR}/" + mv "${NOOBS_DIR}" "${DEPLOY_DIR}/" else - cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" + cp -a "${NOOBS_DIR}" "${DEPLOY_DIR}/" fi diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 905bfb2..b7f2b4a 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -4,9 +4,9 @@ NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}" mkdir -p "${STAGE_WORK_DIR}" if [ "${DEPLOY_ZIP}" == "1" ]; then - IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" + IMG_FILE="${WORK_DIR}/export-image/${IMG_FILENAME}${IMG_SUFFIX}.img" else - IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + IMG_FILE="${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" fi unmount_image "${IMG_FILE}" @@ -38,7 +38,7 @@ umount "${STAGE_WORK_DIR}/rootfs/boot" bsdtar --numeric-owner --format gnutar -C "${STAGE_WORK_DIR}/rootfs" --one-file-system -cpf - . | xz -T0 > "${NOOBS_DIR}/root.tar.xz" if [ "${USE_QCOW2}" = "1" ]; then - rm "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service" + rm "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service" fi unmount_image "${IMG_FILE}" diff --git a/imagetool.sh b/imagetool.sh index b0ebb00..adc1ab7 100755 --- a/imagetool.sh +++ b/imagetool.sh @@ -1,15 +1,15 @@ #!/bin/bash if [ "$(id -u)" != "0" ]; then - echo "Please run as root" 1>&2 - exit 1 + echo "Please run as root" 1>&2 + exit 1 fi progname=$(basename $0) function usage() { - cat << HEREDOC + cat << HEREDOC Usage: Mount Image : $progname [--mount] [--image-name ] [--mount-point ] @@ -36,77 +36,77 @@ IMAGE="" MOUNTPOINT="" nbd_cleanup() { - DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)" - if [ ! -z "${DEVS}" ]; then - for d in $DEVS; do - if [ ! -z "${d}" ]; then - QDEV="$(ps xa | grep $d | grep -v grep)" - if [ -z "${QDEV}" ]; then - kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d" - fi - fi - done - fi + DEVS="$(lsblk | grep nbd | grep disk | cut -d" " -f1)" + if [ ! -z "${DEVS}" ]; then + for d in $DEVS; do + if [ ! -z "${d}" ]; then + QDEV="$(ps xa | grep $d | grep -v grep)" + if [ -z "${QDEV}" ]; then + kpartx -d /dev/$d && echo "Unconnected device map removed: /dev/$d" + fi + fi + done + fi } # As long as there is at least one more argument, keep looping while [[ $# -gt 0 ]]; do - key="$1" - case "$key" in - -h|--help) - usage - exit - ;; - -c|--cleanup) - nbd_cleanup - ;; - -m|--mount) - MOUNT=1 - ;; - -u|--umount) - UMOUNT=1 - ;; - -i|--image-name) - shift - IMAGE="$1" - ;; - -p|--mount-point) - shift - MOUNTPOINT="$1" - ;; - *) - echo "Unknown option '$key'" - usage - exit - ;; - esac - # Shift after checking all the cases to get the next option - shift + key="$1" + case "$key" in + -h|--help) + usage + exit + ;; + -c|--cleanup) + nbd_cleanup + ;; + -m|--mount) + MOUNT=1 + ;; + -u|--umount) + UMOUNT=1 + ;; + -i|--image-name) + shift + IMAGE="$1" + ;; + -p|--mount-point) + shift + MOUNTPOINT="$1" + ;; + *) + echo "Unknown option '$key'" + usage + exit + ;; + esac + # Shift after checking all the cases to get the next option + shift done if [ "${MOUNT}" = "1" ] && [ "${UMOUNT}" = "1" ]; then - usage - echo "Concurrent mount options not possible." - exit + usage + echo "Concurrent mount options not possible." + exit fi if [ "${MOUNT}" = "1" ] && ([ -z "${IMAGE}" ] || [ -z "${MOUNTPOINT}" ]); then - usage - echo "Can not mount image. Image path and/or mount point missing." - exit + usage + echo "Can not mount image. Image path and/or mount point missing." + exit fi if [ "${UMOUNT}" = "1" ] && [ -z "${MOUNTPOINT}" ]; then - usage - echo "Can not umount. Mount point parameter missing." - exit + usage + echo "Can not umount. Mount point parameter missing." + exit fi export NBD_DEV="${NBD_DEV:-/dev/nbd1}" source scripts/qcow2_handling if [ "${MOUNT}" = "1" ]; then - mount_qimage "${MOUNTPOINT}" "${IMAGE}" + mount_qimage "${MOUNTPOINT}" "${IMAGE}" elif [ "${UMOUNT}" = "1" ]; then - umount_qimage "${MOUNTPOINT}" + umount_qimage "${MOUNTPOINT}" fi diff --git a/scripts/qcow2_handling b/scripts/qcow2_handling index 7057dd4..f5528f2 100644 --- a/scripts/qcow2_handling +++ b/scripts/qcow2_handling @@ -16,230 +16,241 @@ export MAP_ROOT_DEV # find and initialize free block device nodes init_nbd() { - modprobe nbd max_part=16 - if [ -z "${NBD_DEV}" ]; then + modprobe nbd max_part=16 + if [ -z "${NBD_DEV}" ]; then for x in /sys/class/block/nbd* ; do - S=`cat $x/size` - if [ "$S" == "0" ] ; then - NBD_DEV=/dev/$(basename $x) - MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1 - MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2 - break - fi + S=`cat $x/size` + if [ "$S" == "0" ] ; then + NBD_DEV=/dev/$(basename $x) + MAP_BOOT_DEV=/dev/mapper/$(basename $x)p1 + MAP_ROOT_DEV=/dev/mapper/$(basename $x)p2 + break + fi done - fi + fi } export -f init_nbd # connect image to block device connect_blkdev() { - init_nbd - qemu-nbd --discard=unmap -c $NBD_DEV "$1" - sync - kpartx -a $NBD_DEV - sync - CURRENT_IMAGE="$1" + init_nbd + qemu-nbd --discard=unmap -c $NBD_DEV "$1" + sync + kpartx -a $NBD_DEV + sync + CURRENT_IMAGE="$1" } export -f connect_blkdev # disconnect image from block device disconnect_blkdev() { - kpartx -d $NBD_DEV - qemu-nbd -d $NBD_DEV - NBD_DEV= - MAP_BOOT_DEV= - MAP_ROOT_DEV= - CURRENT_IMAGE= + kpartx -d $NBD_DEV + qemu-nbd -d $NBD_DEV + NBD_DEV= + MAP_BOOT_DEV= + MAP_ROOT_DEV= + CURRENT_IMAGE= } export -f disconnect_blkdev # mount qcow2 image: mount_image mount_qimage() { - connect_blkdev "$1" - mount -v -t ext4 $MAP_ROOT_DEV "$2" - mkdir -p "${ROOTFS_DIR}/boot" - mount -v -t vfat $MAP_BOOT_DEV "$2/boot" - CURRENT_MOUNTPOINT="$2" + connect_blkdev "$1" + mount -v -t ext4 $MAP_ROOT_DEV "$2" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "$2/boot" + CURRENT_MOUNTPOINT="$2" } export -f mount_qimage # umount qcow2 image: umount_image umount_qimage() { - sync - #umount "$1/boot" - while mount | grep -q "$1"; do - local LOCS - LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) - for loc in $LOCS; do - echo "$loc" - while mountpoint -q "$loc" && ! umount "$loc"; do - sleep 0.1 + sync + #umount "$1/boot" + while mount | grep -q "$1"; do + local LOCS + LOCS=$(mount | grep "$1" | cut -f 3 -d ' ' | sort -r) + for loc in $LOCS; do + echo "$loc" + while mountpoint -q "$loc" && ! umount "$loc"; do + sleep 0.1 + done done - done - done - CURRENT_MOUNTPOINT= - disconnect_blkdev + done + CURRENT_MOUNTPOINT= + disconnect_blkdev } export -f umount_qimage # create base image / backing image / mount image load_qimage() { - if [ -z "${CURRENT_MOUNTPOINT}" ]; then - if [ ! -d "${ROOTFS_DIR}" ]; then mkdir -p "${ROOTFS_DIR}"; fi + if [ -z "${CURRENT_MOUNTPOINT}" ]; then + if [ ! -d "${ROOTFS_DIR}" ]; then + mkdir -p "${ROOTFS_DIR}"; + fi - if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; fi + if [ "${CLEAN}" = "1" ] && [ -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + rm -f "${WORK_DIR}/image-${STAGE}.qcow2"; + fi - if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then - pushd ${WORK_DIR} > /dev/null - init_nbd - if [ -z "${PREV_STAGE}" ]; then - echo "Creating base image: image-${STAGE}.qcow2" - # -o preallocation=falloc - qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE - sync - qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 - sync - sfdisk $NBD_DEV << EOF -,250MiB,b + if [ ! -f "${WORK_DIR}/image-${STAGE}.qcow2" ]; then + pushd ${WORK_DIR} > /dev/null + init_nbd + if [ -z "${PREV_STAGE}" ]; then + echo "Creating base image: image-${STAGE}.qcow2" + # -o preallocation=falloc + qemu-img create -f qcow2 image-${STAGE}.qcow2 $BASE_QCOW2_SIZE + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + sfdisk $NBD_DEV << EOF +4,250MiB,b,* ,,83; EOF - sync - kpartx -a $NBD_DEV - mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV - mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV - sync - else - if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then exit 1; fi - echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2" - qemu-img create -f qcow2 \ - -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ - ${WORK_DIR}/image-${STAGE}.qcow2 - sync - qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 - sync - kpartx -a $NBD_DEV - fi - mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" - mkdir -p "${ROOTFS_DIR}/boot" - mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot" - CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 - CURRENT_MOUNTPOINT=${ROOTFS_DIR} - popd > /dev/null - else - mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" - fi - echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" - fi + sync + kpartx -a $NBD_DEV + mkdosfs -n boot -F 32 -v $MAP_BOOT_DEV + mkfs.ext4 -L rootfs -O "^huge_file,^metadata_csum,^64bit" $MAP_ROOT_DEV + sync + else + if [ ! -f "${WORK_DIR}/image-${PREV_STAGE}.qcow2" ]; then + exit 1; + fi + echo "Creating backing image: image-${STAGE}.qcow2 <- ${WORK_DIR}/image-${PREV_STAGE}.qcow2" + qemu-img create -f qcow2 \ + -o backing_file=${WORK_DIR}/image-${PREV_STAGE}.qcow2 \ + ${WORK_DIR}/image-${STAGE}.qcow2 + sync + qemu-nbd --discard=unmap -c $NBD_DEV image-${STAGE}.qcow2 + sync + kpartx -a $NBD_DEV + fi + + mount -v -t ext4 $MAP_ROOT_DEV "${ROOTFS_DIR}" + mkdir -p "${ROOTFS_DIR}/boot" + mount -v -t vfat $MAP_BOOT_DEV "${ROOTFS_DIR}/boot" + CURRENT_IMAGE=${WORK_DIR}/image-${STAGE}.qcow2 + CURRENT_MOUNTPOINT=${ROOTFS_DIR} + popd > /dev/null + else + mount_qimage "${WORK_DIR}/image-${STAGE}.qcow2" "${ROOTFS_DIR}" + fi + echo "Current image in use: ${CURRENT_IMAGE} (MP: ${CURRENT_MOUNTPOINT})" + fi } export -f load_qimage # umount current image and refresh mount point env var unload_qimage() { - if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then - fstrim -v "${CURRENT_MOUNTPOINT}" || true - umount_qimage "${CURRENT_MOUNTPOINT}" - fi + if [ ! -z "${CURRENT_MOUNTPOINT}" ]; then + fstrim -v "${CURRENT_MOUNTPOINT}" || true + umount_qimage "${CURRENT_MOUNTPOINT}" + fi } export -f unload_qimage # based on: https://github.com/SirLagz/RaspberryPi-ImgAutoSizer # helper function for make_bootable_image, do not call directly function resize_qcow2() { - if [ -z "$CALL_FROM_MBI" ]; then echo "resize_qcow2: cannot be called directly, use make_bootable_image instead"; return 1; fi + if [ -z "$CALL_FROM_MBI" ]; then + echo "resize_qcow2: cannot be called directly, use make_bootable_image instead" + return 1 + fi - ROOT_MARGIN=$((800*1024*1024)) - PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print` - PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '` - PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '` + # ROOT_MARGIN=$((800*1024*1024)) + ROOT_MARGIN=$((1*1024*1024)) + PARTED_OUT=`parted -s -m "$NBD_DEV" unit B print` + PART_NO=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print $1 } '` + PART_START=`echo "$PARTED_OUT" | grep ext4 | awk -F: ' { print substr($2,1,length($2)-1) } '` - e2fsck -y -f $MAP_ROOT_DEV || true + e2fsck -y -f $MAP_ROOT_DEV || true - DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '` - BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }') - BLOCK_SIZE=${BLOCK_SIZE// /} + DATA_SIZE=`resize2fs -P $MAP_ROOT_DEV | awk -F': ' ' { print $2 } '` + BLOCK_SIZE=$(dumpe2fs -h $MAP_ROOT_DEV | grep 'Block size' | awk -F': ' ' { print $2 }') + BLOCK_SIZE=${BLOCK_SIZE// /} - let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE - resize2fs -p $MAP_ROOT_DEV $DATA_SIZE - sleep 1 + let DATA_SIZE=$DATA_SIZE+$ROOT_MARGIN/$BLOCK_SIZE + resize2fs -p $MAP_ROOT_DEV $DATA_SIZE + sleep 1 - let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE - let PART_NEW_END=$PART_START+$PART_NEW_SIZE - ACT1=`parted -s "$NBD_DEV" rm 2` - ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END` - NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '` + let PART_NEW_SIZE=$DATA_SIZE*$BLOCK_SIZE + let PART_NEW_END=$PART_START+$PART_NEW_SIZE + ACT1=`parted -s "$NBD_DEV" rm 2` + ACT2=`parted -s "$NBD_DEV" unit B mkpart primary $PART_START $PART_NEW_END` + NEW_IMG_SIZE=`parted -s -m "$NBD_DEV" unit B print free | tail -1 | awk -F: ' { print substr($2,1,length($2)-1) } '` } export -f resize_qcow2 # create raw img from qcow2: make_bootable_image function make_bootable_image() { - EXPORT_QCOW2="$1" - EXPORT_IMAGE="$2" + EXPORT_QCOW2="$1" + EXPORT_IMAGE="$2" - echo "Connect block device to source qcow2" - connect_blkdev "${EXPORT_QCOW2}" + echo "Connect block device to source qcow2" + connect_blkdev "${EXPORT_QCOW2}" - echo "Resize fs and partition" - CALL_FROM_MBI=1 - resize_qcow2 - sync - CALL_FROM_MBI= + echo "Resize fs and partition" + CALL_FROM_MBI=1 + resize_qcow2 + sync + CALL_FROM_MBI= - echo "Disconnect block device" - disconnect_blkdev + echo "Disconnect block device" + disconnect_blkdev - if [ -z "$NEW_IMG_SIZE" ]; then - echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit." - exit 1 - fi + if [ -z "$NEW_IMG_SIZE" ]; then + echo "NEW_IMG_SIZE could not be calculated, cannot process image. Exit." + exit 1 + fi - echo "Shrinking qcow2 image" - qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE - sync + echo "Shrinking qcow2 image" + qemu-img resize --shrink "${EXPORT_QCOW2}" $NEW_IMG_SIZE + sync - echo "Convert qcow2 to raw image" - qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}" - sync + echo "Convert qcow2 to raw image" + qemu-img convert -f qcow2 -O raw "${EXPORT_QCOW2}" "${EXPORT_IMAGE}" + sync - echo "Get PARTUUIDs from image" - IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")" + echo "Get PARTUUIDs from image" + IMGID="$(blkid -o value -s PTUUID "${EXPORT_IMAGE}")" - BOOT_PARTUUID="${IMGID}-01" - echo "Boot: $BOOT_PARTUUID" - ROOT_PARTUUID="${IMGID}-02" - echo "Root: $ROOT_PARTUUID" + BOOT_PARTUUID="${IMGID}-01" + echo "Boot: $BOOT_PARTUUID" + ROOT_PARTUUID="${IMGID}-02" + echo "Root: $ROOT_PARTUUID" - echo "Mount image" - MOUNTROOT=${WORK_DIR}/tmpimage - mkdir -p $MOUNTROOT + echo "Mount image" + MOUNTROOT=${WORK_DIR}/tmpimage + mkdir -p $MOUNTROOT - MOUNTPT=$MOUNTROOT - PARTITION=2 - mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + MOUNTPT=$MOUNTROOT + PARTITION=2 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 - MOUNTPT=$MOUNTROOT/boot - PARTITION=1 - mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 + MOUNTPT=$MOUNTROOT/boot + PARTITION=1 + mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1 - if [ ! -d "${MOUNTROOT}/root" ]; then - echo "Image damaged or not mounted. Exit." - exit 1 - fi + if [ ! -d "${MOUNTROOT}/root" ]; then + echo "Image damaged or not mounted. Exit." + exit 1 + fi - echo "Setup PARTUUIDs" - if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then - echo "Set UUIDs to make it bootable" - sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" - sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt" - fi + echo "Setup PARTUUIDs" + if [ ! -z "$BOOT_PARTUUID" ] && [ ! -z "$ROOT_PARTUUID" ]; then + echo "Set UUIDs to make it bootable" + sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab" + sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt" + fi - echo "Umount image" - sync - umount "${MOUNTROOT}/boot" || exit 1 - umount "${MOUNTROOT}" || exit 1 + echo "Umount image" + sync + umount "${MOUNTROOT}/boot" || exit 1 + umount "${MOUNTROOT}" || exit 1 - echo "Remove qcow2 export image" - rm -f "${EXPORT_QCOW2}" + echo "Remove qcow2 export image" + rm -f "${EXPORT_QCOW2}" } export -f make_bootable_image