diff --git a/README.md b/README.md index 29d8166..0ef0825 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,24 @@ The following environment variables are supported: If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory. + * `stageX_EXPORT_LIST` (Default: based on presence for `EXPORT_IMAGE` and/or + `EXPORT_NOOBS` in stageX directory (currently stage2, stage4, and stage5) ) + + If set, then instead of determining images to export based on `EXPORT_IMAGE` + and/or `EXPORT_NOOBS` being present in a particular directory the images to + export for a particular stage is based on that stage's `stageX_EXPORT_LIST`. + Only stages with a `stageX_EXPORT_LIST` (e.g. `stage2_EXPORT_LIST`) are + determined this way. All other stages use the default method (presence of + `EXPORT_IMAGE` and/or `EXPORT_NOOBS`). + For stages with a `stageX_EXPORT_LIST`, the contents of EXPORT_LIST are + expected to be directory names in the repo directory. In addition, files + with each directory in EXPORT_LIST translated to upper case and dashes (-) + converted to underscores (\_) in the stage's directory are sourced before + generating the image. (e.g. an 'export-image' in the stage2_EXPORT_LIST + first sources stage2/EXPORT_IMAGE and then generates an images using the + `export-image` directory, which is the same behaviour as the old default + behaviour. + A simple example for building Raspbian: ```bash diff --git a/build.sh b/build.sh index a8247ab..88797a5 100755 --- a/build.sh +++ b/build.sh @@ -87,8 +87,13 @@ run_stage(){ STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs if [ ! -f SKIP_IMAGES ]; then - if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then - EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" + if [ -z "${EXPORT_LIST}" ]; then + get_stage_export "${STAGE_DIR}" + if [ -n "${EXPORT_LIST}" ]; then + EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" + fi + unset EXPORT_LIST + export EXPORT_LIST fi fi if [ ! -f SKIP ]; then @@ -181,6 +186,15 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}" export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"} +if [ "${USE_QEMU}" = "1" ]; then + # shellcheck disable=SC2034 + export stage2_EXPORT_LIST="${stage2_EXPORT_LIST:-export-image}" + # shellcheck disable=SC2034 + export stage4_EXPORT_LIST="${stage4_EXPORT_LIST:-export-image}" + # shellcheck disable=SC2034 + export stage5_EXPORT_LIST="${stage5_EXPORT_LIST:-export-image}" +fi + export CLEAN export IMG_NAME export APT_PROXY @@ -237,20 +251,26 @@ for STAGE_DIR in $STAGE_LIST; do done CLEAN=1 + for EXPORT_DIR in ${EXPORT_DIRS}; do - STAGE_DIR=${BASE_DIR}/export-image - # shellcheck source=/dev/null - source "${EXPORT_DIR}/EXPORT_IMAGE" - EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - run_stage - if [ "${USE_QEMU}" != "1" ]; then - if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then + log "Begin export ${EXPORT_DIR}" + # e.g. stageX with default stage_dirs + get_stage_export "${EXPORT_DIR}" + + for EXPORT_IMAGE in ${EXPORT_LIST}; do + IMAGE_SOURCE="$(echo "${EXPORT_IMAGE}" | tr '[:lower:]' '[:upper:]' | tr '\-' '_')" + if [ -f "${EXPORT_DIR}/${IMAGE_SOURCE}" ]; then + log "Begin ${EXPORT_IMAGE}" + STAGE_DIR="${BASE_DIR}/${EXPORT_IMAGE}" # shellcheck source=/dev/null - source "${EXPORT_DIR}/EXPORT_NOOBS" - STAGE_DIR="${BASE_DIR}/export-noobs" + source "${EXPORT_DIR}/${IMAGE_SOURCE}" + # shellcheck disable=SC2153 + EXPORT_ROOTFS_DIR="${WORK_DIR}/${EXPORT_STAGE}/rootfs" run_stage + log "End ${EXPORT_IMAGE}" fi - fi + done + log "End export ${EXPORT_DIR}" done if [ -x ${BASE_DIR}/postrun.sh ]; then diff --git a/config-swappart.example b/config-swappart.example new file mode 100755 index 0000000..028baa7 --- /dev/null +++ b/config-swappart.example @@ -0,0 +1,8 @@ +#!/bin/bash -e + +export IMG_NAME="swapport-test" +export stage2_EXPORT_LIST="export-swappart-image export-image export-noobs" +export stage4_EXPORT_LIST="export-swappart-image export-image export-noobs" +export stage5_EXPORT_LIST="export-swappart-image export-image export-noobs" +# 2GB Swap +export SWAP_SIZE=$(( 2048 * 1024 * 1024)) diff --git a/export-swappart-image/00-allow-rerun b/export-swappart-image/00-allow-rerun new file mode 120000 index 0000000..ae6d1a5 --- /dev/null +++ b/export-swappart-image/00-allow-rerun @@ -0,0 +1 @@ +../export-image/00-allow-rerun \ No newline at end of file diff --git a/export-swappart-image/01-set-sources b/export-swappart-image/01-set-sources new file mode 120000 index 0000000..3f62d00 --- /dev/null +++ b/export-swappart-image/01-set-sources @@ -0,0 +1 @@ +../export-image/01-set-sources \ No newline at end of file diff --git a/export-swappart-image/02-network b/export-swappart-image/02-network new file mode 120000 index 0000000..93688f2 --- /dev/null +++ b/export-swappart-image/02-network @@ -0,0 +1 @@ +../export-image/02-network \ No newline at end of file diff --git a/export-swappart-image/03-set-partuuid/00-run.sh b/export-swappart-image/03-set-partuuid/00-run.sh new file mode 100644 index 0000000..15390fb --- /dev/null +++ b/export-swappart-image/03-set-partuuid/00-run.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e + +IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + +IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" + +BOOT_PARTUUID="${IMGID}-01" +SWAP_PARTUUID="${IMGID}-02" +ROOT_PARTUUID="${IMGID}-03" + +install -m 0644 files/fstab "${ROOTFS_DIR}/etc/fstab" +sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" +sed -i "s/SWAPDEV/PARTUUID=${SWAP_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" +sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + +sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" diff --git a/export-swappart-image/03-set-partuuid/files/fstab b/export-swappart-image/03-set-partuuid/files/fstab new file mode 100644 index 0000000..b0de775 --- /dev/null +++ b/export-swappart-image/03-set-partuuid/files/fstab @@ -0,0 +1,4 @@ +proc /proc proc defaults 0 0 +BOOTDEV /boot vfat defaults 0 2 +ROOTDEV / ext4 defaults,noatime 0 1 +SWAPDEV none swap sw 0 0 diff --git a/export-swappart-image/04-finalise b/export-swappart-image/04-finalise new file mode 120000 index 0000000..4f18bf3 --- /dev/null +++ b/export-swappart-image/04-finalise @@ -0,0 +1 @@ +../export-image/04-finalise \ No newline at end of file diff --git a/export-swappart-image/prerun.sh b/export-swappart-image/prerun.sh new file mode 100755 index 0000000..eee6181 --- /dev/null +++ b/export-swappart-image/prerun.sh @@ -0,0 +1,71 @@ +#!/bin/bash -e + +IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + +unmount_image "${IMG_FILE}" + +rm -f "${IMG_FILE}" + +rm -rf "${ROOTFS_DIR}" +mkdir -p "${ROOTFS_DIR}" + +BOOT_SIZE="$((256 * 1024 * 1024))" +SWAP_SIZE="${SWAP_SIZE:-$((1024 * 1024 * 1024))}" +ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + +# All partition sizes and starts will be aligned to this size +ALIGN="$((4 * 1024 * 1024))" +# Add this much space to the calculated file size. This allows for +# some overhead (since actual space usage is usually rounded up to the +# filesystem block size) and gives some free space on the resulting +# image. +ROOT_MARGIN=$((800*1024*1024)) + +BOOT_PART_START=$((ALIGN)) +BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) +SWAP_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) +SWAP_PART_SIZE=$(((SWAP_SIZE + ALIGN - 1) / ALIGN * ALIGN)) +ROOT_PART_START=$((SWAP_PART_START + SWAP_PART_SIZE)) +ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) +IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + SWAP_PART_SIZE + ROOT_PART_SIZE)) + +truncate -s "${IMG_SIZE}" "${IMG_FILE}" + +parted --script "${IMG_FILE}" mklabel msdos +parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" +parted --script "${IMG_FILE}" unit B mkpart primary linux-swap "${SWAP_PART_START}" "$((SWAP_PART_START + SWAP_PART_SIZE - 1))" +parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" + +PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) +BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) +BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) + +SWAP_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) +SWAP_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) + +ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^3:' | cut -d':' -f 2 | tr -d B) +ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^3:' | cut -d':' -f 4 | tr -d B) + +BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") +SWAP_DEV=$(losetup --show -f -o "${SWAP_OFFSET}" --sizelimit "${SWAP_LENGTH}" "${IMG_FILE}") +ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") +echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" +echo "swap: offset $SWAP_OFFSET, length $SWAP_LENGTH" +echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + +ROOT_FEATURES="^huge_file" +for FEATURE in metadata_csum 64bit; do + if grep -q "$FEATURE" /etc/mke2fs.conf; then + ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" + fi +done +mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null +mkswap "$SWAP_DEV" >/dev/null +mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + +mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 +mkdir -p "${ROOTFS_DIR}/boot" +mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + +rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" +rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" diff --git a/scripts/common b/scripts/common index e2048d9..9c3e4d5 100644 --- a/scripts/common +++ b/scripts/common @@ -98,3 +98,29 @@ update_issue() { echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using ${PI_GEN}, ${PI_GEN_REPO}, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue" } export -f update_issue + +get_stage_export() { + local stage_dir="$1" + local stage_var + stage_var="$(echo "$(basename "$1")"|tr '\- :/' '_')" + local export_list_var + + export_list_var="\$${stage_var}"_EXPORT_LIST + + eval EXPORT_LIST="$export_list_var" + + if [ -z "${EXPORT_LIST}" ]; then + if [ -e "$stage_dir"/EXPORT_IMAGE ]; then + EXPORT_LIST="export-image" + fi + if [ -e "$stage_dir"/EXPORT_NOOBS ]; then + EXPORT_LIST="${EXPORT_LIST} export-noobs" + fi + fi + + EXPORT_STAGE="$(basename "$1")" + export EXPORT_STAGE + export EXPORT_LIST +} + +export -f get_stage_export diff --git a/stage2/EXPORT_SWAPPART_IMAGE b/stage2/EXPORT_SWAPPART_IMAGE new file mode 100644 index 0000000..a66d490 --- /dev/null +++ b/stage2/EXPORT_SWAPPART_IMAGE @@ -0,0 +1,4 @@ +IMG_SUFFIX="-swap-lite" +if [ "${USE_QEMU}" = "1" ]; then + export IMG_SUFFIX="${IMG_SUFFIX}-qemu" +fi