From 3f2ff883c36a716bb5cf4aee25236af06b9e42d1 Mon Sep 17 00:00:00 2001 From: "Daniel F. Dickinson" Date: Thu, 16 Jan 2020 00:13:53 -0500 Subject: [PATCH 1/2] Allow customizing exports (images) and document There are use cases where pi-gen can be useful that need different image generation that the current defaults. Rather than force such users to require carrying modified versions of the default exports, allow users to specify alternate exports (e.g. instead of export-image and export-noobs, or in addition to these). We do this using a mechanism similar to STAGE_LIST, except that the lists of images are per-stage and default to paying attention to the presence of EXPORT_IMAGE and EXPORT_NOOBS as is currently the case. Signed-off-by: Daniel F. Dickinson --- README.md | 18 ++++++++++++++++++ build.sh | 44 ++++++++++++++++++++++++++++++++------------ scripts/common | 26 ++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 73d4961..a305dbd 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,24 @@ The following environment variables are supported: If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory. + * `stageX_EXPORT_LIST` (Default: based on presence for `EXPORT_IMAGE` and/or + `EXPORT_NOOBS` in stageX directory (currently stage2, stage4, and stage5) ) + + If set, then instead of determining images to export based on `EXPORT_IMAGE` + and/or `EXPORT_NOOBS` being present in a particular directory the images to + export for a particular stage is based on that stage's `stageX_EXPORT_LIST`. + Only stages with a `stageX_EXPORT_LIST` (e.g. `stage2_EXPORT_LIST`) are + determined this way. All other stages use the default method (presence of + `EXPORT_IMAGE` and/or `EXPORT_NOOBS`). + For stages with a `stageX_EXPORT_LIST`, the contents of EXPORT_LIST are + expected to be directory names in the repo directory. In addition, files + with each directory in EXPORT_LIST translated to upper case and dashes (-) + converted to underscores (\_) in the stage's directory are sourced before + generating the image. (e.g. an 'export-image' in the stage2_EXPORT_LIST + first sources stage2/EXPORT_IMAGE and then generates an images using the + `export-image` directory, which is the same behaviour as the old default + behaviour. + A simple example for building Raspbian: ```bash diff --git a/build.sh b/build.sh index 9e78728..1b264d6 100755 --- a/build.sh +++ b/build.sh @@ -87,8 +87,13 @@ run_stage(){ STAGE_WORK_DIR="${WORK_DIR}/${STAGE}" ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs if [ ! -f SKIP_IMAGES ]; then - if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then - EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" + if [ -z "${EXPORT_LIST}" ]; then + get_stage_export "${STAGE_DIR}" + if [ -n "${EXPORT_LIST}" ]; then + EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}" + fi + unset EXPORT_LIST + export EXPORT_LIST fi fi if [ ! -f SKIP ]; then @@ -180,6 +185,15 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}" export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"} +if [ "${USE_QEMU}" = "1" ]; then + # shellcheck disable=SC2034 + export stage2_EXPORT_LIST="${stage2_EXPORT_LIST:-export-image}" + # shellcheck disable=SC2034 + export stage4_EXPORT_LIST="${stage4_EXPORT_LIST:-export-image}" + # shellcheck disable=SC2034 + export stage5_EXPORT_LIST="${stage5_EXPORT_LIST:-export-image}" +fi + export CLEAN export IMG_NAME export APT_PROXY @@ -236,20 +250,26 @@ for STAGE_DIR in $STAGE_LIST; do done CLEAN=1 + for EXPORT_DIR in ${EXPORT_DIRS}; do - STAGE_DIR=${BASE_DIR}/export-image - # shellcheck source=/dev/null - source "${EXPORT_DIR}/EXPORT_IMAGE" - EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs - run_stage - if [ "${USE_QEMU}" != "1" ]; then - if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then + log "Begin export ${EXPORT_DIR}" + # e.g. stageX with default stage_dirs + get_stage_export "${EXPORT_DIR}" + + for EXPORT_IMAGE in ${EXPORT_LIST}; do + IMAGE_SOURCE="$(echo "${EXPORT_IMAGE}" | tr '[:lower:]' '[:upper:]' | tr '\-' '_')" + if [ -f "${EXPORT_DIR}/${IMAGE_SOURCE}" ]; then + log "Begin ${EXPORT_IMAGE}" + STAGE_DIR="${BASE_DIR}/${EXPORT_IMAGE}" # shellcheck source=/dev/null - source "${EXPORT_DIR}/EXPORT_NOOBS" - STAGE_DIR="${BASE_DIR}/export-noobs" + source "${EXPORT_DIR}/${IMAGE_SOURCE}" + # shellcheck disable=SC2153 + EXPORT_ROOTFS_DIR="${WORK_DIR}/${EXPORT_STAGE}/rootfs" run_stage + log "End ${EXPORT_IMAGE}" fi - fi + done + log "End export ${EXPORT_DIR}" done if [ -x ${BASE_DIR}/postrun.sh ]; then diff --git a/scripts/common b/scripts/common index e2048d9..9c3e4d5 100644 --- a/scripts/common +++ b/scripts/common @@ -98,3 +98,29 @@ update_issue() { echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using ${PI_GEN}, ${PI_GEN_REPO}, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue" } export -f update_issue + +get_stage_export() { + local stage_dir="$1" + local stage_var + stage_var="$(echo "$(basename "$1")"|tr '\- :/' '_')" + local export_list_var + + export_list_var="\$${stage_var}"_EXPORT_LIST + + eval EXPORT_LIST="$export_list_var" + + if [ -z "${EXPORT_LIST}" ]; then + if [ -e "$stage_dir"/EXPORT_IMAGE ]; then + EXPORT_LIST="export-image" + fi + if [ -e "$stage_dir"/EXPORT_NOOBS ]; then + EXPORT_LIST="${EXPORT_LIST} export-noobs" + fi + fi + + EXPORT_STAGE="$(basename "$1")" + export EXPORT_STAGE + export EXPORT_LIST +} + +export -f get_stage_export From af84080be323181aaab946b6495b415d213344c5 Mon Sep 17 00:00:00 2001 From: "Daniel F. Dickinson" Date: Thu, 16 Jan 2020 05:19:14 -0500 Subject: [PATCH 2/2] Add new export (image) with a swap partition Demo the new custom export capability with an new export type 'export-swappart-image' that uses a swap partition. Note that it really only makes sense if one tweaks stage2 to not include dphys-swapfile. Signed-off-by: Daniel F. Dickinson --- config-swappart.example | 8 +++ export-swappart-image/00-allow-rerun | 1 + export-swappart-image/01-set-sources | 1 + export-swappart-image/02-network | 1 + .../03-set-partuuid/00-run.sh | 16 +++++ .../03-set-partuuid/files/fstab | 4 ++ export-swappart-image/04-finalise | 1 + export-swappart-image/prerun.sh | 71 +++++++++++++++++++ stage2/EXPORT_SWAPPART_IMAGE | 4 ++ 9 files changed, 107 insertions(+) create mode 100755 config-swappart.example create mode 120000 export-swappart-image/00-allow-rerun create mode 120000 export-swappart-image/01-set-sources create mode 120000 export-swappart-image/02-network create mode 100644 export-swappart-image/03-set-partuuid/00-run.sh create mode 100644 export-swappart-image/03-set-partuuid/files/fstab create mode 120000 export-swappart-image/04-finalise create mode 100755 export-swappart-image/prerun.sh create mode 100644 stage2/EXPORT_SWAPPART_IMAGE diff --git a/config-swappart.example b/config-swappart.example new file mode 100755 index 0000000..028baa7 --- /dev/null +++ b/config-swappart.example @@ -0,0 +1,8 @@ +#!/bin/bash -e + +export IMG_NAME="swapport-test" +export stage2_EXPORT_LIST="export-swappart-image export-image export-noobs" +export stage4_EXPORT_LIST="export-swappart-image export-image export-noobs" +export stage5_EXPORT_LIST="export-swappart-image export-image export-noobs" +# 2GB Swap +export SWAP_SIZE=$(( 2048 * 1024 * 1024)) diff --git a/export-swappart-image/00-allow-rerun b/export-swappart-image/00-allow-rerun new file mode 120000 index 0000000..ae6d1a5 --- /dev/null +++ b/export-swappart-image/00-allow-rerun @@ -0,0 +1 @@ +../export-image/00-allow-rerun \ No newline at end of file diff --git a/export-swappart-image/01-set-sources b/export-swappart-image/01-set-sources new file mode 120000 index 0000000..3f62d00 --- /dev/null +++ b/export-swappart-image/01-set-sources @@ -0,0 +1 @@ +../export-image/01-set-sources \ No newline at end of file diff --git a/export-swappart-image/02-network b/export-swappart-image/02-network new file mode 120000 index 0000000..93688f2 --- /dev/null +++ b/export-swappart-image/02-network @@ -0,0 +1 @@ +../export-image/02-network \ No newline at end of file diff --git a/export-swappart-image/03-set-partuuid/00-run.sh b/export-swappart-image/03-set-partuuid/00-run.sh new file mode 100644 index 0000000..15390fb --- /dev/null +++ b/export-swappart-image/03-set-partuuid/00-run.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e + +IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + +IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')" + +BOOT_PARTUUID="${IMGID}-01" +SWAP_PARTUUID="${IMGID}-02" +ROOT_PARTUUID="${IMGID}-03" + +install -m 0644 files/fstab "${ROOTFS_DIR}/etc/fstab" +sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" +sed -i "s/SWAPDEV/PARTUUID=${SWAP_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" +sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab" + +sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt" diff --git a/export-swappart-image/03-set-partuuid/files/fstab b/export-swappart-image/03-set-partuuid/files/fstab new file mode 100644 index 0000000..b0de775 --- /dev/null +++ b/export-swappart-image/03-set-partuuid/files/fstab @@ -0,0 +1,4 @@ +proc /proc proc defaults 0 0 +BOOTDEV /boot vfat defaults 0 2 +ROOTDEV / ext4 defaults,noatime 0 1 +SWAPDEV none swap sw 0 0 diff --git a/export-swappart-image/04-finalise b/export-swappart-image/04-finalise new file mode 120000 index 0000000..4f18bf3 --- /dev/null +++ b/export-swappart-image/04-finalise @@ -0,0 +1 @@ +../export-image/04-finalise \ No newline at end of file diff --git a/export-swappart-image/prerun.sh b/export-swappart-image/prerun.sh new file mode 100755 index 0000000..eee6181 --- /dev/null +++ b/export-swappart-image/prerun.sh @@ -0,0 +1,71 @@ +#!/bin/bash -e + +IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img" + +unmount_image "${IMG_FILE}" + +rm -f "${IMG_FILE}" + +rm -rf "${ROOTFS_DIR}" +mkdir -p "${ROOTFS_DIR}" + +BOOT_SIZE="$((256 * 1024 * 1024))" +SWAP_SIZE="${SWAP_SIZE:-$((1024 * 1024 * 1024))}" +ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1) + +# All partition sizes and starts will be aligned to this size +ALIGN="$((4 * 1024 * 1024))" +# Add this much space to the calculated file size. This allows for +# some overhead (since actual space usage is usually rounded up to the +# filesystem block size) and gives some free space on the resulting +# image. +ROOT_MARGIN=$((800*1024*1024)) + +BOOT_PART_START=$((ALIGN)) +BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN)) +SWAP_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE)) +SWAP_PART_SIZE=$(((SWAP_SIZE + ALIGN - 1) / ALIGN * ALIGN)) +ROOT_PART_START=$((SWAP_PART_START + SWAP_PART_SIZE)) +ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN - 1) / ALIGN * ALIGN)) +IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + SWAP_PART_SIZE + ROOT_PART_SIZE)) + +truncate -s "${IMG_SIZE}" "${IMG_FILE}" + +parted --script "${IMG_FILE}" mklabel msdos +parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))" +parted --script "${IMG_FILE}" unit B mkpart primary linux-swap "${SWAP_PART_START}" "$((SWAP_PART_START + SWAP_PART_SIZE - 1))" +parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))" + +PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print) +BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B) +BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B) + +SWAP_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B) +SWAP_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B) + +ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^3:' | cut -d':' -f 2 | tr -d B) +ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^3:' | cut -d':' -f 4 | tr -d B) + +BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}") +SWAP_DEV=$(losetup --show -f -o "${SWAP_OFFSET}" --sizelimit "${SWAP_LENGTH}" "${IMG_FILE}") +ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}") +echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" +echo "swap: offset $SWAP_OFFSET, length $SWAP_LENGTH" +echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" + +ROOT_FEATURES="^huge_file" +for FEATURE in metadata_csum 64bit; do + if grep -q "$FEATURE" /etc/mke2fs.conf; then + ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES" + fi +done +mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null +mkswap "$SWAP_DEV" >/dev/null +mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null + +mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4 +mkdir -p "${ROOTFS_DIR}/boot" +mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat + +rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/" +rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/" diff --git a/stage2/EXPORT_SWAPPART_IMAGE b/stage2/EXPORT_SWAPPART_IMAGE new file mode 100644 index 0000000..a66d490 --- /dev/null +++ b/stage2/EXPORT_SWAPPART_IMAGE @@ -0,0 +1,4 @@ +IMG_SUFFIX="-swap-lite" +if [ "${USE_QEMU}" = "1" ]; then + export IMG_SUFFIX="${IMG_SUFFIX}-qemu" +fi