Merge af84080be3 into 08fc0b9a82

README.md

@@ -134,6 +134,24 @@ The following environment variables are supported:
 
     If set, then instead of working through the numeric stages in order, this list will be followed. For example setting to `"stage0 stage1 mystage stage2"` will run the contents of `mystage` before stage2. Note that quotes are needed around the list. An absolute or relative path can be given for stages outside the pi-gen directory.
 
+ * `stageX_EXPORT_LIST` (Default: based on presence for `EXPORT_IMAGE` and/or
+  `EXPORT_NOOBS` in stageX directory (currently stage2, stage4, and stage5) )
+
+    If set, then instead of determining images to export based on `EXPORT_IMAGE`
+    and/or `EXPORT_NOOBS` being present in a particular directory the images to
+    export for a particular stage is based on that stage's `stageX_EXPORT_LIST`.
+    Only stages with a `stageX_EXPORT_LIST` (e.g. `stage2_EXPORT_LIST`) are
+    determined this way.  All other stages use the default method (presence of
+    `EXPORT_IMAGE` and/or `EXPORT_NOOBS`).
+    For stages with a `stageX_EXPORT_LIST`, the contents of EXPORT_LIST are
+    expected to be directory names in the repo directory.  In addition, files
+    with each directory in EXPORT_LIST translated to upper case and dashes (-)
+    converted to underscores (\_) in the stage's directory are sourced before
+    generating the image.  (e.g. an 'export-image' in the stage2_EXPORT_LIST
+    first sources stage2/EXPORT_IMAGE and then generates an images using the
+    `export-image` directory, which is the same behaviour as the old default
+    behaviour.
+
 A simple example for building Raspbian:
 
 ```bash

build.sh

@@ -87,9 +87,14 @@ run_stage(){
 	STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
 	ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs
 	if [ ! -f SKIP_IMAGES ]; then
-		if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
+		if [ -z "${EXPORT_LIST}" ]; then
+			get_stage_export "${STAGE_DIR}"
+			if [ -n "${EXPORT_LIST}" ]; then
 				EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
 			fi
+			unset EXPORT_LIST
+			export EXPORT_LIST
+		fi
 	fi
 	if [ ! -f SKIP ]; then
 		if [ "${CLEAN}" = "1" ]; then
@@ -181,6 +186,15 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}"
 
 export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}
 
+if [ "${USE_QEMU}" = "1" ]; then
+	# shellcheck disable=SC2034
+	export stage2_EXPORT_LIST="${stage2_EXPORT_LIST:-export-image}"
+	# shellcheck disable=SC2034
+	export stage4_EXPORT_LIST="${stage4_EXPORT_LIST:-export-image}"
+	# shellcheck disable=SC2034
+	export stage5_EXPORT_LIST="${stage5_EXPORT_LIST:-export-image}"
+fi
+
 export CLEAN
 export IMG_NAME
 export APT_PROXY
@@ -237,20 +251,26 @@ for STAGE_DIR in $STAGE_LIST; do
 done
 
 CLEAN=1
+
 for EXPORT_DIR in ${EXPORT_DIRS}; do
-	STAGE_DIR=${BASE_DIR}/export-image
+	log "Begin export ${EXPORT_DIR}"
+	# e.g. stageX with default stage_dirs
+	get_stage_export "${EXPORT_DIR}"
+
+	for EXPORT_IMAGE in ${EXPORT_LIST}; do
+		IMAGE_SOURCE="$(echo "${EXPORT_IMAGE}" | tr '[:lower:]' '[:upper:]' | tr '\-' '_')"
+		if [ -f "${EXPORT_DIR}/${IMAGE_SOURCE}" ]; then
+			log "Begin ${EXPORT_IMAGE}"
+			STAGE_DIR="${BASE_DIR}/${EXPORT_IMAGE}"
 			# shellcheck source=/dev/null
-	source "${EXPORT_DIR}/EXPORT_IMAGE"
-	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
-	run_stage
-	if [ "${USE_QEMU}" != "1" ]; then
-		if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
-			# shellcheck source=/dev/null
-			source "${EXPORT_DIR}/EXPORT_NOOBS"
-			STAGE_DIR="${BASE_DIR}/export-noobs"
+			source "${EXPORT_DIR}/${IMAGE_SOURCE}"
+			# shellcheck disable=SC2153
+			EXPORT_ROOTFS_DIR="${WORK_DIR}/${EXPORT_STAGE}/rootfs"
 			run_stage
+			log "End ${EXPORT_IMAGE}"
 		fi
-	fi
+	done
+	log "End export ${EXPORT_DIR}"
 done
 
 if [ -x ${BASE_DIR}/postrun.sh ]; then

config-swappart.example

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+export IMG_NAME="swapport-test"
+export stage2_EXPORT_LIST="export-swappart-image export-image export-noobs"
+export stage4_EXPORT_LIST="export-swappart-image export-image export-noobs"
+export stage5_EXPORT_LIST="export-swappart-image export-image export-noobs"
+# 2GB Swap
+export SWAP_SIZE=$(( 2048 * 1024 * 1024))

export-swappart-image/00-allow-rerun

@@ -0,0 +1 @@
+../export-image/00-allow-rerun

export-swappart-image/01-set-sources

@@ -0,0 +1 @@
+../export-image/01-set-sources

export-swappart-image/02-network

@@ -0,0 +1 @@
+../export-image/02-network

export-swappart-image/03-set-partuuid/00-run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+
+IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+
+IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
+
+BOOT_PARTUUID="${IMGID}-01"
+SWAP_PARTUUID="${IMGID}-02"
+ROOT_PARTUUID="${IMGID}-03"
+
+install -m 0644 files/fstab "${ROOTFS_DIR}/etc/fstab"
+sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+sed -i "s/SWAPDEV/PARTUUID=${SWAP_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+
+sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"

export-swappart-image/03-set-partuuid/files/fstab

@@ -0,0 +1,4 @@
+proc            /proc           proc    defaults          0       0
+BOOTDEV  /boot           vfat    defaults          0       2
+ROOTDEV  /               ext4    defaults,noatime  0       1
+SWAPDEV  none            swap    sw                0       0

export-swappart-image/04-finalise

@@ -0,0 +1 @@
+../export-image/04-finalise

export-swappart-image/prerun.sh

@@ -0,0 +1,71 @@
+#!/bin/bash -e
+
+IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
+
+unmount_image "${IMG_FILE}"
+
+rm -f "${IMG_FILE}"
+
+rm -rf "${ROOTFS_DIR}"
+mkdir -p "${ROOTFS_DIR}"
+
+BOOT_SIZE="$((256 * 1024 * 1024))"
+SWAP_SIZE="${SWAP_SIZE:-$((1024 * 1024 * 1024))}"
+ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)
+
+# All partition sizes and starts will be aligned to this size
+ALIGN="$((4 * 1024 * 1024))"
+# Add this much space to the calculated file size. This allows for
+# some overhead (since actual space usage is usually rounded up to the
+# filesystem block size) and gives some free space on the resulting
+# image.
+ROOT_MARGIN=$((800*1024*1024))
+
+BOOT_PART_START=$((ALIGN))
+BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN))
+SWAP_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
+SWAP_PART_SIZE=$(((SWAP_SIZE + ALIGN  - 1) / ALIGN * ALIGN))
+ROOT_PART_START=$((SWAP_PART_START + SWAP_PART_SIZE))
+ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN  - 1) / ALIGN * ALIGN))
+IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + SWAP_PART_SIZE + ROOT_PART_SIZE))
+
+truncate -s "${IMG_SIZE}" "${IMG_FILE}"
+
+parted --script "${IMG_FILE}" mklabel msdos
+parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
+parted --script "${IMG_FILE}" unit B mkpart primary linux-swap "${SWAP_PART_START}" "$((SWAP_PART_START + SWAP_PART_SIZE - 1))"
+parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"
+
+PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print)
+BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B)
+BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B)
+
+SWAP_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B)
+SWAP_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B)
+
+ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^3:' | cut -d':' -f 2 | tr -d B)
+ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^3:' | cut -d':' -f 4 | tr -d B)
+
+BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}")
+SWAP_DEV=$(losetup --show -f -o "${SWAP_OFFSET}" --sizelimit "${SWAP_LENGTH}" "${IMG_FILE}")
+ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}")
+echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
+echo "swap:  offset $SWAP_OFFSET, length $SWAP_LENGTH"
+echo "/:     offset $ROOT_OFFSET, length $ROOT_LENGTH"
+
+ROOT_FEATURES="^huge_file"
+for FEATURE in metadata_csum 64bit; do
+	if grep -q "$FEATURE" /etc/mke2fs.conf; then
+	    ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
+	fi
+done
+mkdosfs -n boot -F 32 -v "$BOOT_DEV" > /dev/null
+mkswap "$SWAP_DEV" >/dev/null
+mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null
+
+mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
+mkdir -p "${ROOTFS_DIR}/boot"
+mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
+
+rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
+rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"

scripts/common

@@ -98,3 +98,29 @@ update_issue() {
 	echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using ${PI_GEN}, ${PI_GEN_REPO}, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue"
 }
 export -f update_issue
+
+get_stage_export() {
+	local stage_dir="$1"
+	local stage_var
+	stage_var="$(echo "$(basename "$1")"|tr '\- :/' '_')"
+	local export_list_var
+
+	export_list_var="\$${stage_var}"_EXPORT_LIST
+
+	eval EXPORT_LIST="$export_list_var"
+
+	if [ -z "${EXPORT_LIST}" ]; then
+		if [ -e "$stage_dir"/EXPORT_IMAGE ]; then
+			EXPORT_LIST="export-image"
+		fi
+		if [ -e "$stage_dir"/EXPORT_NOOBS ]; then
+			EXPORT_LIST="${EXPORT_LIST} export-noobs"
+		fi
+	fi
+
+  EXPORT_STAGE="$(basename "$1")"
+	export EXPORT_STAGE
+	export EXPORT_LIST
+}
+
+export -f get_stage_export

stage2/EXPORT_SWAPPART_IMAGE

@@ -0,0 +1,4 @@
+IMG_SUFFIX="-swap-lite"
+if [ "${USE_QEMU}" = "1" ]; then
+	export IMG_SUFFIX="${IMG_SUFFIX}-qemu"
+fi