From 3d61bbfb710f9d51bd78f86e43e96957bf99a3eb Mon Sep 17 00:00:00 2001 From: Claus Strasburger Date: Mon, 16 Jan 2017 14:04:56 +0100 Subject: [PATCH] Added Docker support - replaced necessity for devicemapper (through kpartx) by using parted and losetup with offsets - added Dockerfile - added dependency for parted and grep - added hints to README.md - common: loop through unmounts, fix shellcheck warnings --- .dockerignore | 3 ++ Dockerfile | 12 +++++++ README.md | 18 +++++++++-- build-docker.sh | 73 ++++++++++++++++++++++++++++++++++++++++++ depends | 6 +++- export-image/prerun.sh | 18 +++++++++-- export-noobs/prerun.sh | 18 +++++++++-- scripts/common | 63 ++++++++++++++++++++---------------- 8 files changed, 173 insertions(+), 38 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile mode change 100644 => 100755 README.md create mode 100755 build-docker.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4b019f9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +output/ +work/ +deploy/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c442b6d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM debian:jessie + +RUN apt-get -y update && \ + apt-get -y install \ + git vim parted \ + quilt realpath qemu-user-static debootstrap zerofree pxz zip dosfstools \ + bsdtar libcap2-bin rsync grep \ + && rm -rf /var/lib/apt/lists/* + +COPY . /pi-gen/ + +VOLUME [ "/pi-gen/work", "/pi-gen/deploy", "/pi-gen/output" ] diff --git a/README.md b/README.md old mode 100644 new mode 100755 index af2cba3..e241c1b --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ _Tool used to create the raspberrypi.org Raspbian images_ ## Dependencies -`quilt kpartx realpath qemu-user-static debootstrap zerofree pxz zip dosfstools bsdtar libcap2-bin` +`quilt parted realpath qemu-user-static debootstrap zerofree pxz zip dosfstools bsdtar libcap2-bin grep rsync` ## Config @@ -35,10 +35,22 @@ A simple example for building Raspbian: IMG_NAME='Raspbian' ``` +## Docker Build + +```bash +vi config # Edit your config file. See above. +./build-docker.sh +``` +If everything goes well, your finished image will be in the `output` folder. + +If something breaks along the line, you can edit the corresponding scripts, and +continue: +``` +CONTINUE=1 ./build-docker.sh +``` + ## Stage Anatomy - - ### Raspbian Stage Overview The build of Raspbian is divided up into several stages for logical clarity diff --git a/build-docker.sh b/build-docker.sh new file mode 100755 index 0000000..a14d42b --- /dev/null +++ b/build-docker.sh @@ -0,0 +1,73 @@ +#!/bin/bash -e +DOCKER="docker" +set +e +$DOCKER ps >/dev/null 2>&1 +if [ $? != 0 ]; then + DOCKER="sudo docker" +fi +if ! $DOCKER ps >/dev/null; then + echo "error connecting to docker:" + $DOCKER ps + exit 1 +fi +set -e + +config_mount= +if [ -f config ]; then + config_mount="-v $(pwd)/config:/pi-gen/config:ro" + source config +fi + +CONTAINER_NAME=${CONTAINER_NAME:-pigen_work} +CONTINUE=${CONTINUE:-0} + +if [ "$*" != "" ] || [ -z "${IMG_NAME}" ]; then + if [ -z "${IMG_NAME}" ]; then + echo "IMG_NAME not set in 'build'" 1>&2 + echo 1>&2 + fi + cat >&2 < ) + CONTAINER_NAME=pigen_work set a name for the build container + CONTINUE=0 continue from a previously started container +EOF + exit 1 +fi + +CONTAINER_EXISTS=$($DOCKER ps -a --filter name="$CONTAINER_NAME" -q) +CONTAINER_RUNNING=$($DOCKER ps --filter name="$CONTAINER_NAME" -q) +if [ "$CONTAINER_RUNNING" != "" ]; then + echo "The build is already running in container $CONTAINER_NAME. Aborting." + exit 1 +fi +if [ "$CONTAINER_EXISTS" != "" ] && [ "$CONTINUE" != "1" ]; then + echo "Container $CONTAINER_NAME already exists and you did not specify CONTINUE=1. Aborting." + echo "You can delete the existing container like this:" + echo " docker rm $CONTAINER_NAME" + exit 1 +fi + +$DOCKER build -t pi-gen . +if [ "$CONTAINER_EXISTS" != "" ]; then + trap "echo 'got CTRL+C... please wait 5s';docker stop -t 5 ${CONTAINER_NAME}_cont" SIGINT SIGTERM + time $DOCKER run --rm --privileged \ + --volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \ + pi-gen \ + bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && + cd /pi-gen; ./build.sh; + rsync -avPc work/*/build.log deploy/*.zip /output/" & + wait +else + trap "echo 'got CTRL+C... please wait 5s'; docker stop -t 5 ${CONTAINER_NAME}" SIGINT SIGTERM + $DOCKER run --name "${CONTAINER_NAME}" --privileged \ + -v $(pwd)/output:/output \ + ${config_mount} \ + pi-gen \ + bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static && + cd /pi-gen; ./build.sh && + rsync -avPc work/*/build.log deploy/*.zip /output/" & + wait +fi +echo "Done! your output should be in output/" diff --git a/depends b/depends index a4c48cd..c453281 100644 --- a/depends +++ b/depends @@ -1,8 +1,12 @@ quilt +parted +realpath qemu-arm-static:qemu-user-static debootstrap -kpartx zerofree +zerofree pxz zip mkdosfs:dosfstools capsh:libcap2-bin bsdtar +grep +rsync \ No newline at end of file diff --git a/export-image/prerun.sh b/export-image/prerun.sh index e1aa05d..a3581b3 100755 --- a/export-image/prerun.sh +++ b/export-image/prerun.sh @@ -34,9 +34,21 @@ p w EOF -LOOP_DEV=`kpartx -asv ${IMG_FILE} | grep -E -o -m1 'loop[[:digit:]]+' | head -n 1` -BOOT_DEV=/dev/mapper/${LOOP_DEV}p1 -ROOT_DEV=/dev/mapper/${LOOP_DEV}p2 +PARTED_OUT=$(parted -s ${IMG_FILE} unit b print) +BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \ +| cut -d" " -f 2 | tr -d B) +BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \ +| cut -d" " -f 4 | tr -d B) + +ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \ +| cut -d" " -f 2 | tr -d B) +ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \ +| cut -d" " -f 4 | tr -d B) + +BOOT_DEV=$(losetup --show -f -o ${BOOT_OFFSET} --sizelimit ${BOOT_LENGTH} ${IMG_FILE}) +ROOT_DEV=$(losetup --show -f -o ${ROOT_OFFSET} --sizelimit ${ROOT_LENGTH} ${IMG_FILE}) +echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" +echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" mkdosfs -n boot -S 512 -s 16 -v $BOOT_DEV > /dev/null mkfs.ext4 -O ^huge_file $ROOT_DEV > /dev/null diff --git a/export-noobs/prerun.sh b/export-noobs/prerun.sh index 1b5d5a2..a3bb29f 100755 --- a/export-noobs/prerun.sh +++ b/export-noobs/prerun.sh @@ -9,9 +9,21 @@ cp ${WORK_DIR}/export-image/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}.img ${STAGE_WOR rm -rf ${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX} -LOOP_DEV=`kpartx -asv ${IMG_FILE} | grep -E -o -m1 'loop[[:digit:]]+' | head -n 1` -BOOT_DEV=/dev/mapper/${LOOP_DEV}p1 -ROOT_DEV=/dev/mapper/${LOOP_DEV}p2 +PARTED_OUT=$(parted -s ${IMG_FILE} unit b print) +BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \ +| cut -d" " -f 2 | tr -d B) +BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \ +| cut -d" " -f 4 | tr -d B) + +ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \ +| cut -d" " -f 2 | tr -d B) +ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \ +| cut -d" " -f 4 | tr -d B) + +BOOT_DEV=$(losetup --show -f -o ${BOOT_OFFSET} --sizelimit ${BOOT_LENGTH} ${IMG_FILE}) +ROOT_DEV=$(losetup --show -f -o ${ROOT_OFFSET} --sizelimit ${ROOT_LENGTH} ${IMG_FILE}) +echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH" +echo "/: offset $ROOT_OFFSET, length $ROOT_LENGTH" mkdir -p ${STAGE_WORK_DIR}/rootfs mkdir -p ${NOOBS_DIR} diff --git a/scripts/common b/scripts/common index 0920fb0..a67140f 100644 --- a/scripts/common +++ b/scripts/common @@ -1,10 +1,11 @@ log (){ - date +"[%T] $@" | tee -a ${LOG_FILE} + date +"[%T] $@" | tee -a "${LOG_FILE}" } export -f log bootstrap(){ - local ARCH=$(dpkg --print-architecture) + local ARCH + ARCH=$(dpkg --print-architecture) export http_proxy=${APT_PROXY} @@ -22,12 +23,12 @@ bootstrap(){ export -f bootstrap copy_previous(){ - if [ ! -d ${PREV_ROOTFS_DIR} ]; then + if [ ! -d "${PREV_ROOTFS_DIR}" ]; then echo "Previous stage rootfs not found" false fi - mkdir -p ${ROOTFS_DIR} - rsync -aHAXx ${PREV_ROOTFS_DIR}/ ${ROOTFS_DIR}/ + mkdir -p "${ROOTFS_DIR}" + rsync -aHAXx "${PREV_ROOTFS_DIR}/" "${ROOTFS_DIR}/" } export -f copy_previous @@ -38,10 +39,11 @@ unmount(){ DIR=$1 fi - while mount | grep -q $DIR; do - local LOCS=`mount | grep $DIR | cut -f 3 -d ' ' | sort -r` + while mount | grep -q "$DIR"; do + local LOCS + LOCS=$(mount | grep "$DIR" | cut -f 3 -d ' ' | sort -r) for loc in $LOCS; do - umount $loc + umount "$loc" done done } @@ -50,42 +52,47 @@ export -f unmount unmount_image(){ sync sleep 1 - local LOOP_DEV=$(losetup -j ${1} | cut -f1 -d':') - if [ -n "${LOOP_DEV}" ]; then - local MOUNTED_DIR=$(mount | grep $(basename ${LOOP_DEV}) | head -n 1 | cut -f 3 -d ' ') - if [ -n "${MOUNTED_DIR}" ]; then - unmount $(dirname ${MOUNTED_DIR}) + local LOOP_DEVICES + LOOP_DEVICES=$(losetup -j "${1}" | cut -f1 -d':') + for LOOP_DEV in ${LOOP_DEVICES}; do + if [ -n "${LOOP_DEV}" ]; then + local MOUNTED_DIR + MOUNTED_DIR=$(mount | grep "$(basename "${LOOP_DEV}")" | head -n 1 | cut -f 3 -d ' ') + if [ -n "${MOUNTED_DIR}" ] && [ "${MOUNTED_DIR}" != "/" ]; then + unmount "$(dirname "${MOUNTED_DIR}")" + fi + sleep 1 + losetup -d "${LOOP_DEV}" fi - sleep 1 - kpartx -ds ${LOOP_DEV} - losetup -d ${LOOP_DEV} - fi + done } export -f unmount_image on_chroot() { - if ! mount | grep -q `realpath ${ROOTFS_DIR}/proc`; then - mount -t proc proc ${ROOTFS_DIR}/proc + if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/proc)"; then + mount -t proc proc "${ROOTFS_DIR}/proc" fi - if ! mount | grep -q `realpath ${ROOTFS_DIR}/dev`; then - mount --bind /dev ${ROOTFS_DIR}/dev + if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/dev)"; then + mount --bind /dev "${ROOTFS_DIR}/dev" fi - if ! mount | grep -q `realpath ${ROOTFS_DIR}/dev/pts`; then - mount --bind /dev/pts ${ROOTFS_DIR}/dev/pts + if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/dev/pts)"; then + mount --bind /dev/pts "${ROOTFS_DIR}/dev/pts" fi - if ! mount | grep -q `realpath ${ROOTFS_DIR}/sys`; then - mount --bind /sys ${ROOTFS_DIR}/sys + if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/sys)"; then + mount --bind /sys "${ROOTFS_DIR}/sys" fi - capsh --drop=cap_setfcap --chroot=${ROOTFS_DIR}/ -- "$@" + capsh --drop=cap_setfcap "--chroot=${ROOTFS_DIR}/" -- "$@" } export -f on_chroot update_issue() { - local GIT_HASH=$(git rev-parse HEAD) - echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using pi-gen, https://github.com/RPi-Distro/pi-gen, ${GIT_HASH}, ${1}" > ${ROOTFS_DIR}/etc/rpi-issue + local GIT_HASH + GIT_HASH=$(git rev-parse HEAD) + echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using pi-gen, https://github.com/RPi-Distro/pi-gen, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue" } export -f update_issue +