Merge pull request #17 from Landrash/dev

Update pi-gen build scripts

.dockerignore

@@ -0,0 +1,3 @@
+output/
+work/
+deploy/

Dockerfile

@@ -0,0 +1,12 @@
+FROM debian:jessie
+
+RUN apt-get -y update && \
+    apt-get -y install \
+        git vim parted \
+        quilt realpath qemu-user-static debootstrap zerofree pxz zip dosfstools \
+        bsdtar libcap2-bin rsync grep \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . /pi-gen/
+
+VOLUME [ "/pi-gen/work", "/pi-gen/deploy"]

README.md

@@ -5,6 +5,7 @@ The Haspbian image is built with the same script that generates the official [Ra
 
 By default the Haspbian image is built on a Debian 8 droplet on Digital Ocean and takes about 30 minutes to build on the cheapest droplet. Dependencies and everything is handled by the build script with the exception of `git`.
 
+<<<<<<< HEAD
 Build instructions:
 - Install git. `sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install git`
 - Clone the `rpi_gen` code. `git clone https://github.com/home-assistant/pi-gen.git`
@@ -16,7 +17,7 @@ Build instructions:
 
 ### Dependencies
 
-`quilt qemu-arm-static:qemu-user-static debootstrap kpartx zerofree pxz zip mkdosfs:dosfstools capsh:libcap2-bin bsdtar`
+`quilt parted realpath qemu-user-static debootstrap zerofree pxz zip dosfstools bsdtar libcap2-bin grep rsync`
 
 ## Config
 
@@ -35,6 +36,26 @@ A simple example for building Hassbian:
 IMG_NAME='Hassbian'
 ```
 
+## Docker Build
+
+```bash
+vi config         # Edit your config file. See above.
+./build-docker.sh
+```
+If everything goes well, your finished image will be in the `deploy/` folder.
+You can then remove the build container with `docker rm pigen_work`
+
+If something breaks along the line, you can edit the corresponding scripts, and
+continue:
+
+```
+CONTINUE=1 ./build-docker.sh
+```
+
+There is a possibility that even when running from a docker container, the installation of `qemu-user-static` will silently fail when building the image because `binfmt-support` _must be enabled on the underlying kernel_. An easy fix is to ensure `binfmt-support` is installed on the host machine before starting the `./build-docker.sh` script (or using your own docker build solution).
+
+## Stage Anatomy
+
 ### Raspbian Stage Overview
 
 The build of Hassbian is divided up into several stages for logical clarity
@@ -81,17 +102,13 @@ maintenance and allows for more easy customization.
    The original **Stage 3** and **Stage 4** are removed since they are not
    used on the HASSbian image.
 
-
-
-   
 ### Stage specification
 If you wish to build up to a specified stage (such as building up to stage 2 for a lite system), place an empty file named `SKIP` in each of the `./stage` directories you wish not to include.
 
 Then remove the `EXPORT*` files from `./stage4` (if building up to stage 2) or from `./stage2` (if building a minimal system).
 
 ```
-# Example for building a lite system without Home Assistant
+## Example for building a lite system without Home Assistant
 $ touch ./stage3/SKIP 
 $ rm stage3/EXPORT*
 ```
-

build-docker.sh

@@ -0,0 +1,73 @@
+#!/bin/bash -e
+DOCKER="docker"
+set +e
+$DOCKER ps >/dev/null 2>&1
+if [ $? != 0 ]; then
+	DOCKER="sudo docker"
+fi
+if ! $DOCKER ps >/dev/null; then
+	echo "error connecting to docker:"
+	$DOCKER ps
+	exit 1
+fi
+set -e
+
+config_mount=
+if [ -f config ]; then
+	config_mount="-v $(pwd)/config:/pi-gen/config:ro"
+	source config
+fi
+
+CONTAINER_NAME=${CONTAINER_NAME:-pigen_work}
+CONTINUE=${CONTINUE:-0}
+
+if [ "$*" != "" ] || [ -z "${IMG_NAME}" ]; then
+	if [ -z "${IMG_NAME}" ]; then
+		echo "IMG_NAME not set in 'build'" 1>&2
+		echo 1>&2
+	fi
+	cat >&2 <<EOF
+Usage:
+    build-docker.sh [options]
+Optional environment arguments: ( =<default> )
+    CONTAINER_NAME=pigen_work  set a name for the build container
+    CONTINUE=0                  continue from a previously started container
+EOF
+	exit 1
+fi
+
+CONTAINER_EXISTS=$($DOCKER ps -a --filter name="$CONTAINER_NAME" -q)
+CONTAINER_RUNNING=$($DOCKER ps --filter name="$CONTAINER_NAME" -q)
+if [ "$CONTAINER_RUNNING" != "" ]; then
+	echo "The build is already running in container $CONTAINER_NAME. Aborting."
+	exit 1
+fi
+if [ "$CONTAINER_EXISTS" != "" ] && [ "$CONTINUE" != "1" ]; then
+	echo "Container $CONTAINER_NAME already exists and you did not specify CONTINUE=1. Aborting."
+	echo "You can delete the existing container like this:"
+	echo "  docker rm $CONTAINER_NAME"
+	exit 1
+fi
+
+$DOCKER build -t pi-gen .
+if [ "$CONTAINER_EXISTS" != "" ]; then
+	trap "echo 'got CTRL+C... please wait 5s';docker stop -t 5 ${CONTAINER_NAME}_cont" SIGINT SIGTERM
+	time $DOCKER run --rm --privileged \
+		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
+		pi-gen \
+		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
+	cd /pi-gen; ./build.sh;
+	rsync -av work/*/build.log deploy/" &
+	wait
+else
+	trap "echo 'got CTRL+C... please wait 5s'; docker stop -t 5 ${CONTAINER_NAME}" SIGINT SIGTERM
+	$DOCKER run --name "${CONTAINER_NAME}" --privileged \
+		-v $(pwd)/deploy:/pi-gen/deploy \
+		${config_mount} \
+		pi-gen \
+		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
+	cd /pi-gen; ./build.sh &&
+	rsync -av work/*/build.log deploy/" &
+	wait
+fi
+echo "Done! Your image(s) should be in deploy/"

depends

@@ -1,8 +1,12 @@
 quilt
+parted
+realpath
 qemu-arm-static:qemu-user-static
 debootstrap
-kpartx zerofree
+zerofree
 pxz zip
 mkdosfs:dosfstools
 capsh:libcap2-bin
 bsdtar
+grep
+rsync

export-image/prerun.sh

@@ -34,9 +34,21 @@ p
 w
 EOF
 
-LOOP_DEV=`kpartx -asv ${IMG_FILE} | grep -E -o -m1 'loop[[:digit:]]+' | head -n 1`
+PARTED_OUT=$(parted -s ${IMG_FILE} unit b print)
-BOOT_DEV=/dev/mapper/${LOOP_DEV}p1
+BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \
-ROOT_DEV=/dev/mapper/${LOOP_DEV}p2
+| cut -d" " -f 2 | tr -d B)
+BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \
+| cut -d" " -f 4 | tr -d B)
+
+ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \
+| cut -d" " -f 2 | tr -d B)
+ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \
+| cut -d" " -f 4 | tr -d B)
+
+BOOT_DEV=$(losetup --show -f -o ${BOOT_OFFSET} --sizelimit ${BOOT_LENGTH} ${IMG_FILE})
+ROOT_DEV=$(losetup --show -f -o ${ROOT_OFFSET} --sizelimit ${ROOT_LENGTH} ${IMG_FILE})
+echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
+echo "/:     offset $ROOT_OFFSET, length $ROOT_LENGTH"
 
 mkdosfs -n boot -S 512 -s 16 -v $BOOT_DEV > /dev/null
 mkfs.ext4 -O ^huge_file $ROOT_DEV > /dev/null

export-noobs/prerun.sh

@@ -9,9 +9,21 @@ cp ${WORK_DIR}/export-image/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}.img ${STAGE_WOR
 
 rm -rf ${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}
 
-LOOP_DEV=`kpartx -asv ${IMG_FILE} | grep -E -o -m1 'loop[[:digit:]]+' | head -n 1`
+PARTED_OUT=$(parted -s ${IMG_FILE} unit b print)
-BOOT_DEV=/dev/mapper/${LOOP_DEV}p1
+BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \
-ROOT_DEV=/dev/mapper/${LOOP_DEV}p2
+| cut -d" " -f 2 | tr -d B)
+BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 1'| xargs echo -n \
+| cut -d" " -f 4 | tr -d B)
+
+ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \
+| cut -d" " -f 2 | tr -d B)
+ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^ 2'| xargs echo -n \
+| cut -d" " -f 4 | tr -d B)
+
+BOOT_DEV=$(losetup --show -f -o ${BOOT_OFFSET} --sizelimit ${BOOT_LENGTH} ${IMG_FILE})
+ROOT_DEV=$(losetup --show -f -o ${ROOT_OFFSET} --sizelimit ${ROOT_LENGTH} ${IMG_FILE})
+echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
+echo "/:     offset $ROOT_OFFSET, length $ROOT_LENGTH"
 
 mkdir -p ${STAGE_WORK_DIR}/rootfs
 mkdir -p ${NOOBS_DIR}

scripts/common

@@ -1,10 +1,11 @@
 log (){
-	date +"[%T] $@" | tee -a ${LOG_FILE}
+	date +"[%T] $@" | tee -a "${LOG_FILE}"
 }
 export -f log
 
 bootstrap(){
-	local ARCH=$(dpkg --print-architecture)
+	local ARCH
+	ARCH=$(dpkg --print-architecture)
 
 	export http_proxy=${APT_PROXY}
 
@@ -22,12 +23,12 @@ bootstrap(){
 export -f bootstrap
 
 copy_previous(){
-	if [ ! -d ${PREV_ROOTFS_DIR} ]; then
+	if [ ! -d "${PREV_ROOTFS_DIR}" ]; then
 		echo "Previous stage rootfs not found"
 		false
 	fi
-	mkdir -p ${ROOTFS_DIR}
+	mkdir -p "${ROOTFS_DIR}"
-	rsync -aHAXx ${PREV_ROOTFS_DIR}/ ${ROOTFS_DIR}/
+	rsync -aHAXx "${PREV_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
 }
 export -f copy_previous
 
@@ -38,10 +39,11 @@ unmount(){
 		DIR=$1
 	fi
 
-	while mount | grep -q $DIR; do
+	while mount | grep -q "$DIR"; do
-		local LOCS=`mount | grep $DIR | cut -f 3 -d ' ' | sort -r`
+		local LOCS
+		LOCS=$(mount | grep "$DIR" | cut -f 3 -d ' ' | sort -r)
 		for loc in $LOCS; do
-			umount $loc
+			umount "$loc"
 		done
 	done
 }
@@ -50,42 +52,47 @@ export -f unmount
 unmount_image(){
 	sync
 	sleep 1
-	local LOOP_DEV=$(losetup -j ${1} | cut -f1 -d':')
+	local LOOP_DEVICES
+	LOOP_DEVICES=$(losetup -j "${1}" | cut -f1 -d':')
+	for LOOP_DEV in ${LOOP_DEVICES}; do
 		if [ -n "${LOOP_DEV}" ]; then
-		local MOUNTED_DIR=$(mount | grep $(basename ${LOOP_DEV}) | head -n 1 | cut -f 3 -d ' ')
+			local MOUNTED_DIR
-		if [ -n "${MOUNTED_DIR}" ]; then
+			MOUNTED_DIR=$(mount | grep "$(basename "${LOOP_DEV}")" | head -n 1 | cut -f 3 -d ' ')
-			unmount $(dirname ${MOUNTED_DIR})
+			if [ -n "${MOUNTED_DIR}" ] && [ "${MOUNTED_DIR}" != "/" ]; then
+				unmount "$(dirname "${MOUNTED_DIR}")"
 			fi
 			sleep 1
-		kpartx -ds ${LOOP_DEV}
+			losetup -d "${LOOP_DEV}"
-		losetup -d ${LOOP_DEV}
 		fi
+	done
 }
 export -f unmount_image
 
 on_chroot() {
-	if ! mount | grep -q `realpath ${ROOTFS_DIR}/proc`; then
+	if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/proc)"; then
-		mount -t proc proc ${ROOTFS_DIR}/proc
+		mount -t proc proc "${ROOTFS_DIR}/proc"
 	fi
 
-	if ! mount | grep -q `realpath ${ROOTFS_DIR}/dev`; then
+	if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/dev)"; then
-		mount --bind /dev ${ROOTFS_DIR}/dev
+		mount --bind /dev "${ROOTFS_DIR}/dev"
 	fi
 	
-	if ! mount | grep -q `realpath ${ROOTFS_DIR}/dev/pts`; then
+	if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/dev/pts)"; then
-		mount --bind /dev/pts ${ROOTFS_DIR}/dev/pts
+		mount --bind /dev/pts "${ROOTFS_DIR}/dev/pts"
 	fi
 
-	if ! mount | grep -q `realpath ${ROOTFS_DIR}/sys`; then
+	if ! mount | grep -q "$(realpath "${ROOTFS_DIR}"/sys)"; then
-		mount --bind /sys ${ROOTFS_DIR}/sys
+		mount --bind /sys "${ROOTFS_DIR}/sys"
 	fi
 
-	capsh --drop=cap_setfcap --chroot=${ROOTFS_DIR}/ -- "$@"
+	capsh --drop=cap_setfcap "--chroot=${ROOTFS_DIR}/" -- "$@"
 }
 export -f on_chroot
 
 update_issue() {
-	local GIT_HASH=$(git rev-parse HEAD)
+	local GIT_HASH
-	echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using pi-gen, https://github.com/RPi-Distro/pi-gen, ${GIT_HASH}, ${1}" > ${ROOTFS_DIR}/etc/rpi-issue
+	GIT_HASH=$(git rev-parse HEAD)
+	echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using pi-gen, https://github.com/RPi-Distro/pi-gen, ${GIT_HASH}, ${1}" > "${ROOTFS_DIR}/etc/rpi-issue"
 }
 export -f update_issue
+

stage2/01-sys-tweaks/00-debconf

@@ -1,6 +1,13 @@
 # Encoding to use on the console:
 # Choices: ARMSCII-8, CP1251, CP1255, CP1256, GEORGIAN-ACADEMY, GEORGIAN-PS, IBM1133, ISIRI-3342, ISO-8859-1, ISO-8859-10, ISO-8859-11, ISO-8859-13, ISO-8859-14, ISO-8859-15, ISO-8859-16, ISO-8859-2, ISO-8859-3, ISO-8859-4, ISO-8859-5, ISO-8859-6, ISO-8859-7, ISO-8859-8, ISO-8859-9, KOI8-R, KOI8-U, TIS-620, UTF-8, VISCII
 console-setup   console-setup/charmap47 select  UTF-8
+# Character set to support:
+# Choices: . Arabic, # Armenian, # Cyrillic - KOI8-R and KOI8-U, # Cyrillic - non-Slavic languages, # Cyrillic - Slavic languages (also Bosnian and Serbian Latin), . Ethiopic, # Georgian, # Greek, # Hebrew, # Lao, # Latin1 and Latin5 - western Europe and Turkic languages, # Latin2 - central Europe and Romanian, # Latin3 and Latin8 - Chichewa; Esperanto; Irish; Maltese and Welsh, # Latin7 - Lithuanian; Latvian; Maori and Marshallese, . Latin - Vietnamese, # Thai, . Combined - Latin; Slavic Cyrillic; Hebrew; basic Arabic, . Combined - Latin; Slavic Cyrillic; Greek, . Combined - Latin; Slavic and non-Slavic Cyrillic, Guess optimal character set
+console-setup   console-setup/codeset47 select  Guess optimal character set
+# Font for the console:
+# Choices: Fixed, Goha, GohaClassic, Terminus, TerminusBold, TerminusBoldVGA, VGA, Do not change the boot/kernel font, Let the 
+system select a suitable font
+console-setup   console-setup/fontface47        select  Do not change the boot/kernel font
 # Geographic area:
 # Choices: Africa, America, Antarctica, Australia, Arctic Ocean, Asia, Atlantic Ocean, Europe, Indian Ocean, Pacific Ocean, System V timezones, US, None of the above
 tzdata	tzdata/Areas	select	Etc

stage2/01-sys-tweaks/00-patches/03-console-setup.diff

@@ -1,17 +0,0 @@
-Index: jessie-stage2/rootfs/etc/default/console-setup
-===================================================================
---- jessie-stage2.orig/rootfs/etc/default/console-setup
-+++ jessie-stage2/rootfs/etc/default/console-setup
-@@ -6,9 +6,9 @@ ACTIVE_CONSOLES="/dev/tty[1-6]"
- 
- CHARMAP="UTF-8"
- 
--CODESET="Lat15"
--FONTFACE="Fixed"
--FONTSIZE="8x16"
-+CODESET="guess"
-+FONTFACE=""
-+FONTSIZE=""
- 
- VIDEOMODE=
- 

stage2/01-sys-tweaks/00-patches/series

@@ -1,6 +1,5 @@
 01-useradd.diff
 02-swap.diff
-03-console-setup.diff
 04-inputrc.diff
 05-path.diff
 06-rc_local.diff