Update os.json

Fixes #132

.dockerignore

@@ -1,3 +1,5 @@
 output/
 work/
 deploy/
+apt-cacher-ng/
+.git/objects/*

Dockerfile

@@ -1,10 +1,12 @@
 FROM debian:stretch
 
+ENV DEBIAN_FRONTEND noninteractive
+
 RUN apt-get -y update && \
     apt-get -y install \
         git vim parted \
         quilt realpath qemu-user-static debootstrap zerofree pxz zip dosfstools \
-        bsdtar libcap2-bin rsync grep udev xz-utils \
+        bsdtar libcap2-bin rsync grep udev xz-utils curl xxd \
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

README.md

@@ -62,10 +62,16 @@ The following environment variables are supported:
    system for each build stage, amounting to tens of gigabytes in the case of
    Raspbian.
    
+   **CAUTION**: If your working directory is on an NTFS partition you probably won't be able to build. Make sure this is a proper Linux filesystem.
+
  * `DEPLOY_DIR`  (Default: `"$BASE_DIR/deploy"`)
 
    Output directory for target system images and NOOBS bundles.
 
+ * `USE_QEMU` (Default: `"0"`)
+
+   This enable the Qemu mode and set filesystem and image suffix if set to 1.
+
 
 A simple example for building Raspbian:
 
@@ -166,7 +172,7 @@ from `./stage2` (if building a minimal system).
 # Example for building a lite system
 echo "IMG_NAME='Raspbian'" > config
 touch ./stage3/SKIP ./stage4/SKIP ./stage5/SKIP
-rm stage4/EXPORT*
+rm stage4/EXPORT* stage5/EXPORT*
 sudo ./build.sh  # or ./build-docker.sh
 ```
 

build-docker.sh

@@ -13,10 +13,9 @@ if ! $DOCKER ps >/dev/null; then
 fi
 set -e
 
-
-config_mount=()
+config_file=()
 if [ -f config ]; then
-	config_mount=("-v" "$(pwd)/config:/pi-gen/config:ro")
+	config_file=("--env-file" "$(pwd)/config")
 	source config
 fi
 
@@ -25,7 +24,7 @@ CONTINUE=${CONTINUE:-0}
 
 if [ "$*" != "" ] || [ -z "${IMG_NAME}" ]; then
 	if [ -z "${IMG_NAME}" ]; then
-		echo "IMG_NAME not set in 'build'" 1>&2
+		echo "IMG_NAME not set in 'config'" 1>&2
 		echo 1>&2
 	fi
 	cat >&2 <<EOF
@@ -47,13 +46,13 @@ fi
 if [ "$CONTAINER_EXISTS" != "" ] && [ "$CONTINUE" != "1" ]; then
 	echo "Container $CONTAINER_NAME already exists and you did not specify CONTINUE=1. Aborting."
 	echo "You can delete the existing container like this:"
-	echo "  docker rm -v $CONTAINER_NAME"
+	echo "  $DOCKER rm -v $CONTAINER_NAME"
 	exit 1
 fi
 
 $DOCKER build -t pi-gen .
 if [ "$CONTAINER_EXISTS" != "" ]; then
-	trap "echo 'got CTRL+C... please wait 5s';docker stop -t 5 ${CONTAINER_NAME}_cont" SIGINT SIGTERM
+	trap "echo 'got CTRL+C... please wait 5s'; $DOCKER stop -t 5 ${CONTAINER_NAME}_cont" SIGINT SIGTERM
 	time $DOCKER run --rm --privileged \
 		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
 		-e IMG_NAME=${IMG_NAME}\
@@ -61,17 +60,21 @@ if [ "$CONTAINER_EXISTS" != "" ]; then
 		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
 	cd /pi-gen; ./build.sh;
 	rsync -av work/*/build.log deploy/" &
-	wait
+	wait "$!"
 else
-	trap "echo 'got CTRL+C... please wait 5s'; docker stop -t 5 ${CONTAINER_NAME}" SIGINT SIGTERM
-	$DOCKER run --name "${CONTAINER_NAME}" --privileged \
+	trap "echo 'got CTRL+C... please wait 5s'; $DOCKER stop -t 5 ${CONTAINER_NAME}" SIGINT SIGTERM
+	time $DOCKER run --name "${CONTAINER_NAME}" --privileged \
 		-e IMG_NAME=${IMG_NAME}\
-		-v "$(pwd)/deploy:/pi-gen/deploy" \
-		"${config_mount[@]}" \
+		"${config_file[@]}" \
 		pi-gen \
 		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
 	cd /pi-gen; ./build.sh &&
 	rsync -av work/*/build.log deploy/" &
-	wait
+	wait "$!"
 fi
+echo "copying results from deploy/"
+$DOCKER cp "${CONTAINER_NAME}":/pi-gen/deploy .
+ls -lah deploy
+$DOCKER rm -v $CONTAINER_NAME
+
 echo "Done! Your image(s) should be in deploy/"

build.sh

@@ -119,6 +119,7 @@ if [ "$(id -u)" != "0" ]; then
 	exit 1
 fi
 
+
 if [ -f config ]; then
 	source config
 fi
@@ -128,6 +129,7 @@ if [ -z "${IMG_NAME}" ]; then
 	exit 1
 fi
 
+export USE_QEMU=${USE_QEMU:-0}
 export IMG_DATE=${IMG_DATE:-"$(date +%Y-%m-%d)"}
 
 export BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -177,11 +179,13 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 	source "${EXPORT_DIR}/EXPORT_IMAGE"
 	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename ${EXPORT_DIR})/rootfs
 	run_stage
+	if [ "${USE_QEMU}" != "1" ]; then
 		if [ -e ${EXPORT_DIR}/EXPORT_NOOBS ]; then
 			source ${EXPORT_DIR}/EXPORT_NOOBS
 			STAGE_DIR=${BASE_DIR}/export-noobs
 			run_stage
 		fi
+	fi
 done
 
 if [ -x postrun.sh ]; then

depends

@@ -11,3 +11,5 @@ bsdtar
 grep
 rsync
 xz:xz-utils
+curl
+xxd

export-image/03-set-partuuid/00-run.sh

@@ -2,7 +2,7 @@
 
 IMG_FILE="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}.img"
 
-IMGID="$(fdisk -l ${IMG_FILE} | sed -n 's/Disk identifier: 0x\([^ ]*\)/\1/p')"
+IMGID="$(dd if=${IMG_FILE} skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
 
 BOOT_PARTUUID="${IMGID}-01"
 ROOT_PARTUUID="${IMGID}-02"

export-image/04-finalise/01-run.sh

@@ -15,8 +15,10 @@ fi
 rm -f ${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache
 rm -f ${ROOTFS_DIR}/usr/sbin/policy-rc.d
 rm -f ${ROOTFS_DIR}/usr/bin/qemu-arm-static
-if [ -e ${ROOTFS_DIR}/etc/ld.so.preload.disabled ]; then
+if [ "${USE_QEMU}" != "1" ]; then
+	if [ -e ${ROOTFS_DIR}/etc/ld.so.preload.disabled ]; then
 		mv ${ROOTFS_DIR}/etc/ld.so.preload.disabled ${ROOTFS_DIR}/etc/ld.so.preload
+	fi
 fi
 
 rm -f ${ROOTFS_DIR}/etc/apt/sources.list~
@@ -43,6 +45,7 @@ for _FILE in $(find ${ROOTFS_DIR}/var/log/ -type f); do
 done
 
 rm -f "${ROOTFS_DIR}/root/.vnc/private.key"
+rm -f "${ROOTFS_DIR}/etc/vnc/updateid"
 
 update_issue $(basename ${EXPORT_DIR})
 install -m 644 ${ROOTFS_DIR}/etc/rpi-issue ${ROOTFS_DIR}/boot/issue.txt

export-image/prerun.sh

@@ -12,7 +12,8 @@ mkdir -p ${ROOTFS_DIR}
 BOOT_SIZE=$(du --apparent-size -s ${EXPORT_ROOTFS_DIR}/boot --block-size=1 | cut -f 1)
 TOTAL_SIZE=$(du --apparent-size -s ${EXPORT_ROOTFS_DIR} --exclude var/cache/apt/archives --block-size=1 | cut -f 1)
 
-IMG_SIZE=$((BOOT_SIZE + TOTAL_SIZE + (800 * 1024 * 1024)))
+ROUND_SIZE="$((4 * 1024 * 1024))"
+IMG_SIZE=$(((BOOT_SIZE + TOTAL_SIZE + (800 * 1024 * 1024) + ROUND_SIZE) / ROUND_SIZE * ROUND_SIZE))
 
 truncate -s ${IMG_SIZE} ${IMG_FILE}
 fdisk -H 255 -S 63 ${IMG_FILE} <<EOF
@@ -58,7 +59,7 @@ for FEATURE in metadata_csum 64bit; do
 	fi
 done
 mkdosfs -n boot -F 32 -v $BOOT_DEV > /dev/null
-mkfs.ext4 -O $ROOT_FEATURES $ROOT_DEV > /dev/null
+mkfs.ext4 -L rootfs -O $ROOT_FEATURES $ROOT_DEV > /dev/null
 
 mount -v $ROOT_DEV ${ROOTFS_DIR} -t ext4
 mkdir -p ${ROOTFS_DIR}/boot

export-noobs/00-release/files/os.json

@@ -10,7 +10,8 @@
         "Pi Model",
         "Pi 2",
         "Pi Zero",
-        "Pi 3"
+        "Pi 3",
+        "Pi Compute Module 3"
     ],
     "url": "http://www.raspbian.org/",
     "username": "pi",

export-noobs/00-release/files/release_notes.txt

@@ -1,4 +1,21 @@
 UNRELEASED:
+  * Added battery monitor plugin for taskbar - works on x86 images or first-generation Pi-Top
+  * Added cutdown mode to PCManFM file manager to reduce complexity
+  * Added ability to rename files in PCManFM by clicking name when selected
+  * Bug fix in Bluetooth ALSA module to reduce truncation of audio at end of playback
+  * Various small tweaks, bug fixes and theme modifications
+  * New kernel and firmware
+2017-09-07:
+  * Disable predictable network interface names for Ethernet devices
+  * Bug fix for keyboard settings dialog in Raspberry Pi Configuration
+  * Bug fix for crash on some videos and animations in Chromium
+  * Bug fix for taskbar crash when running RealVNC server
+  * Bug fix for reloading projects with extensions in Scratch 2
+  * Bug fix for MAC address problem in Bluetooth
+  * Simple mode and new icons in Thonny
+  * New Japanese translations in Raspberry Pi Configuration
+  * Install fonts-droid-fallback for international fonts
+2017-08-16:
   * Based on Raspbian Stretch (Debian version 9)
   * Version 60 of Chromium browser included
   * Version 3.0.1 of Sonic Pi included
@@ -9,6 +26,7 @@ UNRELEASED:
   * Various desktop applications modified to prompt for sudo password if needed
   * lxinput control options for mouse speed simplified
   * lxpanel plugins moved into separate packages
+  * Wireless firmware for Pi 3 and Pi 0W modified to address Broadpwn exploit
   * Latest kernel and firmware
   * Various small tweaks, bug fixes and theme modifications
 2017-07-05:

export-noobs/prerun.sh

@@ -33,8 +33,8 @@ mount $BOOT_DEV ${STAGE_WORK_DIR}/rootfs/boot
 
 ln -sv "/lib/systemd/system/apply_noobs_os_config.service" "$ROOTFS_DIR/etc/systemd/system/multi-user.target.wants/apply_noobs_os_config.service"
 
-bsdtar --format gnutar --use-compress-program pxz -C ${STAGE_WORK_DIR}/rootfs/boot -cpf ${NOOBS_DIR}/boot.tar.xz .
+bsdtar --numeric-owner --format gnutar --use-compress-program pxz -C ${STAGE_WORK_DIR}/rootfs/boot -cpf ${NOOBS_DIR}/boot.tar.xz .
 umount ${STAGE_WORK_DIR}/rootfs/boot
-bsdtar --format gnutar --use-compress-program pxz -C ${STAGE_WORK_DIR}/rootfs --one-file-system -cpf ${NOOBS_DIR}/root.tar.xz .
+bsdtar --numeric-owner --format gnutar --use-compress-program pxz -C ${STAGE_WORK_DIR}/rootfs --one-file-system -cpf ${NOOBS_DIR}/root.tar.xz .
 
 unmount_image ${IMG_FILE}

stage1/02-net-tweaks/00-run.sh

@@ -3,6 +3,4 @@
 install -m 644 files/ipv6.conf ${ROOTFS_DIR}/etc/modprobe.d/ipv6.conf
 install -m 644 files/hostname ${ROOTFS_DIR}/etc/hostname
 
-on_chroot << EOF
-dpkg-divert --add --local /lib/udev/rules.d/75-persistent-net-generator.rules
-EOF
+ln -sf /dev/null ${ROOTFS_DIR}/etc/systemd/network/99-default.link

stage2/01-sys-tweaks/00-packages

@@ -20,3 +20,4 @@ rpi-update
 libmtp-runtime
 rsync
 htop
+man-db

stage2/01-sys-tweaks/01-run.sh

@@ -17,8 +17,28 @@ systemctl disable nfs-common
 systemctl disable rpcbind
 systemctl disable ssh
 systemctl enable regenerate_ssh_host_keys
+EOF
+
+if [ "${USE_QEMU}" = "1" ]; then
+	echo "enter QEMU mode"
+	install -m 644 files/90-qemu.rules		${ROOTFS_DIR}/etc/udev/rules.d/
+	if [ -e ${ROOTFS_DIR}/etc/ld.so.preload.disabled ]; then
+		rm ${ROOTFS_DIR}/etc/ld.so.preload.disabled
+		touch ${ROOTFS_DIR}/etc/ld.so.preload.disabled
+	fi
+	if [ -e ${ROOTFS_DIR}/etc/ld.so.preload ]; then
+		rm ${ROOTFS_DIR}/etc/ld.so.preload
+		touch ${ROOTFS_DIR}/etc/ld.so.preload
+	fi
+	on_chroot << EOF
+systemctl disable resize2fs_once
+EOF
+	echo "leaving QEMU mode"
+else
+	on_chroot << EOF
 systemctl enable resize2fs_once
 EOF
+fi
 
 on_chroot << \EOF
 for GRP in input spi i2c gpio; do

stage2/01-sys-tweaks/files/90-qemu.rules

@@ -0,0 +1,3 @@
+KERNEL=="sda", SYMLINK+="mmcblk0"
+KERNEL=="sda?", SYMLINK+="mmcblk0p%n"
+KERNEL=="sda2", SYMLINK+="root"

stage2/02-net-tweaks/00-packages

@@ -1,3 +1,4 @@
 wpasupplicant wireless-tools firmware-atheros firmware-brcm80211 firmware-libertas firmware-misc-nonfree firmware-realtek
 raspberrypi-net-mods
 dhcpcd5
+net-tools

stage2/EXPORT_IMAGE

@@ -1 +1,4 @@
 IMG_SUFFIX="-lite"
+if [ "${USE_QEMU}" = "1" ]; then
+	export IMG_SUFFIX="${IMG_SUFFIX}-qemu"
+fi

stage3/00-install-packages/00-packages

@@ -9,3 +9,5 @@ gvfs
 rfkill
 chromium-browser rpi-chromium-mods
 gldriver-test
+fonts-droid-fallback
+fonts-liberation2

stage4/00-install-packages/00-packages

@@ -8,7 +8,6 @@ python-serial python3-serial
 python-picamera python3-picamera
 debian-reference-en dillo x2x
 raspberrypi-net-mods raspberrypi-ui-mods
-fonts-liberation2
 smartsim
 python-pip python3-pip
 python3-numpy

stage4/EXPORT_IMAGE

@@ -1 +1,4 @@
 IMG_SUFFIX="-4GB"
+if [ "${USE_QEMU}" = "1" ]; then
+	export IMG_SUFFIX="${IMG_SUFFIX}-qemu"
+fi

stage5/EXPORT_IMAGE

@@ -1 +1,4 @@
 IMG_SUFFIX=""
+if [ "${USE_QEMU}" = "1" ]; then
+	export IMG_SUFFIX="${IMG_SUFFIX}-qemu"
+fi