export-noobs: umount boot before archiving root

bsdtar does not add directories which are mountpoints. This removes /boot from
the archive and prevents init_resize.sh from working.

License.md

@@ -0,0 +1,13 @@
+Copyright (c) 2015 Raspberry Pi (Trading) Ltd.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1,10 +1,87 @@
 #TODO
 
-1. Image export
-1. NOOBS export
-1. Simplify running a single stage
 1. Documentation
 
 #Dependencies
 
-`quilt kpartx realpath qemu-user-static debootstrap zerofree`
+`quilt kpartx realpath qemu-user-static debootstrap zerofree pxz zip dosfstools`
+
+#Config
+
+Upon execution, `build.sh` will source the file `config` in the current
+working directory.  This bash shell fragment is intended to set needed
+environment variables.
+
+The following environment variables are supported:
+
+ * `IMG_NAME` **required** (Default: unset)
+
+   The name of the image to build with the current stage directories.  Setting
+   `IMG_NAME=Raspbian` is logical for an unmodified RPi-Distro/pi-gen build,
+   but you should use something else for a customized version.  Export files
+   in stages may add suffixes to `IMG_NAME`.
+
+ * `APT_PROXY` (Default: unset)
+
+   If you require the use of an apt proxy, set it here.  This proxy setting
+   will not be included in the image, making it safe to use an `apt-cacher` or
+   similar package for development.
+
+A simple example for building Raspbian:
+
+```bash
+IMG_NAME='Raspbian'
+```
+
+#Stage Anatomy
+
+
+
+#Raspbian Stage Overview
+
+The build of Raspbian is divided up into several stages for logical clarity
+and modularity.  This causes some initial complexity, but it simplifies
+maintenance and allows for more easy customization.
+
+ - Stage 0, bootstrap.  The primary purpose of this stage is to create a
+   usable filesystem.  This is accomplished largely through the use of
+   `debootstrap`, which creates a minimal filesystem suitable for use as a
+   base.tgz on Debian systems.  This stage also configures apt settings and
+   installs `raspberrypi-bootloader` which is missed by debootstrap.  The
+   minimal core is installed but not configured, and the system will not quite
+   boot yet.
+
+ - Stage 1, truly minimal system.  This stage makes the system bootable by
+   installing system files like `/etc/fstab`, configures the bootloader, makes
+   the network operable, and installs packages like raspi-config.  At this
+   stage the system should boot to a local console from which you have the
+   means to perform basic tasks needed to configure and install the system.
+   This is as minimal as a system can possibly get, and its arguably not
+   really usable yet in a traditional sense yet.  Still, if you want minimal,
+   this is minimal and the rest you could reasonably do yourself as sysadmin.
+
+ - State 2, lite system.  This stage produces the Raspbian-Lite image.  It
+   installs some optimized memory functions, sets timezone and charmap
+   defaults, installs fake-hwclock and ntp, wifi and bluetooth support,
+   dphys-swapfile, and other basics for managing the hardware.  It also
+   creates necessary groups and gives the pi user access to sudo and the
+   standard console hardware permission groups.
+
+   There are a few tools that may not make a whole lot of sense here for
+   development purposes on a minimal system such as basic python and lua
+   packages as well as the `build-essential` package.  They are lumped right
+   in with more essential packages presently, though they need not be with
+   pi-gen.  These are understandable for Raspbian's target audience, but if
+   you were looking for something between truly minimal and Raspbian-lite,
+   here's where you start trimming.
+
+ - Stage 3, desktop system.  Here's where you get the full desktop system
+   with X11 and LXDE, web browsers, git for development, Raspbian custom UI
+   enhancements, etc.  This is a base desktop system, with some development
+   tools installed.
+
+ - Stage 4, complete Raspbian system.  More development tools, an email
+   client, learning tools like Scratch, specialized packages like sonic-pi and
+   wolfram-engine, system documentation, office productivity, etc.  This is
+   the stage that installs all of the things that make Raspbian friendly to
+   new users.

build.sh

@@ -7,7 +7,7 @@ run_sub_stage()
 	for i in {00..99}; do
 		if [ -f ${i}-debconf ]; then
 			log "Begin ${SUB_STAGE_DIR}/${i}-debconf"
-			on_chroot sh -e - << EOF
+			on_chroot << EOF
 debconf-set-selections <<SELEOF
 `cat ${i}-debconf`
 SELEOF
@@ -16,9 +16,9 @@ EOF
 		fi
 		if [ -f ${i}-packages-nr ]; then
 			log "Begin ${SUB_STAGE_DIR}/${i}-packages-nr"
-			PACKAGES=`cat $i-packages-nr | tr '\n' ' '`
+			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < ${i}-packages-nr)"
 			if [ -n "$PACKAGES" ]; then
-				on_chroot sh -e - << EOF
+				on_chroot << EOF
 apt-get install --no-install-recommends -y $PACKAGES
 EOF
 			fi
@@ -26,9 +26,9 @@ EOF
 		fi
 		if [ -f ${i}-packages ]; then
 			log "Begin ${SUB_STAGE_DIR}/${i}-packages"
-			PACKAGES=`cat $i-packages | tr '\n' ' '`
+			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < ${i}-packages)"
 			if [ -n "$PACKAGES" ]; then
-				on_chroot sh -e - << EOF
+				on_chroot << EOF
 apt-get install -y $PACKAGES
 EOF
 			fi
@@ -42,8 +42,9 @@ EOF
 				rm -rf *-pc
 			fi
 			QUILT_PATCHES=${SUB_STAGE_DIR}/${i}-patches
-			mkdir -p ${i}-pc
-			ln -sf ${i}-pc .pc
+			SUB_STAGE_QUILT_PATCH_DIR="$(basename $SUB_STAGE_DIR)-pc"
+			mkdir -p $SUB_STAGE_QUILT_PATCH_DIR
+			ln -snf $SUB_STAGE_QUILT_PATCH_DIR .pc
 			if [ -e ${SUB_STAGE_DIR}/${i}-patches/EDIT ]; then
 				echo "Dropping into bash to edit patches..."
 				bash
@@ -68,7 +69,7 @@ EOF
 		fi
 		if [ -f ${i}-run-chroot ]; then
 			log "Begin ${SUB_STAGE_DIR}/${i}-run-chroot"
-			on_chroot sh -e - < ${i}-run-chroot
+			on_chroot < ${i}-run-chroot
 			log "End ${SUB_STAGE_DIR}/${i}-run-chroot"
 		fi
 	done
@@ -76,6 +77,7 @@ EOF
 	log "End ${SUB_STAGE_DIR}"
 }
 
+
 run_stage(){
 	log "Begin ${STAGE_DIR}"
 	STAGE=$(basename ${STAGE_DIR})
@@ -155,6 +157,10 @@ export QUILT_NO_DIFF_TIMESTAMPS=1
 export QUILT_REFRESH_ARGS="-p ab"
 
 source ${SCRIPT_DIR}/common
+source ${SCRIPT_DIR}/dependencies_check
+
+
+dependencies_check ${BASE_DIR}/depends
 
 mkdir -p ${WORK_DIR}
 log "Begin ${BASE_DIR}"
@@ -163,13 +169,16 @@ for STAGE_DIR in ${BASE_DIR}/stage*; do
 	run_stage
 done
 
-STAGE_DIR=${BASE_DIR}/export-image
-
 CLEAN=1
 for EXPORT_DIR in ${EXPORT_DIRS}; do
+	STAGE_DIR=${BASE_DIR}/export-image
 	IMG_SUFFIX=$(cat ${EXPORT_DIR}/EXPORT_IMAGE)
 	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename ${EXPORT_DIR})/rootfs
 	run_stage
+	if [ -e ${EXPORT_DIR}/EXPORT_NOOBS ]; then
+		STAGE_DIR=${BASE_DIR}/export-noobs
+		run_stage
+	fi
 done
 
 log "End ${BASE_DIR}"

depends

@@ -0,0 +1,8 @@
+quilt
+qemu-arm-static:qemu-user-static
+debootstrap
+kpartx zerofree
+pxz zip
+mkdosfs:dosfstools
+capsh:libcap2-bin
+bsdtar

export-image/01-set-sources/01-run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 apt-get update
 apt-get -y dist-upgrade
 apt-get clean

export-image/03-finalise/01-run.sh

@@ -2,10 +2,15 @@
 
 IMG_FILE="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}.img"
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 /etc/init.d/fake-hwclock stop
+hardlink -t /usr/share/doc
 EOF
 
+if [ -d ${ROOTFS_DIR}/home/pi/.config ]; then
+	chmod 700 ${ROOTFS_DIR}/home/pi/.config
+fi
+
 rm -f ${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache
 rm -f ${ROOTFS_DIR}/usr/sbin/policy-rc.d
 rm -f ${ROOTFS_DIR}/usr/bin/qemu-arm-static

export-image/prerun.sh

@@ -45,4 +45,4 @@ mount -v $ROOT_DEV ${ROOTFS_DIR} -t ext4
 mkdir -p ${ROOTFS_DIR}/boot
 mount -v $BOOT_DEV ${ROOTFS_DIR}/boot -t vfat
 
-rsync ${EXPORT_ROOTFS_DIR}/ ${ROOTFS_DIR}/ -aHAX
+rsync -aHAXx ${EXPORT_ROOTFS_DIR}/ ${ROOTFS_DIR}/

export-noobs/00-release/00-run.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -e
+
+NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}"
+
+install -v -m 744	files/partition_setup.sh	${NOOBS_DIR}/
+install -v		files/partitions.json		${NOOBS_DIR}/
+install -v		files/os.json			${NOOBS_DIR}/
+install -v		files/Raspbian.png		${NOOBS_DIR}/
+install -v		files/release_notes.txt		${NOOBS_DIR}/
+
+tar -v -c -C		files/marketing			-f ${NOOBS_DIR}/marketing.tar .
+
+BOOT_SIZE=$(xz --robot -l ${NOOBS_DIR}/boot.tar.xz  | grep totals | cut -f 5)
+ROOT_SIZE=$(xz --robot -l ${NOOBS_DIR}/root.tar.xz  | grep totals | cut -f 5)
+
+BOOT_SIZE=$(expr ${BOOT_SIZE} / 1000000 \+ 1)
+ROOT_SIZE=$(expr ${ROOT_SIZE} / 1000000 \+ 1)
+
+BOOT_NOM=$(expr ${BOOT_SIZE} \* 3)
+ROOT_NOM=$(expr ${ROOT_SIZE} \+ 400)
+
+sed ${NOOBS_DIR}/partitions.json -i -e "s|BOOT_SIZE|${BOOT_SIZE}|"
+sed ${NOOBS_DIR}/partitions.json -i -e "s|ROOT_SIZE|${ROOT_SIZE}|"
+
+sed ${NOOBS_DIR}/partitions.json -i -e "s|BOOT_NOM|${BOOT_NOM}|"
+sed ${NOOBS_DIR}/partitions.json -i -e "s|ROOT_NOM|${ROOT_NOM}|"
+
+sed ${NOOBS_DIR}/release_notes.txt -i -e "s|UNRELEASED|${IMG_DATE}|"
+sed ${NOOBS_DIR}/os.json -i -e "s|UNRELEASED|${IMG_DATE}|"
+
+cp -a ${NOOBS_DIR} ${DEPLOY_DIR}/

export-noobs/00-release/files/os.json

@@ -0,0 +1,18 @@
+{
+    "description": "A community-created port of Debian jessie for the Raspberry Pi",
+    "feature_level": 35120124,
+    "kernel": "4.4",
+    "name": "Raspbian",
+    "password": "raspberry",
+    "release_date": "UNRELEASED",
+    "supported_hex_revisions": "2,3,4,5,6,7,8,9,d,e,f,10,11,12,14,19,1040,1041,0092,0093,2082",
+    "supported_models": [
+        "Pi Model",
+        "Pi 2",
+        "Pi Zero",
+        "Pi 3"
+    ],
+    "url": "http://www.raspbian.org/",
+    "username": "pi",
+    "version": "jessie"
+}

export-noobs/00-release/files/partition_setup.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -ex
+
+if [ -z "$part1" ] || [ -z "$part2" ]; then
+  printf "Error: missing environment variable part1 or part2\n" 1>&2
+  exit 1
+fi
+
+mkdir -p /tmp/1 /tmp/2
+
+mount "$part1" /tmp/1
+mount "$part2" /tmp/2
+
+sed /tmp/1/cmdline.txt -i -e "s|root=/dev/[^ ]*|root=${part2}|"
+sed /tmp/2/etc/fstab -i -e "s|^.* / |${part2}  / |"
+sed /tmp/2/etc/fstab -i -e "s|^.* /boot |${part1}  /boot |"
+
+if ! grep -q resize /proc/cmdline; then
+  sed -i 's| quiet init=/usr/lib/raspi-config/init_resize.sh||' /tmp/1/cmdline.txt
+fi
+
+umount /tmp/1
+umount /tmp/2

export-noobs/00-release/files/partitions.json

@@ -0,0 +1,19 @@
+{
+    "partitions": [
+        {
+            "filesystem_type": "FAT",
+            "label": "boot",
+            "partition_size_nominal": BOOT_NOM,
+            "uncompressed_tarball_size": BOOT_SIZE,
+            "want_maximised": false
+        },
+        {
+            "filesystem_type": "ext4",
+            "label": "root",
+            "mkfs_options": "-O ^huge_file",
+            "partition_size_nominal": ROOT_NOM,
+            "uncompressed_tarball_size": ROOT_SIZE,
+            "want_maximised": true
+        }
+    ]
+}

export-noobs/00-release/files/release_notes.txt

@@ -0,0 +1,168 @@
+UNRELEASED:
+  * SSH disabled by default; can be enabled by creating a file with name "ssh" in boot partition
+  * Prompt for password change at boot when SSH enabled with default password unchanged
+  * Adobe Flash Player included
+  * Updates to hardware video acceleration in Chromium browser
+  * Greeter now uses background image from last set in Appearance Settings rather than pi user
+  * Updated version of Scratch
+  * Rastrack option removed from raspi-config and Raspberry Pi Configuration
+  * Ability to disable graphical boot splash screen added to raspi-config and Raspberry Pi Configuration
+  * Appearance Settings dialog made tabbed to work better on small screens
+  * Raspberry Pi Configuration now requires current password to change password
+  * Various small bug fixes
+  * Updated firmware and kernel
+2016-09-23:
+  * New PIXEL desktop environment - new icon set, window design, desktop images, splash screen and greeter
+  * Chromium web browser included
+  * Infinality font rendering patches included
+  * RealVNC server and viewer included
+  * SenseHAT emulator included
+  * Rfkill entries added to Wifi and Bluetooth panel plugins
+  * Updates to various standard applications, including Scratch and NodeRED
+  * Various bug fixes, tweaks and translation updates
+  * Updated firmware and kernel (https://github.com/raspberrypi/firmware/commit/ad8608c08b122b2c228dba0ff5070d6e9519faf5)
+2016-05-27:
+  * Fixed crash of lxpanel when D-bus not accessible
+  * Fixed permissions for D-bus Bluetooth access
+  * Removed sudo from shutdown options
+  * Appearance of tooltips updated in theme
+  * Fixed ejecter plugin grabbing focus
+  * raspi-config command line and GUI apps tidied; unnecessary reboots removed
+  * More error detection in piclone; copying of volume names and IDs added
+  * Updated translation files
+2016-05-10:
+  * New version of Scratch, which no longer requires sudo
+  * New version of BlueJ
+  * New version of NodeRED
+  * New version of pypy
+  * pigpio included
+  * geany editor included
+  * SD Card Copier added (can be used to duplicate or back up the Pi)
+  * Bluetooth plugin added to taskbar
+  * Volume control on taskbar now compatible with Bluetooth devices
+  * New shutdown helper application
+  * Mouse double-click speed setting added to mouse and keyboard preference application
+  * Option to enable / disable 1-wire interface and remote access to pigpio added to Raspberry Pi config application
+  * File system automatically expanded on first boot
+  * Empty Wastebasket option added to right-click menu 
+  * Ctrl-Alt-T can be used to open a terminal window
+  * Various small bug fixes and appearance tweaks 
+  * Updated firmware and kernel (https://github.com/raspberrypi/firmware/commit/cc6d7bf8b4c03a2a660ff9fdf4083fc165620866)
+2016-03-18:
+  * updated firmware and kernel (https://github.com/raspberrypi/firmware/commit/951799bbcd795ddf27769d14acf4813fdcbe53dc)
+  * use serial0 in cmdline.txt
+  * wpa_supplicant.conf country default to GB (allows use of channels 12 and 13)
+2016-02-26:
+  * Support added for Pi 3, including Wifi and Bluetooth
+  * Option to set wifi country code added to raspi-config
+2016-02-09:
+  * dtb that uses mmc sdcard driver (fixes problems experienced with certain SD cards)
+2016-02-03:
+  * new version of Sonic Pi (2.9)
+  * new version of Scratch (15/1/16)
+  * new version of Node-Red (2.5)
+  * new version of Wolfram (10.3)
+  * optional experimental GL desktop driver (can be enabled using advanced options in command-line raspi-config)
+  * new version of Java (1.8.0_65)
+  * new version of WiringPi
+  * raspi-gpio included
+  * ping no longer requires sudo (except NOOBS installs)
+  * support for more USB audio devices in lxpanel
+  * bug fix for creation of new menus in Alacarte
+  * various changes to raspi-config and GUI to tidy up board support and fix bugs, and updated translations
+  * small tweaks to theme to support GL driver
+2015-11-21:
+  * Included IBM Node-RED IoT application
+  * Included graphical package manager
+  * Included accelerated pixman library
+  * Updated Epiphany browser to improve video compatibility
+  * Updated Scratch with performance improvements and bug fixes
+  * Updated Raspberry Pi configuration to allow boot to pause while
+    network is established
+  * Various minor bug fixes
+2015-09-25:
+  * Based on Debian Jessie
+  * Upgraded applications - Epiphany browser, Scratch and Sonic Pi
+  * Included applications - LibreOffice, Claws Mail, Greenfoot, BlueJ
+  * Included utilities - Alacarte menu editor, Lxkeymap, scrot, tree, pip
+  * New GUI-based Raspberry Pi Configuration application
+  * GPIO control now possible without need for sudo
+  * Web link to Magpi magazine included
+  * New taskbar plugin to eject mounted USB drives
+  * Default boot is now to GUI not desktop
+  * Look and feel now based on GTK+3 default theme
+  * Print screen key launches scrot to produce screenshot
+  * Common keyboards autodetected by GUI and drivers loaded accordingly
+  * Numerous small tweaks and bugfixes
+2015-05-05:
+  * Updated UI changes
+  * Updated firmware
+  * Install raspberrypi-net-mods
+  * Install avahi-daemon
+  * Add user pi to new i2c and spi groups
+  * Modified udev rules for i2c and spi devices
+2015-02-16:
+  * Newer firmware with various fixes
+  * New Sonic Pi release
+  * Pi2 compatible RPi.GPIO
+  * Updated Wolfram Mathematica
+2015-01-31:
+  * Support for Pi2
+  * Newer firmware
+  * New Sonic Pi release
+  * Updated Scratch
+  * New Wolfram Mathematica release
+  * Updated Epiphany
+2014-12-24:
+  * Fix regression with omission of python-pygame
+2014-12-22:
+  * New firmware with variosu fixes and improvements
+  * New UI configuration for lxde
+  * Various package updates
+  * python3-pygame preinstalled
+  * 'nuscratch', scratch running on the Cog StackVM
+  * Misc other changes
+2014-09-09:
+  * New firmware with various fixes and improvements
+  * Minecraft Pi pre-installed
+  * Sonic Pi upgraded to 2.0
+  * Include Epiphany browser work from Collabora
+  * Switch to Java 8 from Java 7
+  * Updated Mathematica
+  * Misc minor configuration changes
+2014-06-20:
+  * New firmware with various fixes, and kernel bugfix
+2014-06-02:
+  * Many, many firmware updates with major USB improvements
+  * pyserial installed by default
+  * picamera installed by default
+2014-01-07:
+  * Firmware updated
+  * Some space saved on the root filesystem
+2013-12-20:
+  * Firmware updated, includes V4L2 fixes
+  * Update omxplayer
+2013-12-18:
+  * Firmware updated and now using kernel 3.10. Many, many improvements
+  * fbturbo XOrg driver is now included and enabled by default. Thanks to 
+    ssvb https://github.com/ssvb/xf86-video-fbturbo
+  * Update Scratch image with further bug fixes
+  * Include Wolfram Mathematica
+  * Update to PyPy 2.2
+  * Update omxplayer
+  * Include v4l-utils for use with experimental V4L2 Raspberry Pi camera driver
+  * Update squeak-vm to fix issues with loading JPEGs
+2013-09-25:
+  * Update Scratch image for further performance improvements
+  * Include Oracle JDK
+  * At least a 4GiB SD card is now required (see above)
+  * Include PyPy 2.1
+  * Include base piface packages
+  * Update raspi-config to include bugfix for inheriting language settings
+    from NOOBS
+2013-09-10:
+  * Updated to current top of tree firmware
+  * Update squeak-vm, including fastblit optimised for the Raspbery Pi 
+  * Include Sonic Pi and a fixed jackd2 package
+  * Support boot to Scratch
+  * Inherit keyboard and language settings from NOOBS

export-noobs/prerun.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -e
+
+IMG_FILE="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}.img"
+NOOBS_DIR="${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}"
+unmount_image ${IMG_FILE}
+
+mkdir -p ${STAGE_WORK_DIR}
+cp ${WORK_DIR}/export-image/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}.img ${STAGE_WORK_DIR}/
+
+rm -rf ${STAGE_WORK_DIR}/${IMG_DATE}-${IMG_NAME}${IMG_SUFFIX}
+
+LOOP_DEV=`kpartx -asv ${IMG_FILE} | grep -E -o -m1 'loop[[:digit:]]+' | head -n 1`
+BOOT_DEV=/dev/mapper/${LOOP_DEV}p1
+ROOT_DEV=/dev/mapper/${LOOP_DEV}p2
+
+mkdir -p ${STAGE_WORK_DIR}/rootfs
+mkdir -p ${NOOBS_DIR}
+
+mount $ROOT_DEV ${STAGE_WORK_DIR}/rootfs
+mount $BOOT_DEV ${STAGE_WORK_DIR}/rootfs/boot
+
+bsdtar --format gnutar --use-compress-program pxz -C ${STAGE_WORK_DIR}/rootfs/boot -cpf ${NOOBS_DIR}/boot.tar.xz .
+umount ${STAGE_WORK_DIR}/rootfs/boot
+bsdtar --format gnutar --use-compress-program pxz -C ${STAGE_WORK_DIR}/rootfs --one-file-system -cpf ${NOOBS_DIR}/root.tar.xz .
+
+unmount_image ${IMG_FILE}

scripts/common

@@ -4,20 +4,20 @@ log (){
 export -f log
 
 bootstrap(){
-	ARCH=$(dpkg --print-architecture)
+	local ARCH=$(dpkg --print-architecture)
 
 	export http_proxy=${APT_PROXY}
 
 	if [ "$ARCH" !=  "armhf" ]; then
-		BOOTSTRAP_CMD=qemu-debootstrap
+		local BOOTSTRAP_CMD=qemu-debootstrap
 	else
-		BOOTSTRAP_CMD=debootstrap
+		local BOOTSTRAP_CMD=debootstrap
 	fi
 
-	${BOOTSTRAP_CMD} --components=main,contrib,non-free \
+	capsh --drop=cap_setfcap -- -c "${BOOTSTRAP_CMD} --components=main,contrib,non-free \
 		--arch armhf\
 		--no-check-gpg \
-		$1 $2 $3
+		$1 $2 $3"
 }
 export -f bootstrap
 
@@ -27,7 +27,7 @@ copy_previous(){
 		false
 	fi
 	mkdir -p ${ROOTFS_DIR}
-	rsync -aHAX ${PREV_ROOTFS_DIR}/ ${ROOTFS_DIR}/
+	rsync -aHAXx ${PREV_ROOTFS_DIR}/ ${ROOTFS_DIR}/
 }
 export -f copy_previous
 
@@ -39,7 +39,7 @@ unmount(){
 	fi
 
 	while mount | grep -q $DIR; do
-		LOCS=`mount | grep $DIR | cut -f 3 -d ' ' | sort -r`
+		local LOCS=`mount | grep $DIR | cut -f 3 -d ' ' | sort -r`
 		for loc in $LOCS; do
 			umount $loc
 		done
@@ -50,9 +50,12 @@ export -f unmount
 unmount_image(){
 	sync
 	sleep 1
-	unmount $(dirname ${1})
-	LOOP_DEV=$(losetup -j ${1} | cut -f1 -d':')
+	local LOOP_DEV=$(losetup -j ${1} | cut -f1 -d':')
 	if [ -n "${LOOP_DEV}" ]; then
+		local MOUNTED_DIR=$(mount | grep $(basename ${LOOP_DEV}) | head -n 1 | cut -f 3 -d ' ')
+		if [ -n "${MOUNTED_DIR}" ]; then
+			unmount $(dirname ${MOUNTED_DIR})
+		fi
 		sleep 1
 		kpartx -ds ${LOOP_DEV}
 		losetup -d ${LOOP_DEV}
@@ -77,12 +80,12 @@ on_chroot() {
 		mount --bind /sys ${ROOTFS_DIR}/sys
 	fi
 
-	chroot ${ROOTFS_DIR}/ "$@"
+	capsh --drop=cap_setfcap --chroot=${ROOTFS_DIR}/ -- "$@"
 }
 export -f on_chroot
 
 update_issue() {
-	GIT_HASH=$(git rev-parse HEAD)
+	local GIT_HASH=$(git rev-parse HEAD)
 	echo -e "Raspberry Pi reference ${IMG_DATE}\nGenerated using pi-gen, https://github.com/RPi-Distro/pi-gen, ${GIT_HASH}, ${1}" > ${ROOTFS_DIR}/etc/rpi-issue
 }
 export -f update_issue

scripts/dependencies_check

@@ -0,0 +1,30 @@
+# dependencies_check
+# $@	Dependnecy files to check
+#
+# Each dependency is in the form of a tool to test for, optionally followed by
+# a : and the name of a package if the package on a Debian-ish system is not
+# named for the tool (i.e., qemu-user-static).
+dependencies_check()
+{
+	local depfile deps missing
+
+	for depfile in "$@"; do
+		if [[ -e "$depfile" ]]; then
+			deps="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < ${BASE_DIR}/depends)"
+
+		fi
+		for dep in $deps; do
+			if ! hash ${dep%:*} 2>/dev/null; then
+				missing="${missing:+$missing }${dep#*:}"
+			fi
+		done
+	done
+
+	if [[ "$missing" ]]; then
+		echo "Reqired dependencies not installed"
+		echo
+		echo "This can be resolved on Debian/Raspbian systems by installing:"
+		echo "$missing"
+		false
+	fi
+}

scripts/remove-comments.sed

@@ -0,0 +1,11 @@
+# Deletes comments and collapses whitespace in ##-packages files
+
+# Append (N)ext line to buffer
+# if (!)not ($)buffer is EOF, (b)ranch to (:)label loop
+:loop
+N
+$ !b loop
+
+# Buffer is "line1\nline2\n...lineN", del comments and collapse whitespace
+s/#[^\n]*//g
+s/[[:space:]]\{1,\}/ /g

stage0/01-configure-apt/00-run.sh

@@ -11,7 +11,7 @@ else
 fi
 
 on_chroot apt-key add - < files/raspberrypi.gpg.key
-on_chroot sh -e - << EOF
+on_chroot << EOF
 apt-get update
 apt-get dist-upgrade -y
 EOF

stage1/01-sys-tweaks/00-run.sh

@@ -5,7 +5,7 @@ install -m 644 files/noclear.conf ${ROOTFS_DIR}/etc/systemd/system/[email protected]
 install -m 744 files/policy-rc.d ${ROOTFS_DIR}/usr/sbin/policy-rc.d #TODO: Necessary in systemd?
 install -v -m 644 files/fstab ${ROOTFS_DIR}/etc/fstab
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 if ! id -u pi >/dev/null 2>&1; then
 	adduser --disabled-password --gecos "" pi
 fi

stage1/02-net-tweaks/00-run.sh

@@ -4,6 +4,6 @@ install -m 644 files/ipv6.conf ${ROOTFS_DIR}/etc/modprobe.d/ipv6.conf
 install -m 644 files/interfaces ${ROOTFS_DIR}/etc/network/interfaces
 install -m 644 files/hostname ${ROOTFS_DIR}/etc/hostname
 
-on_chroot sh -e - << EOF
+on_chroot << EOF
 dpkg-divert --add --local /lib/udev/rules.d/75-persistent-net-generator.rules
 EOF

stage2/01-sys-tweaks/00-patches/00-sudoers.diff

@@ -1,10 +0,0 @@
-Index: jessie-stage2/rootfs/etc/sudoers
-===================================================================
---- jessie-stage2.orig/rootfs/etc/sudoers
-+++ jessie-stage2/rootfs/etc/sudoers
-@@ -25,3 +25,5 @@ root	ALL=(ALL:ALL) ALL
- # See sudoers(5) for more information on "#include" directives:
- 
- #includedir /etc/sudoers.d
-+
-+pi ALL=(ALL) NOPASSWD: ALL

stage2/01-sys-tweaks/00-patches/series

@@ -1,4 +1,3 @@
-00-sudoers.diff
 01-useradd.diff
 02-swap.diff
 03-console-setup.diff

stage2/01-sys-tweaks/01-run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-install -m 755 files/regenerate_ssh_host_keys		${ROOTFS_DIR}/etc/init.d/
+install -m 644 files/regenerate_ssh_host_keys.service	${ROOTFS_DIR}/lib/systemd/system/
 install -m 755 files/apply_noobs_os_config		${ROOTFS_DIR}/etc/init.d/
 install -m 755 files/resize2fs_once			${ROOTFS_DIR}/etc/init.d/
 
@@ -8,10 +8,9 @@ install -d						${ROOTFS_DIR}/etc/systemd/system/rc-local.service.d
 install -m 644 files/ttyoutput.conf			${ROOTFS_DIR}/etc/systemd/system/rc-local.service.d/
 
 install -m 644 files/50raspi				${ROOTFS_DIR}/etc/apt/apt.conf.d/
-install -m 644 files/98-rpi.conf			${ROOTFS_DIR}/etc/sysctl.d/
 
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 systemctl disable hwclock.sh
 systemctl disable nfs-common
 systemctl disable rpcbind
@@ -21,7 +20,7 @@ systemctl enable apply_noobs_os_config
 systemctl enable resize2fs_once
 EOF
 
-on_chroot sh -e - << \EOF
+on_chroot << \EOF
 for GRP in input spi i2c gpio; do
 	groupadd -f -r $GRP
 done
@@ -30,11 +29,11 @@ for GRP in adm dialout cdrom audio users sudo video games plugdev input gpio spi
 done
 EOF
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 setupcon --force --save-only -v
 EOF
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 usermod --pass='*' root
 EOF
 

stage2/01-sys-tweaks/files/98-rpi.conf

@@ -1,3 +0,0 @@
-kernel.printk = 3 4 1 3
-vm.swappiness=1
-vm.min_free_kbytes = 8192

stage2/01-sys-tweaks/files/regenerate_ssh_host_keys

@@ -1,36 +0,0 @@
-#!/bin/sh
-### BEGIN INIT INFO
-# Provides:          regenerate_ssh_host_keys
-# Required-Start:
-# Required-Stop:
-# Default-Start: 2
-# Default-Stop:
-# Short-Description: Regenerate ssh host keys
-# Description:
-### END INIT INFO
-
-. /lib/lsb/init-functions
-
-set -e
-
-case "$1" in
-  start)
-    log_daemon_msg "Regenerating ssh host keys (in background)"
-    nohup sh -c "if [ -e /dev/hwrng ]; then
-          dd if=/dev/hwrng of=/dev/urandom count=1 bs=4096
-      fi; \
-      yes | ssh-keygen -q -N '' -t dsa -f /etc/ssh/ssh_host_dsa_key && \
-      yes | ssh-keygen -q -N '' -t rsa -f /etc/ssh/ssh_host_rsa_key && \
-      yes | ssh-keygen -q -N '' -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key && \
-      yes | ssh-keygen -q -N '' -t ed25519 -f /etc/ssh/ssh_host_ed25519_key && \
-      systemctl enable ssh && sync && \
-      rm /etc/init.d/regenerate_ssh_host_keys && \
-      update-rc.d regenerate_ssh_host_keys remove && \
-      printf '\nfinished\n' && systemctl start ssh" > /var/log/regen_ssh_keys.log 2>&1 &
-    log_end_msg $?
-    ;;
-  *)
-    echo "Usage: $0 start" >&2
-    exit 3
-    ;;
-esac

stage2/01-sys-tweaks/files/regenerate_ssh_host_keys.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Regenerate SSH host keys
+
+[Service]
+Type=oneshot
+ExecStartPre=/bin/sh -c "if [ -e /dev/hwrng ]; then dd if=/dev/hwrng of=/dev/urandom count=1 bs=4096; fi"
+ExecStart=/usr/bin/ssh-keygen -A
+ExecStartPost=/bin/rm /lib/systemd/system/regenerate_ssh_host_keys.service ; /usr/sbin/update-rc.d regenerate_ssh_host_keys remove
+
+[Install]
+WantedBy=multi-user.target

stage2/02-net-tweaks/00-packages

@@ -1,2 +1,3 @@
 wpasupplicant wireless-tools firmware-atheros firmware-brcm80211 firmware-libertas firmware-ralink firmware-realtek
+raspberrypi-net-mods
 dhcpcd5

stage2/02-net-tweaks/00-patches/00-interfaces.diff

@@ -1,27 +0,0 @@
-Index: jessie-stage2/rootfs/etc/network/interfaces
-===================================================================
---- jessie-stage2.orig/rootfs/etc/network/interfaces
-+++ jessie-stage2/rootfs/etc/network/interfaces
-@@ -1,4 +1,20 @@
--auto lo
-+# interfaces(5) file used by ifup(8) and ifdown(8)
-+
-+# Please note that this file is written to be used with dhcpcd
-+# For static IP, consult /etc/dhcpcd.conf and 'man dhcpcd.conf'
- 
-+# Include files from /etc/network/interfaces.d:
-+source-directory /etc/network/interfaces.d
-+
-+auto lo
- iface lo inet loopback
--iface eth0 inet dhcp
-+
-+iface eth0 inet manual
-+
-+allow-hotplug wlan0
-+iface wlan0 inet manual
-+    wpa-conf /etc/wpa_supplicant/wpa_supplicant.conf
-+
-+allow-hotplug wlan1
-+iface wlan1 inet manual
-+    wpa-conf /etc/wpa_supplicant/wpa_supplicant.conf

stage2/02-net-tweaks/00-patches/series

@@ -1 +0,0 @@
-00-interfaces.diff

stage2/03-cleanup/00-run.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 apt-get clean
 EOF

stage3/00-install-packages/00-debconf

@@ -0,0 +1,2 @@
+# Adobe Flash Player. Copyright 1996-2015. Adobe Systems Incorporated. All Rights Reserved.
+rpi-chromium-mods	rpi-chromium-mods/adobe	note

stage3/00-install-packages/00-packages

@@ -5,4 +5,6 @@ git-core rpi-update
 omxplayer
 raspberrypi-artwork
 policykit-1
-udisks
+gvfs
+rfkill
+chromium-browser rpi-chromium-mods

stage3/00-install-packages/01-run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+on_chroot << EOF
+update-alternatives --install /usr/bin/x-www-browser \
+  x-www-browser /usr/bin/chromium-browser 86
+update-alternatives --install /usr/bin/gnome-www-browser \
+  gnome-www-browser /usr/bin/chromium-browser 86
+EOF
+

stage3/01-tweaks/00-patches/01-lightdm.diff

@@ -1,13 +0,0 @@
-Index: jessie-stage3/rootfs/etc/lightdm/lightdm-gtk-greeter.conf
-===================================================================
---- jessie-stage3.orig/rootfs/etc/lightdm/lightdm-gtk-greeter.conf
-+++ jessie-stage3/rootfs/etc/lightdm/lightdm-gtk-greeter.conf
-@@ -16,7 +16,7 @@
- # screensaver-timeout = Timeout (in seconds) until the screen blanks when the greeter is called as lockscreen
- # 
- [greeter]
--background=/usr/share/images/desktop-base/login-background.svg
-+background=#ffffff
- theme-name=Adwaita
- #icon-theme-name=
- #font-name=

stage3/01-tweaks/00-patches/series

@@ -1 +0,0 @@
-01-lightdm.diff

stage3/01-tweaks/00-run.sh

@@ -1,15 +1,3 @@
 #!/bin/bash -e
 
-on_chroot sh -e - <<EOF
-update-alternatives --set libgksu-gconf-defaults /usr/share/libgksu/debian/gconf-defaults.libgksu-sudo
-update-gconf-defaults
-EOF
-
-on_chroot sh -e - <<EOF
-update-alternatives --install /usr/share/images/desktop-base/desktop-background \
-desktop-background /usr/share/raspberrypi-artwork/raspberry-pi-logo.png 100
-EOF
-
-rm -f							${ROOTFS_DIR}/etc/systemd/system/dhcpcd.service.d/wait.conf
-install -m 644 files/55-storage.pkla			${ROOTFS_DIR}/etc/polkit-1/localauthority/50-local.d/
-install -m 644 files/75source-profile		${ROOTFS_DIR}/etc/X11/Xsession.d/
+rm -f ${ROOTFS_DIR}/etc/systemd/system/dhcpcd.service.d/wait.conf

stage3/01-tweaks/files/55-storage.pkla

@@ -1,6 +0,0 @@
-[Storage Permissions]
-Identity=unix-group:plugdev
-Action=org.freedesktop.udisks.filesystem-mount;org.freedesktop.udisks.drive-eject;org.freedesktop.udisks.drive-detach;org.freedesktop.udisks.luks-unlock;org.freedesktop.udisks.inhibit-polling;org.freedesktop.udisks.drive-set-spindown
-ResultAny=yes
-ResultActive=yes
-ResultInactive=no

stage3/01-tweaks/files/75source-profile

@@ -1,2 +0,0 @@
-[ -f /etc/profile ] && . /etc/profile
-[ -f "$HOME/.profile" ] && . "$HOME/.profile"

stage4/00-install-packages/00-packages

@@ -26,3 +26,12 @@ geany
 piclone
 pulseaudio-module-bluetooth
 wiringpi pigpio python-pigpio python3-pigpio raspi-gpio python-gpiozero python3-gpiozero python3-rpi.gpio
+python-spidev python3-spidev
+realvnc-vnc-server realvnc-vnc-viewer
+python-twython python3-twython
+python-smbus python3-smbus
+python-flask python3-flask
+python-picraft python3-picraft
+python3-codebug-tether python3-codebug-i2c-tether
+python-sense-emu python3-sense-emu sense-emu-tools python-sense-emu-doc
+pprompt

stage4/00-install-packages/01-run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+rm -f "${ROOTFS_DIR}/root/.vnc/private.key"

stage4/01-console-autologin/00-run.sh

@@ -1,5 +1,3 @@
 #!/bin/bash -e
 
 ln -sf /etc/systemd/system/[email protected] ${ROOTFS_DIR}/etc/systemd/system/getty.target.wants/[email protected]
-
-install -v -m 644 files/40-scratch.rules ${ROOTFS_DIR}/etc/udev/rules.d/

stage4/01-tweaks/files/40-scratch.rules

@@ -1 +0,0 @@
-ATTRS{idVendor}=="0694", ATTRS{idProduct}=="0003", SUBSYSTEMS=="usb", ACTION=="add", MODE="0666", GROUP="plugdev"

stage4/02-extras/00-patches/0-autologin.diff

@@ -1,13 +0,0 @@
-Index: jessie-stage4/rootfs/etc/lightdm/lightdm.conf
-===================================================================
---- jessie-stage4.orig/rootfs/etc/lightdm/lightdm.conf
-+++ jessie-stage4/rootfs/etc/lightdm/lightdm.conf
-@@ -114,7 +114,7 @@
- #session-setup-script=
- #session-cleanup-script=
- #autologin-guest=false
--#autologin-user=
-+autologin-user=pi
- #autologin-user-timeout=0
- #autologin-in-background=false
- #autologin-session=UNIMPLEMENTED

stage4/02-extras/00-patches/series

@@ -1 +0,0 @@
-0-autologin.diff

stage4/03-cleanup/00-run.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -e
 
-on_chroot sh -e - <<EOF
+on_chroot << EOF
 apt-get clean
 EOF