adding all ML frameworks and manual install of GGC instead of script

pull/388/head
Samuel Cozannet 2020-02-04 10:44:11 +01:00
parent c507c2109c
commit 1111e00bdb
20 changed files with 99 additions and 502 deletions

View File

@ -1,6 +1,7 @@
FROM i386/debian:buster
ENV DEBIAN_FRONTEND noninteractive
ENV XZ_DEFAULTS --memlimit=1024MiB
RUN apt-get -y update && \
apt-get -y install \

View File

@ -7,7 +7,7 @@ Index: jessie-stage2/rootfs/etc/dphys-swapfile
# set size to absolute value, leaving empty (default) then uses computed value
# you most likely don't want this, unless you have an special disk situation
-#CONF_SWAPSIZE=
+CONF_SWAPSIZE=100
+CONF_SWAPSIZE=1024
# set size to computed value, this times RAM size, dynamically adapts,
# guarantees that there is enough swap without wasting disk space on excess

View File

@ -1,3 +0,0 @@
libffi-dev
libssl-dev
python-configparser

View File

@ -1,8 +0,0 @@
#!/bin/bash -e
install -m 755 files/install-docker.sh "${ROOTFS_DIR}/tmp/"
on_chroot << EOF
/tmp/install-docker.sh
usermod -aG docker pi
EOF

View File

@ -1,476 +0,0 @@
#!/bin/sh
set -e
# This script is meant for quick & easy install via:
# $ curl -fsSL https://get.docker.com -o get-docker.sh
# $ sh get-docker.sh
#
# For test builds (ie. release candidates):
# $ curl -fsSL https://test.docker.com -o test-docker.sh
# $ sh test-docker.sh
#
# NOTE: Make sure to verify the contents of the script
# you downloaded matches the contents of install.sh
# located at https://github.com/docker/docker-install
# before executing.
#
# Git commit from https://github.com/docker/docker-install when
# the script was uploaded (Should only be modified by upload job):
SCRIPT_COMMIT_SHA="f45d7c11389849ff46a6b4d94e0dd1ffebca32c1"
# The channel to install from:
# * nightly
# * test
# * stable
# * edge (deprecated)
DEFAULT_CHANNEL_VALUE="stable"
if [ -z "$CHANNEL" ]; then
CHANNEL=$DEFAULT_CHANNEL_VALUE
fi
DEFAULT_DOWNLOAD_URL="https://download.docker.com"
if [ -z "$DOWNLOAD_URL" ]; then
DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL
fi
DEFAULT_REPO_FILE="docker-ce.repo"
if [ -z "$REPO_FILE" ]; then
REPO_FILE="$DEFAULT_REPO_FILE"
fi
mirror=''
DRY_RUN=${DRY_RUN:-}
while [ $# -gt 0 ]; do
case "$1" in
--mirror)
mirror="$2"
shift
;;
--dry-run)
DRY_RUN=1
;;
--*)
echo "Illegal option $1"
;;
esac
shift $(( $# > 0 ? 1 : 0 ))
done
case "$mirror" in
Aliyun)
DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce"
;;
AzureChinaCloud)
DOWNLOAD_URL="https://mirror.azure.cn/docker-ce"
;;
esac
command_exists() {
command -v "$@" > /dev/null 2>&1
}
is_dry_run() {
if [ -z "$DRY_RUN" ]; then
return 1
else
return 0
fi
}
deprecation_notice() {
distro=$1
date=$2
echo
echo "DEPRECATION WARNING:"
echo " The distribution, $distro, will no longer be supported in this script as of $date."
echo " If you feel this is a mistake please submit an issue at https://github.com/docker/docker-install/issues/new"
echo
sleep 10
}
get_distribution() {
lsb_dist=""
# Every system that we officially support has /etc/os-release
if [ -r /etc/os-release ]; then
lsb_dist="$(. /etc/os-release && echo "$ID")"
fi
# Returning an empty string here should be alright since the
# case statements don't act unless you provide an actual value
echo "$lsb_dist"
}
add_debian_backport_repo() {
debian_version="$1"
backports="deb http://ftp.debian.org/debian $debian_version-backports main"
if ! grep -Fxq "$backports" /etc/apt/sources.list; then
(set -x; $sh_c "echo \"$backports\" >> /etc/apt/sources.list")
fi
}
echo_docker_as_nonroot() {
if is_dry_run; then
return
fi
if command_exists docker && [ -e /var/run/docker.sock ]; then
(
set -x
$sh_c 'docker version'
) || true
fi
your_user=your-user
[ "$user" != 'root' ] && your_user="$user"
# intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output
echo "If you would like to use Docker as a non-root user, you should now consider"
echo "adding your user to the \"docker\" group with something like:"
echo
echo " sudo usermod -aG docker $your_user"
echo
echo "Remember that you will have to log out and back in for this to take effect!"
echo
echo "WARNING: Adding a user to the \"docker\" group will grant the ability to run"
echo " containers which can be used to obtain root privileges on the"
echo " docker host."
echo " Refer to https://docs.docker.com/engine/security/security/#docker-daemon-attack-surface"
echo " for more information."
}
# Check if this is a forked Linux distro
check_forked() {
# Check for lsb_release command existence, it usually exists in forked distros
if command_exists lsb_release; then
# Check if the `-u` option is supported
set +e
lsb_release -a -u > /dev/null 2>&1
lsb_release_exit_code=$?
set -e
# Check if the command has exited successfully, it means we're in a forked distro
if [ "$lsb_release_exit_code" = "0" ]; then
# Print info about current distro
cat <<-EOF
You're using '$lsb_dist' version '$dist_version'.
EOF
# Get the upstream release info
lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]')
dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]')
# Print info about upstream distro
cat <<-EOF
Upstream release is '$lsb_dist' version '$dist_version'.
EOF
else
if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then
if [ "$lsb_dist" = "osmc" ]; then
# OSMC runs Raspbian
lsb_dist=raspbian
else
# We're Debian and don't even know it!
lsb_dist=debian
fi
dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
case "$dist_version" in
10)
dist_version="buster"
;;
9)
dist_version="stretch"
;;
8|'Kali Linux 2')
dist_version="jessie"
;;
esac
fi
fi
fi
}
semverParse() {
major="${1%%.*}"
minor="${1#$major.}"
minor="${minor%%.*}"
patch="${1#$major.$minor.}"
patch="${patch%%[-.]*}"
}
ee_notice() {
echo
echo
echo " WARNING: $1 is now only supported by Docker EE"
echo " Check https://store.docker.com for information on Docker EE"
echo
echo
}
do_install() {
echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA"
if command_exists docker; then
docker_version="$(docker -v | cut -d ' ' -f3 | cut -d ',' -f1)"
MAJOR_W=1
MINOR_W=10
semverParse "$docker_version"
shouldWarn=0
if [ "$major" -lt "$MAJOR_W" ]; then
shouldWarn=1
fi
if [ "$major" -le "$MAJOR_W" ] && [ "$minor" -lt "$MINOR_W" ]; then
shouldWarn=1
fi
cat >&2 <<-'EOF'
Warning: the "docker" command appears to already exist on this system.
If you already have Docker installed, this script can cause trouble, which is
why we're displaying this warning and provide the opportunity to cancel the
installation.
If you installed the current Docker package using this script and are using it
EOF
if [ $shouldWarn -eq 1 ]; then
cat >&2 <<-'EOF'
again to update Docker, we urge you to migrate your image store before upgrading
to v1.10+.
You can find instructions for this here:
https://github.com/docker/docker/wiki/Engine-v1.10.0-content-addressability-migration
EOF
else
cat >&2 <<-'EOF'
again to update Docker, you can safely ignore this message.
EOF
fi
cat >&2 <<-'EOF'
You may press Ctrl+C now to abort this script.
EOF
( set -x; sleep 20 )
fi
user="$(id -un 2>/dev/null || true)"
sh_c='sh -c'
if [ "$user" != 'root' ]; then
if command_exists sudo; then
sh_c='sudo -E sh -c'
elif command_exists su; then
sh_c='su -c'
else
cat >&2 <<-'EOF'
Error: this installer needs the ability to run commands as root.
We are unable to find either "sudo" or "su" available to make this happen.
EOF
exit 1
fi
fi
if is_dry_run; then
sh_c="echo"
fi
# perform some very rudimentary platform detection
lsb_dist=$( get_distribution )
lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"
case "$lsb_dist" in
ubuntu)
if command_exists lsb_release; then
dist_version="$(lsb_release --codename | cut -f2)"
fi
if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then
dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")"
fi
;;
debian|raspbian)
dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')"
case "$dist_version" in
10)
dist_version="buster"
;;
9)
dist_version="stretch"
;;
8)
dist_version="jessie"
;;
esac
;;
centos)
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
;;
rhel|ol|sles)
ee_notice "$lsb_dist"
exit 1
;;
*)
if command_exists lsb_release; then
dist_version="$(lsb_release --release | cut -f2)"
fi
if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then
dist_version="$(. /etc/os-release && echo "$VERSION_ID")"
fi
;;
esac
# Check if this is a forked Linux distro
check_forked
# Run setup for each distro accordingly
case "$lsb_dist" in
ubuntu|debian|raspbian)
pre_reqs="apt-transport-https ca-certificates curl"
if [ "$lsb_dist" = "debian" ]; then
# libseccomp2 does not exist for debian jessie main repos for aarch64
if [ "$(uname -m)" = "aarch64" ] && [ "$dist_version" = "jessie" ]; then
add_debian_backport_repo "$dist_version"
fi
fi
if ! command -v gpg > /dev/null; then
pre_reqs="$pre_reqs gnupg"
fi
apt_repo="deb [arch=$(dpkg --print-architecture)] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL"
(
if ! is_dry_run; then
set -x
fi
$sh_c 'apt-get update -qq >/dev/null'
$sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null"
$sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" | apt-key add -qq - >/dev/null"
$sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list"
$sh_c 'apt-get update -qq >/dev/null'
)
pkg_version=""
if [ -n "$VERSION" ]; then
if is_dry_run; then
echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
else
# Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel
pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/~ce~.*/g" | sed "s/-/.*/g").*-0~$lsb_dist"
search_command="apt-cache madison 'docker-ce' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3"
pkg_version="$($sh_c "$search_command")"
echo "INFO: Searching repository for VERSION '$VERSION'"
echo "INFO: $search_command"
if [ -z "$pkg_version" ]; then
echo
echo "ERROR: '$VERSION' not found amongst apt-cache madison results"
echo
exit 1
fi
search_command="apt-cache madison 'docker-ce-cli' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3"
# Don't insert an = for cli_pkg_version, we'll just include it later
cli_pkg_version="$($sh_c "$search_command")"
pkg_version="=$pkg_version"
fi
fi
(
if ! is_dry_run; then
set -x
fi
if [ -n "$cli_pkg_version" ]; then
$sh_c "apt-get install -y -qq --no-install-recommends docker-ce-cli=$cli_pkg_version >/dev/null"
fi
$sh_c "apt-get install -y -qq --no-install-recommends docker-ce$pkg_version >/dev/null"
)
echo_docker_as_nonroot
exit 0
;;
centos|fedora)
yum_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE"
if ! curl -Ifs "$yum_repo" > /dev/null; then
echo "Error: Unable to curl repository file $yum_repo, is it valid?"
exit 1
fi
if [ "$lsb_dist" = "fedora" ]; then
pkg_manager="dnf"
config_manager="dnf config-manager"
enable_channel_flag="--set-enabled"
disable_channel_flag="--set-disabled"
pre_reqs="dnf-plugins-core"
pkg_suffix="fc$dist_version"
else
pkg_manager="yum"
config_manager="yum-config-manager"
enable_channel_flag="--enable"
disable_channel_flag="--disable"
pre_reqs="yum-utils"
pkg_suffix="el"
fi
(
if ! is_dry_run; then
set -x
fi
$sh_c "$pkg_manager install -y -q $pre_reqs"
$sh_c "$config_manager --add-repo $yum_repo"
if [ "$CHANNEL" != "stable" ]; then
$sh_c "$config_manager $disable_channel_flag docker-ce-*"
$sh_c "$config_manager $enable_channel_flag docker-ce-$CHANNEL"
fi
$sh_c "$pkg_manager makecache"
)
pkg_version=""
if [ -n "$VERSION" ]; then
if is_dry_run; then
echo "# WARNING: VERSION pinning is not supported in DRY_RUN"
else
pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g").*$pkg_suffix"
search_command="$pkg_manager list --showduplicates 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'"
pkg_version="$($sh_c "$search_command")"
echo "INFO: Searching repository for VERSION '$VERSION'"
echo "INFO: $search_command"
if [ -z "$pkg_version" ]; then
echo
echo "ERROR: '$VERSION' not found amongst $pkg_manager list results"
echo
exit 1
fi
search_command="$pkg_manager list --showduplicates 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'"
# It's okay for cli_pkg_version to be blank, since older versions don't support a cli package
cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)"
# Cut out the epoch and prefix with a '-'
pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)"
fi
fi
(
if ! is_dry_run; then
set -x
fi
# install the correct cli version first
if [ -n "$cli_pkg_version" ]; then
$sh_c "$pkg_manager install -y -q docker-ce-cli-$cli_pkg_version"
fi
$sh_c "$pkg_manager install -y -q docker-ce$pkg_version"
)
echo_docker_as_nonroot
exit 0
;;
*)
echo
echo "ERROR: Unsupported distribution '$lsb_dist'"
echo
exit 1
;;
esac
exit 1
}
# wrapped up in a function so that we have some protection against only getting
# half the file during "curl | sh"
do_install

View File

@ -0,0 +1 @@
openjdk-8-jdk

View File

@ -0,0 +1,24 @@
#!/bin/bash -e
install -m 755 files/install-greengrass.sh "${ROOTFS_DIR}/bin/"
install -m 644 files/greengrass.service "${ROOTFS_DIR}/etc/systemd/system/greengrass.service"
install -m 755 files/S02greengrass "${ROOTFS_DIR}/etc/init.d/S02greengrass"
sed -i.bak -e 's/$/"splash plymouth.ignore-serial-consoles cgroup_enable=memory cgroup_memory=1"/' "${ROOTFS_DIR}/boot/cmdline.txt"
[ -f "${ROOTFS_DIR}/etc/sysctl.d/98-rpi.conf" ] || touch "${ROOTFS_DIR}/etc/sysctl.d/98-rpi.conf"
cat >> "${ROOTFS_DIR}/etc/sysctl.d/98-rpi.conf" << EOF
fs.protected_hardlinks = 1
fs.protected_symlinks = 1
EOF
wget -c -q -O greengrass.tar.gz "https://d1onfpft10uf5o.cloudfront.net/greengrass-core/downloads/1.10.0/greengrass-raspbian-armv7l-1.10.0.tar.gz"
tar xfvz greengrass.tar.gz -C ${ROOTFS_DIR}/
rm -f greengrass.tar.gz
sudo wget -O ${ROOTFS_DIR}//greengrass/certs/root.ca.pem https://www.amazontrust.com/repository/AmazonRootCA1.pem
on_chroot << EOF
adduser --system ggc_user
addgroup --system ggc_group
systemctl enable greengrass.service
EOF

View File

@ -1,7 +1,5 @@
#!/bin/sh
mkdir -p /greengrass/certs
mkdir -p /greengrass/config
cp /boot/certs/* /greengrass/certs/
cp /boot/config/* /greengrass/config/
cd /greengrass/ggc/core
./greengrassd \$@

View File

@ -1,12 +0,0 @@
#!/bin/bash -e
install -m 755 files/install-greengrass.sh "${ROOTFS_DIR}/bin/"
install -m 644 files/greengrass.service /etc/systemd/system/greengrass.service
install -m 755 files/S02greengrass /etc/init.d/S02greengrass
on_chroot << EOF
sed -i -e 's/$/"splash plymouth.ignore-serial-consoles cgroup_enable=memory cgroup_memory=1"/' /boot/cmdline.txt
/bin/install-greengrass.sh bootstrap-greengrass
modprobe configs
systemctl enable greengrass.service
EOF

View File

@ -0,0 +1,4 @@
liblapack3
libopenblas-dev
liblapack-dev
python3.7-dev

View File

@ -0,0 +1,22 @@
#!/bin/bash -e
# Note: Setting versions because of https://github.com/EdjeElectronics/TensorFlow-Object-Detection-on-the-Raspberry-Pi/issues/67
on_chroot << EOF
# Installing for Python 2.7
pip2 install numpy>=1.16.2
pip2 install opencv-python==3.4.6.27
pip2 install python-scipy
pip2 install wheel
pip2 install picamera
# For Python3
pip3 install numpy>=1.16.2
pip3 install opencv-python==3.4.6.27
pip3 install python-scipy
pip3 install wheel
pip3 install picamera
if [ ! -r /dev/raw1394 ]; then
sudo ln /dev/null /dev/raw1394
fi
EOF

View File

@ -0,0 +1,4 @@
python-dev
python-nose
python-pip
zip

View File

@ -0,0 +1,36 @@
#!/bin/bash -e
# Install MXNet
wget -c -q -O mxnet.tar.gz https://d1onfpft10uf5o.cloudfront.net/greengrass-ml-installers/mxnet/ggc-mxnet-v1.2.1-python-raspi.tar.gz
tar -xzf mxnet.tar.gz -c "${ROOTFS_DIR}/opt"
rm -f mxnet.tar.gz
on_chroot << EOF
cd /opt/ggc-mxnet-v1.2.1-python-raspi
pip install mxnet-1.2.1-py2.py3-none-any.whl
EOF
# Install Tensorflow
wget -c -q -O tensorflow.tar.gz https://d1onfpft10uf5o.cloudfront.net/greengrass-ml-installers/tf/greengrass_ML@Edge_TF_v1_4_0_installer_cp27_raspi3_armv7.tar.gz
tar -xzf tensorflow.tar.gz -c "${ROOTFS_DIR}/opt"
rm -f tensorflow.tar.gz
on_chroot << EOF
cd /opt
pip install tensorflow-1.4.0-cp27-none-linux_armv7l.whl
EOF
# Install DL Framework
wget -c -q -O dlr.tar.gz https://d1onfpft10uf5o.cloudfront.net/greengrass-ml-installers/dlr/dlr-1.0-py2-armv7l.tar.gz
tar -xzf dlr.tar.gz -c "${ROOTFS_DIR}/opt"
rm -f dlr.tar.gz
on_chroot << EOF
cd /opt/dlr-1.0-py2-armv7l
easy_install dlr-1.0-py2.7-linux-armv7l.egg
EOF
### NOT DOING THESE FOR NOW. Not sure what the use case is
# Install Chainer
# wget -c -q -O chainer.tar.gz https://s3.amazonaws.com/chainer-greengrass-packages/chainer-installer-for-raspi-v4.0.0.tar.gz
# tar -xzf chainer.tar.gz

4
stageX/EXPORT_IMAGE Normal file
View File

@ -0,0 +1,4 @@
IMG_SUFFIX="-full"
if [ "${USE_QEMU}" = "1" ]; then
export IMG_SUFFIX="${IMG_SUFFIX}-qemu"
fi

2
stageX/EXPORT_NOOBS Normal file
View File

@ -0,0 +1,2 @@
NOOBS_NAME="Raspbian Full"
NOOBS_DESCRIPTION="A port of Debian with desktop and recommended applications"