From 1768bdaa4cb484b752fb096da140de7baedb69cf Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Sun, 2 Dec 2018 00:21:42 -0800 Subject: [PATCH] Build external dependencies local to tree --- .gitmodules | 12 +++++ copyextfiles.sh | 68 -------------------------- deps/01-download.sh | 30 ++++++++++++ deps/02-extract.sh | 38 +++++++++++++++ deps/03-build.sh | 57 ++++++++++++++++++++++ deps/04-copy.sh | 76 ++++++++++++++++++++++++++++++ deps/allwpilib | 1 + deps/pynetworktables | 1 + deps/robotpy-cscore | 1 + deps/thirdparty-opencv | 1 + deps/tools/Makefile | 21 +++++++++ {tools => deps/tools}/setuidgids.c | 0 tools/Makefile | 36 -------------- 13 files changed, 238 insertions(+), 104 deletions(-) create mode 100644 .gitmodules delete mode 100755 copyextfiles.sh create mode 100755 deps/01-download.sh create mode 100755 deps/02-extract.sh create mode 100755 deps/03-build.sh create mode 100755 deps/04-copy.sh create mode 160000 deps/allwpilib create mode 160000 deps/pynetworktables create mode 160000 deps/robotpy-cscore create mode 160000 deps/thirdparty-opencv create mode 100644 deps/tools/Makefile rename {tools => deps/tools}/setuidgids.c (100%) delete mode 100644 tools/Makefile diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f902ee0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,12 @@ +[submodule "deps/robotpy-cscore"] + path = deps/robotpy-cscore + url = git@github.com:robotpy/robotpy-cscore.git +[submodule "deps/pynetworktables"] + path = deps/pynetworktables + url = git@github.com:robotpy/pynetworktables.git +[submodule "deps/allwpilib"] + path = deps/allwpilib + url = git@github.com:wpilibsuite/allwpilib.git +[submodule "deps/thirdparty-opencv"] + path = deps/thirdparty-opencv + url = git@github.com:wpilibsuite/thirdparty-opencv.git diff --git a/copyextfiles.sh b/copyextfiles.sh deleted file mode 100755 index e8ac933..0000000 --- a/copyextfiles.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh - -mkdir -p stage2/01-sys-tweaks/extfiles - -# -# tools -# -cp tools/setuidgids stage2/01-sys-tweaks/extfiles/ -cp tools/_cscore.so stage2/01-sys-tweaks/extfiles/_cscore.cpython-35m-arm-linux-gnueabihf.so - -# -# openjdk -# -cp ../raspbian-openjdk/jdk_11.0.1-strip.tar.gz stage2/01-sys-tweaks/extfiles/ - -# -# thirdparty-opencv -# - -sh -c 'cd ../thirdparty-opencv/buildShared/linux-raspbian/lib && tar czf - libopencv*' > stage2/01-sys-tweaks/extfiles/libopencv.tar.gz - -cp ../thirdparty-opencv/buildShared/linux-raspbian/bin/opencv-*.jar stage2/01-sys-tweaks/extfiles/ - -# the opencv build names the python .so with the build platform name instead -# of the target platform, so rename it -cp ../thirdparty-opencv/buildShared/linux-raspbian/lib/python3/cv2.*.so stage2/01-sys-tweaks/extfiles/cv2.cpython-35m-arm-linux-gnueabihf.so - -# -# robotpy-cscore -# -sh -c 'cd ../robotpy-cscore/ && tar czf - cscore' > stage2/01-sys-tweaks/extfiles/robotpy-cscore.tar.gz - -# -# pynetworktables -# -sh -c 'cd ../pynetworktables/ && tar czf - networktables ntcore' > stage2/01-sys-tweaks/extfiles/pynetworktables.tar.gz - -# -# allwpilib -# - -cp \ - ../allwpilib/wpiutil/build/libs/wpiutil/shared/raspbian/release/libwpiutil.so* \ - ../allwpilib/wpiutil/build/libs/wpiutil/shared/raspbian/debug/libwpiutild.so* \ - ../allwpilib/wpiutil/build/libs/wpiutil.jar \ - ../allwpilib/cscore/build/libs/cscore/shared/raspbian/release/libcscore.so* \ - ../allwpilib/cscore/build/libs/cscore/shared/raspbian/debug/libcscored.so* \ - ../allwpilib/cscore/build/libs/cscoreJNIShared/shared/raspbian/release/libcscorejni.so* \ - ../allwpilib/cscore/build/libs/cscore.jar \ - ../allwpilib/ntcore/build/libs/ntcore/shared/raspbian/release/libntcore.so* \ - ../allwpilib/ntcore/build/libs/ntcore/shared/raspbian/debug/libntcored.so* \ - ../allwpilib/ntcore/build/libs/ntcoreJNIShared/shared/raspbian/release/libntcorejni.so* \ - ../allwpilib/ntcore/build/libs/ntcore.jar \ - ../allwpilib/cameraserver/build/libs/cameraserver/shared/raspbian/release/libcameraserver.so* \ - ../allwpilib/cameraserver/build/libs/cameraserver/shared/raspbian/debug/libcameraserverd.so* \ - ../allwpilib/cameraserver/build/libs/cameraserver.jar \ - stage2/01-sys-tweaks/extfiles/ - -sh -c 'cd ../allwpilib/wpiutil/src/main/native/include && tar czf - uv.h uv wpi' > stage2/01-sys-tweaks/extfiles/wpiutil-include.tar.gz -sh -c 'cd ../allwpilib/cscore/src/main/native/include && tar czf - .' > stage2/01-sys-tweaks/extfiles/cscore-include.tar.gz -sh -c 'cd ../allwpilib/ntcore/src/main/native/include && tar czf - .' > stage2/01-sys-tweaks/extfiles/ntcore-include.tar.gz -sh -c 'cd ../allwpilib/cameraserver/src/main/native/include && tar czf - cameraserver vision' > stage2/01-sys-tweaks/extfiles/cameraserver-include.tar.gz - -cp \ - ../allwpilib/cameraserver/build/exe/multiCameraServer/raspbian/multiCameraServer \ - ../allwpilib/wpiutil/build/exe/netconsoleTee/raspbian/netconsoleTee \ - ../allwpilib/wpiutil/build/exe/rpiConfigServer/raspbian/rpiConfigServer \ - stage2/01-sys-tweaks/extfiles/ diff --git a/deps/01-download.sh b/deps/01-download.sh new file mode 100755 index 0000000..a6a179f --- /dev/null +++ b/deps/01-download.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +mkdir -p 01-download +pushd 01-download + +# raspbian toolchain +wget -nc -nv \ + https://github.com/wpilibsuite/raspbian-toolchain/releases/download/v1.1.0/Raspbian9-Linux-Toolchain-6.3.0.tar.gz + +# openjdk binaries +wget -nc -nv \ + https://github.com/wpilibsuite/raspbian-openjdk/releases/download/v2019-11.0.1-1/jdk_11.0.1-strip.tar.gz + +# python headers/libs +wget -nc -nv \ + http://archive.raspbian.org/raspbian/pool/main/p/python3.5/libpython3.5_3.5.3-1+deb9u1_armhf.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/python3.5/libpython3.5-dev_3.5.3-1+deb9u1_armhf.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/python3.5/libpython3.5-minimal_3.5.3-1+deb9u1_armhf.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/python3.5/python3.5-dev_3.5.3-1+deb9u1_armhf.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/python3.5/python3.5-minimal_3.5.3-1+deb9u1_armhf.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/python3.5/python3.5_3.5.3-1+deb9u1_armhf.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/python-numpy/python3-numpy_1.12.1-3_armhf.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/pybind11/python3-pybind11_2.2.4-1_all.deb \ + http://archive.raspbian.org/raspbian/pool/main/p/pybind11/pybind11-dev_2.2.4-1_all.deb + +# opencv sources +wget -nc -nv \ + https://github.com/opencv/opencv/archive/3.4.4.tar.gz + +popd diff --git a/deps/02-extract.sh b/deps/02-extract.sh new file mode 100755 index 0000000..5f9d852 --- /dev/null +++ b/deps/02-extract.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +mkdir -p 02-extract +pushd 02-extract + +# raspbian toolchain +tar xzf ../01-download/Raspbian9-Linux-Toolchain-*.tar.gz + +# additional headers/libs +pushd raspbian9/arm-raspbian9-linux-gnueabihf + +# Extract data to toolchain basedir +for var in ../../../01-download/*.deb +do +ar p "$var" data.tar.xz | tar xJf - +done + +# move the arm-linux-gnueabihf libs to just the base "lib" +sh -c 'cd lib && ln -s arm-linux-gnueabihf/* .' +sh -c 'cd usr/lib/debug && ln -s ../arm-linux-gnueabihf/debug/* .' +sh -c 'cd usr/lib && ln -s arm-linux-gnueabihf/* .' + +# change absolute symlinks into relative symlinks +find . -lname '/*' | \ +while read l ; do + echo ln -sf $(echo $(echo $l | sed 's|/[^/]*|/..|g')$(readlink $l) | sed 's/.....//') $l +done | \ +sh + +popd + +# opencv sources +tar xzf ../01-download/3.4.4.tar.gz +mv opencv-3.4.4 opencv +cp ../thirdparty-opencv/arm-pi-gnueabihf.toolchain.cmake . +cp -R ../thirdparty-opencv/jni . + +popd diff --git a/deps/03-build.sh b/deps/03-build.sh new file mode 100755 index 0000000..db91ae0 --- /dev/null +++ b/deps/03-build.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +export PATH=${PWD}/02-extract/raspbian9/bin:${PATH} + +# opencv +mkdir -p 03-build/opencv-build +pushd 03-build/opencv-build +cmake ../../02-extract/opencv \ + -DWITH_CUDA=OFF \ + -DWITH_IPP=OFF \ + -DWITH_ITT=OFF \ + -DWITH_OPENCL=OFF \ + -DWITH_FFMPEG=OFF \ + -DWITH_OPENEXR=OFF \ + -DWITH_GSTREAMER=OFF \ + -DWITH_LAPACK=OFF \ + -DWITH_GTK=ON \ + -DWITH_1394=OFF \ + -DWITH_JASPER=OFF \ + -DWITH_TIFF=OFF \ + -DBUILD_JPEG=ON \ + -DBUILD_PNG=ON \ + -DBUILD_ZLIB=ON \ + -DBUILD_TESTS=OFF \ + -DPython_ADDITIONAL_VERSIONS=3.5 \ + -DWITH_WEBP=OFF \ + -DBUILD_JAVA=ON \ + -DBUILD_WITH_STATIC_CRT=OFF \ + -DWITH_PROTOBUF=OFF \ + -DWITH_DIRECTX=OFF \ + -DENABLE_CXX11=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=${PWD}/../../02-extract/arm-pi-gnueabihf.toolchain.cmake \ + -DCMAKE_MAKE_PROGRAM=make \ + -DENABLE_NEON=ON \ + -DENABLE_VFPV3=ON \ + -DBUILD_opencv_python3=ON \ + -DPYTHON3_INCLUDE_PATH=${PWD}/../../02-extract/raspbian9/arm-raspbian9-linux-gnueabihf/usr/include/python3.5m \ + -DPYTHON3_NUMPY_INCLUDE_DIRS=${PWD}/../../02-extract/raspbian9/arm-raspbian9-linux-gnueabihf/usr/include/python3.5m/numpy \ + -DOPENCV_EXTRA_FLAGS_DEBUG=-Og \ + -DCMAKE_MODULE_PATH=${PWD}/../../thirdparty-opencv/arm-frc-modules \ + || die +make -j3 || die +make install || die + +popd + +# wpiutil, cscore, ntcore, cameraserver +pushd allwpilib +./gradlew -PonlyRaspbian :wpiutil:build :cscore:build :ntcore:build :cameraserver:build || die +popd + +# tools +pushd tools +make || die +popd diff --git a/deps/04-copy.sh b/deps/04-copy.sh new file mode 100755 index 0000000..fac74e2 --- /dev/null +++ b/deps/04-copy.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +DEST=${PWD}/../stage2/01-sys-tweaks/extfiles + +mkdir -p ${DEST} + +# +# tools +# + +cp tools/setuidgids ${DEST}/ +cp tools/_cscore.so ${DEST}/_cscore.cpython-35m-arm-linux-gnueabihf.so + +# +# openjdk +# + +cp 01-download/jdk_11.0.1-strip.tar.gz ${DEST}/ + +# +# thirdparty-opencv +# + +sh -c 'cd 03-build/opencv-build/install/lib && tar czf - libopencv*' > ${DEST}/libopencv.tar.gz + +sh -c 'cd 03-build/opencv-build/install/include && tar czf - .' > ${DEST}/opencv-include.tar.gz + +cp 03-build/opencv-build/install/share/OpenCV/java/opencv-*.jar ${DEST}/ + +# the opencv build names the python .so with the build platform name instead +# of the target platform, so rename it +cp 03-build/opencv-build/install/python/cv2/python-*/cv2.*.so ${DEST}/cv2.cpython-35m-arm-linux-gnueabihf.so + +# +# robotpy-cscore +# + +sh -c 'cd robotpy-cscore && tar czf - cscore' > ${DEST}/robotpy-cscore.tar.gz + +# +# pynetworktables +# + +sh -c 'cd pynetworktables && tar czf - networktables ntcore' > ${DEST}/pynetworktables.tar.gz + +# +# allwpilib +# + +cp \ + allwpilib/wpiutil/build/libs/wpiutil/shared/release/libwpiutil.so* \ + allwpilib/wpiutil/build/libs/wpiutil/shared/debug/libwpiutild.so* \ + allwpilib/wpiutil/build/libs/wpiutil.jar \ + allwpilib/cscore/build/libs/cscore/shared/release/libcscore.so* \ + allwpilib/cscore/build/libs/cscore/shared/debug/libcscored.so* \ + allwpilib/cscore/build/libs/cscoreJNIShared/shared/release/libcscorejni.so* \ + allwpilib/cscore/build/libs/cscore.jar \ + allwpilib/ntcore/build/libs/ntcore/shared/release/libntcore.so* \ + allwpilib/ntcore/build/libs/ntcore/shared/debug/libntcored.so* \ + allwpilib/ntcore/build/libs/ntcoreJNIShared/shared/release/libntcorejni.so* \ + allwpilib/ntcore/build/libs/ntcore.jar \ + allwpilib/cameraserver/build/libs/cameraserver/shared/release/libcameraserver.so* \ + allwpilib/cameraserver/build/libs/cameraserver/shared/debug/libcameraserverd.so* \ + allwpilib/cameraserver/build/libs/cameraserver.jar \ + ${DEST}/ + +sh -c 'cd allwpilib/wpiutil/src/main/native/include && tar czf - uv.h uv wpi' > ${DEST}/wpiutil-include.tar.gz +sh -c 'cd allwpilib/cscore/src/main/native/include && tar czf - .' > ${DEST}/cscore-include.tar.gz +sh -c 'cd allwpilib/ntcore/src/main/native/include && tar czf - .' > ${DEST}/ntcore-include.tar.gz +sh -c 'cd allwpilib/cameraserver/src/main/native/include && tar czf - cameraserver vision' > ${DEST}/cameraserver-include.tar.gz + +cp \ + allwpilib/cameraserver/build/exe/multiCameraServer/multiCameraServer \ + allwpilib/wpiutil/build/exe/netconsoleTee/netconsoleTee \ + allwpilib/wpiutil/build/exe/rpiConfigServer/rpiConfigServer \ + ${DEST}/ diff --git a/deps/allwpilib b/deps/allwpilib new file mode 160000 index 0000000..26e8e58 --- /dev/null +++ b/deps/allwpilib @@ -0,0 +1 @@ +Subproject commit 26e8e587f9264b7d22b65a6b2934e04c25871349 diff --git a/deps/pynetworktables b/deps/pynetworktables new file mode 160000 index 0000000..59b5d13 --- /dev/null +++ b/deps/pynetworktables @@ -0,0 +1 @@ +Subproject commit 59b5d1328274e78393202fe2a7e09845f5ff63a5 diff --git a/deps/robotpy-cscore b/deps/robotpy-cscore new file mode 160000 index 0000000..efc857a --- /dev/null +++ b/deps/robotpy-cscore @@ -0,0 +1 @@ +Subproject commit efc857aeb40168cb79e82b572fab2f051456980c diff --git a/deps/thirdparty-opencv b/deps/thirdparty-opencv new file mode 160000 index 0000000..579bc6b --- /dev/null +++ b/deps/thirdparty-opencv @@ -0,0 +1 @@ +Subproject commit 579bc6b26169c690bdadb52150d4396e696eed31 diff --git a/deps/tools/Makefile b/deps/tools/Makefile new file mode 100644 index 0000000..233f20e --- /dev/null +++ b/deps/tools/Makefile @@ -0,0 +1,21 @@ +COMPILER=../02-extract/raspbian9/bin/arm-raspbian9-linux-gnueabihf- + +ALL: setuidgids _cscore.so + +setuidgids: setuidgids.c + ${COMPILER}gcc -O -Wall -D_GNU_SOURCE -o $@ $< + +_cscore.so: ../robotpy-cscore/src/_cscore.cpp ../robotpy-cscore/src/ndarray_converter.cpp + ${COMPILER}g++ -O -Wall -fvisibility=hidden -shared -fPIC -o $@ \ + -I../03-build/opencv-build/install/include \ + -I../allwpilib/wpiutil/src/main/native/include \ + -I../allwpilib/cscore/src/main/native/include \ + -I../02-extract/raspbian9/arm-raspbian9-linux-gnueabihf/usr/include/python3.5 \ + -L../allwpilib/cscore/build/libs/cscore/shared/release \ + -L../allwpilib/wpiutil/build/libs/wpiutil/shared/release \ + -L../03-build/opencv-build/install/lib \ + ../robotpy-cscore/src/_cscore.cpp \ + ../robotpy-cscore/src/ndarray_converter.cpp \ + -lcscore \ + -lwpiutil \ + -lopencv_highgui -lopencv_imgcodecs -lopencv_imgproc -lopencv_core diff --git a/tools/setuidgids.c b/deps/tools/setuidgids.c similarity index 100% rename from tools/setuidgids.c rename to deps/tools/setuidgids.c diff --git a/tools/Makefile b/tools/Makefile deleted file mode 100644 index 9063e34..0000000 --- a/tools/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -ALL: setuidgids _cscore.so - -setuidgids: setuidgids.c - arm-raspbian9-linux-gnueabihf-gcc -O -Wall -D_GNU_SOURCE -o $@ $< - -_cscore.so: ../../robotpy-cscore/src/_cscore.cpp ../../robotpy-cscore/src/ndarray_converter.cpp - arm-raspbian9-linux-gnueabihf-g++ -O -Wall -fvisibility=hidden -shared -fPIC -o $@ \ - -I../../thirdparty-opencv/opencv/include \ - -I../../thirdparty-opencv/opencv/modules/core/include \ - -I../../thirdparty-opencv/opencv/modules/calib3d/include \ - -I../../thirdparty-opencv/opencv/modules/features2d/include \ - -I../../thirdparty-opencv/opencv/modules/flann/include \ - -I../../thirdparty-opencv/opencv/modules/highgui/include \ - -I../../thirdparty-opencv/opencv/modules/imgcodecs/include \ - -I../../thirdparty-opencv/opencv/modules/imgproc/include \ - -I../../thirdparty-opencv/opencv/modules/ml/include \ - -I../../thirdparty-opencv/opencv/modules/objdetect/include \ - -I../../thirdparty-opencv/opencv/modules/photo/include \ - -I../../thirdparty-opencv/opencv/modules/shape/include \ - -I../../thirdparty-opencv/opencv/modules/stitching/include \ - -I../../thirdparty-opencv/opencv/modules/superres/include \ - -I../../thirdparty-opencv/opencv/modules/video/include \ - -I../../thirdparty-opencv/opencv/modules/videoio/include \ - -I../../thirdparty-opencv/opencv/modules/videostab/include \ - -I../../thirdparty-opencv/buildShared/linux-raspbian \ - -I../../allwpilib/wpiutil/src/main/native/include \ - -I../../allwpilib/cscore/src/main/native/include \ - -I/usr/local/arm-raspbian9-linux-gnueabihf/usr/include/python3.5 \ - -L../../allwpilib/cscore/build/libs/cscore/shared/raspbian/release \ - -L../../allwpilib/wpiutil/build/libs/wpiutil/shared/raspbian/release \ - -L../../thirdparty-opencv/buildShared/linux-raspbian/lib \ - ../../robotpy-cscore/src/_cscore.cpp \ - ../../robotpy-cscore/src/ndarray_converter.cpp \ - -lcscore \ - -lwpiutil \ - -lopencv_highgui -lopencv_imgcodecs -lopencv_imgproc -lopencv_core