Build external dependencies local to tree

pull/229/head
Peter Johnson 2018-12-02 00:21:42 -08:00
parent 4632d11a1e
commit 1768bdaa4c
No known key found for this signature in database
GPG Key ID: D39DD4DA7D41E329
13 changed files with 238 additions and 104 deletions

12
.gitmodules vendored Normal file
View File

@ -0,0 +1,12 @@
[submodule "deps/robotpy-cscore"]
path = deps/robotpy-cscore
url = git@github.com:robotpy/robotpy-cscore.git
[submodule "deps/pynetworktables"]
path = deps/pynetworktables
url = git@github.com:robotpy/pynetworktables.git
[submodule "deps/allwpilib"]
path = deps/allwpilib
url = git@github.com:wpilibsuite/allwpilib.git
[submodule "deps/thirdparty-opencv"]
path = deps/thirdparty-opencv
url = git@github.com:wpilibsuite/thirdparty-opencv.git

View File

@ -1,68 +0,0 @@
#!/bin/sh
mkdir -p stage2/01-sys-tweaks/extfiles
#
# tools
#
cp tools/setuidgids stage2/01-sys-tweaks/extfiles/
cp tools/_cscore.so stage2/01-sys-tweaks/extfiles/_cscore.cpython-35m-arm-linux-gnueabihf.so
#
# openjdk
#
cp ../raspbian-openjdk/jdk_11.0.1-strip.tar.gz stage2/01-sys-tweaks/extfiles/
#
# thirdparty-opencv
#
sh -c 'cd ../thirdparty-opencv/buildShared/linux-raspbian/lib && tar czf - libopencv*' > stage2/01-sys-tweaks/extfiles/libopencv.tar.gz
cp ../thirdparty-opencv/buildShared/linux-raspbian/bin/opencv-*.jar stage2/01-sys-tweaks/extfiles/
# the opencv build names the python .so with the build platform name instead
# of the target platform, so rename it
cp ../thirdparty-opencv/buildShared/linux-raspbian/lib/python3/cv2.*.so stage2/01-sys-tweaks/extfiles/cv2.cpython-35m-arm-linux-gnueabihf.so
#
# robotpy-cscore
#
sh -c 'cd ../robotpy-cscore/ && tar czf - cscore' > stage2/01-sys-tweaks/extfiles/robotpy-cscore.tar.gz
#
# pynetworktables
#
sh -c 'cd ../pynetworktables/ && tar czf - networktables ntcore' > stage2/01-sys-tweaks/extfiles/pynetworktables.tar.gz
#
# allwpilib
#
cp \
../allwpilib/wpiutil/build/libs/wpiutil/shared/raspbian/release/libwpiutil.so* \
../allwpilib/wpiutil/build/libs/wpiutil/shared/raspbian/debug/libwpiutild.so* \
../allwpilib/wpiutil/build/libs/wpiutil.jar \
../allwpilib/cscore/build/libs/cscore/shared/raspbian/release/libcscore.so* \
../allwpilib/cscore/build/libs/cscore/shared/raspbian/debug/libcscored.so* \
../allwpilib/cscore/build/libs/cscoreJNIShared/shared/raspbian/release/libcscorejni.so* \
../allwpilib/cscore/build/libs/cscore.jar \
../allwpilib/ntcore/build/libs/ntcore/shared/raspbian/release/libntcore.so* \
../allwpilib/ntcore/build/libs/ntcore/shared/raspbian/debug/libntcored.so* \
../allwpilib/ntcore/build/libs/ntcoreJNIShared/shared/raspbian/release/libntcorejni.so* \
../allwpilib/ntcore/build/libs/ntcore.jar \
../allwpilib/cameraserver/build/libs/cameraserver/shared/raspbian/release/libcameraserver.so* \
../allwpilib/cameraserver/build/libs/cameraserver/shared/raspbian/debug/libcameraserverd.so* \
../allwpilib/cameraserver/build/libs/cameraserver.jar \
stage2/01-sys-tweaks/extfiles/
sh -c 'cd ../allwpilib/wpiutil/src/main/native/include && tar czf - uv.h uv wpi' > stage2/01-sys-tweaks/extfiles/wpiutil-include.tar.gz
sh -c 'cd ../allwpilib/cscore/src/main/native/include && tar czf - .' > stage2/01-sys-tweaks/extfiles/cscore-include.tar.gz
sh -c 'cd ../allwpilib/ntcore/src/main/native/include && tar czf - .' > stage2/01-sys-tweaks/extfiles/ntcore-include.tar.gz
sh -c 'cd ../allwpilib/cameraserver/src/main/native/include && tar czf - cameraserver vision' > stage2/01-sys-tweaks/extfiles/cameraserver-include.tar.gz
cp \
../allwpilib/cameraserver/build/exe/multiCameraServer/raspbian/multiCameraServer \
../allwpilib/wpiutil/build/exe/netconsoleTee/raspbian/netconsoleTee \
../allwpilib/wpiutil/build/exe/rpiConfigServer/raspbian/rpiConfigServer \
stage2/01-sys-tweaks/extfiles/

30
deps/01-download.sh vendored Executable file
View File

@ -0,0 +1,30 @@
#!/bin/bash
mkdir -p 01-download
pushd 01-download
# raspbian toolchain
wget -nc -nv \
https://github.com/wpilibsuite/raspbian-toolchain/releases/download/v1.1.0/Raspbian9-Linux-Toolchain-6.3.0.tar.gz
# openjdk binaries
wget -nc -nv \
https://github.com/wpilibsuite/raspbian-openjdk/releases/download/v2019-11.0.1-1/jdk_11.0.1-strip.tar.gz
# python headers/libs
wget -nc -nv \
http://archive.raspbian.org/raspbian/pool/main/p/python3.5/libpython3.5_3.5.3-1+deb9u1_armhf.deb \
http://archive.raspbian.org/raspbian/pool/main/p/python3.5/libpython3.5-dev_3.5.3-1+deb9u1_armhf.deb \
http://archive.raspbian.org/raspbian/pool/main/p/python3.5/libpython3.5-minimal_3.5.3-1+deb9u1_armhf.deb \
http://archive.raspbian.org/raspbian/pool/main/p/python3.5/python3.5-dev_3.5.3-1+deb9u1_armhf.deb \
http://archive.raspbian.org/raspbian/pool/main/p/python3.5/python3.5-minimal_3.5.3-1+deb9u1_armhf.deb \
http://archive.raspbian.org/raspbian/pool/main/p/python3.5/python3.5_3.5.3-1+deb9u1_armhf.deb \
http://archive.raspbian.org/raspbian/pool/main/p/python-numpy/python3-numpy_1.12.1-3_armhf.deb \
http://archive.raspbian.org/raspbian/pool/main/p/pybind11/python3-pybind11_2.2.4-1_all.deb \
http://archive.raspbian.org/raspbian/pool/main/p/pybind11/pybind11-dev_2.2.4-1_all.deb
# opencv sources
wget -nc -nv \
https://github.com/opencv/opencv/archive/3.4.4.tar.gz
popd

38
deps/02-extract.sh vendored Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
mkdir -p 02-extract
pushd 02-extract
# raspbian toolchain
tar xzf ../01-download/Raspbian9-Linux-Toolchain-*.tar.gz
# additional headers/libs
pushd raspbian9/arm-raspbian9-linux-gnueabihf
# Extract data to toolchain basedir
for var in ../../../01-download/*.deb
do
ar p "$var" data.tar.xz | tar xJf -
done
# move the arm-linux-gnueabihf libs to just the base "lib"
sh -c 'cd lib && ln -s arm-linux-gnueabihf/* .'
sh -c 'cd usr/lib/debug && ln -s ../arm-linux-gnueabihf/debug/* .'
sh -c 'cd usr/lib && ln -s arm-linux-gnueabihf/* .'
# change absolute symlinks into relative symlinks
find . -lname '/*' | \
while read l ; do
echo ln -sf $(echo $(echo $l | sed 's|/[^/]*|/..|g')$(readlink $l) | sed 's/.....//') $l
done | \
sh
popd
# opencv sources
tar xzf ../01-download/3.4.4.tar.gz
mv opencv-3.4.4 opencv
cp ../thirdparty-opencv/arm-pi-gnueabihf.toolchain.cmake .
cp -R ../thirdparty-opencv/jni .
popd

57
deps/03-build.sh vendored Executable file
View File

@ -0,0 +1,57 @@
#!/bin/bash
export PATH=${PWD}/02-extract/raspbian9/bin:${PATH}
# opencv
mkdir -p 03-build/opencv-build
pushd 03-build/opencv-build
cmake ../../02-extract/opencv \
-DWITH_CUDA=OFF \
-DWITH_IPP=OFF \
-DWITH_ITT=OFF \
-DWITH_OPENCL=OFF \
-DWITH_FFMPEG=OFF \
-DWITH_OPENEXR=OFF \
-DWITH_GSTREAMER=OFF \
-DWITH_LAPACK=OFF \
-DWITH_GTK=ON \
-DWITH_1394=OFF \
-DWITH_JASPER=OFF \
-DWITH_TIFF=OFF \
-DBUILD_JPEG=ON \
-DBUILD_PNG=ON \
-DBUILD_ZLIB=ON \
-DBUILD_TESTS=OFF \
-DPython_ADDITIONAL_VERSIONS=3.5 \
-DWITH_WEBP=OFF \
-DBUILD_JAVA=ON \
-DBUILD_WITH_STATIC_CRT=OFF \
-DWITH_PROTOBUF=OFF \
-DWITH_DIRECTX=OFF \
-DENABLE_CXX11=ON \
-DBUILD_SHARED_LIBS=ON \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_TOOLCHAIN_FILE=${PWD}/../../02-extract/arm-pi-gnueabihf.toolchain.cmake \
-DCMAKE_MAKE_PROGRAM=make \
-DENABLE_NEON=ON \
-DENABLE_VFPV3=ON \
-DBUILD_opencv_python3=ON \
-DPYTHON3_INCLUDE_PATH=${PWD}/../../02-extract/raspbian9/arm-raspbian9-linux-gnueabihf/usr/include/python3.5m \
-DPYTHON3_NUMPY_INCLUDE_DIRS=${PWD}/../../02-extract/raspbian9/arm-raspbian9-linux-gnueabihf/usr/include/python3.5m/numpy \
-DOPENCV_EXTRA_FLAGS_DEBUG=-Og \
-DCMAKE_MODULE_PATH=${PWD}/../../thirdparty-opencv/arm-frc-modules \
|| die
make -j3 || die
make install || die
popd
# wpiutil, cscore, ntcore, cameraserver
pushd allwpilib
./gradlew -PonlyRaspbian :wpiutil:build :cscore:build :ntcore:build :cameraserver:build || die
popd
# tools
pushd tools
make || die
popd

76
deps/04-copy.sh vendored Executable file
View File

@ -0,0 +1,76 @@
#!/bin/sh
DEST=${PWD}/../stage2/01-sys-tweaks/extfiles
mkdir -p ${DEST}
#
# tools
#
cp tools/setuidgids ${DEST}/
cp tools/_cscore.so ${DEST}/_cscore.cpython-35m-arm-linux-gnueabihf.so
#
# openjdk
#
cp 01-download/jdk_11.0.1-strip.tar.gz ${DEST}/
#
# thirdparty-opencv
#
sh -c 'cd 03-build/opencv-build/install/lib && tar czf - libopencv*' > ${DEST}/libopencv.tar.gz
sh -c 'cd 03-build/opencv-build/install/include && tar czf - .' > ${DEST}/opencv-include.tar.gz
cp 03-build/opencv-build/install/share/OpenCV/java/opencv-*.jar ${DEST}/
# the opencv build names the python .so with the build platform name instead
# of the target platform, so rename it
cp 03-build/opencv-build/install/python/cv2/python-*/cv2.*.so ${DEST}/cv2.cpython-35m-arm-linux-gnueabihf.so
#
# robotpy-cscore
#
sh -c 'cd robotpy-cscore && tar czf - cscore' > ${DEST}/robotpy-cscore.tar.gz
#
# pynetworktables
#
sh -c 'cd pynetworktables && tar czf - networktables ntcore' > ${DEST}/pynetworktables.tar.gz
#
# allwpilib
#
cp \
allwpilib/wpiutil/build/libs/wpiutil/shared/release/libwpiutil.so* \
allwpilib/wpiutil/build/libs/wpiutil/shared/debug/libwpiutild.so* \
allwpilib/wpiutil/build/libs/wpiutil.jar \
allwpilib/cscore/build/libs/cscore/shared/release/libcscore.so* \
allwpilib/cscore/build/libs/cscore/shared/debug/libcscored.so* \
allwpilib/cscore/build/libs/cscoreJNIShared/shared/release/libcscorejni.so* \
allwpilib/cscore/build/libs/cscore.jar \
allwpilib/ntcore/build/libs/ntcore/shared/release/libntcore.so* \
allwpilib/ntcore/build/libs/ntcore/shared/debug/libntcored.so* \
allwpilib/ntcore/build/libs/ntcoreJNIShared/shared/release/libntcorejni.so* \
allwpilib/ntcore/build/libs/ntcore.jar \
allwpilib/cameraserver/build/libs/cameraserver/shared/release/libcameraserver.so* \
allwpilib/cameraserver/build/libs/cameraserver/shared/debug/libcameraserverd.so* \
allwpilib/cameraserver/build/libs/cameraserver.jar \
${DEST}/
sh -c 'cd allwpilib/wpiutil/src/main/native/include && tar czf - uv.h uv wpi' > ${DEST}/wpiutil-include.tar.gz
sh -c 'cd allwpilib/cscore/src/main/native/include && tar czf - .' > ${DEST}/cscore-include.tar.gz
sh -c 'cd allwpilib/ntcore/src/main/native/include && tar czf - .' > ${DEST}/ntcore-include.tar.gz
sh -c 'cd allwpilib/cameraserver/src/main/native/include && tar czf - cameraserver vision' > ${DEST}/cameraserver-include.tar.gz
cp \
allwpilib/cameraserver/build/exe/multiCameraServer/multiCameraServer \
allwpilib/wpiutil/build/exe/netconsoleTee/netconsoleTee \
allwpilib/wpiutil/build/exe/rpiConfigServer/rpiConfigServer \
${DEST}/

1
deps/allwpilib vendored Submodule

@ -0,0 +1 @@
Subproject commit 26e8e587f9264b7d22b65a6b2934e04c25871349

1
deps/pynetworktables vendored Submodule

@ -0,0 +1 @@
Subproject commit 59b5d1328274e78393202fe2a7e09845f5ff63a5

1
deps/robotpy-cscore vendored Submodule

@ -0,0 +1 @@
Subproject commit efc857aeb40168cb79e82b572fab2f051456980c

1
deps/thirdparty-opencv vendored Submodule

@ -0,0 +1 @@
Subproject commit 579bc6b26169c690bdadb52150d4396e696eed31

21
deps/tools/Makefile vendored Normal file
View File

@ -0,0 +1,21 @@
COMPILER=../02-extract/raspbian9/bin/arm-raspbian9-linux-gnueabihf-
ALL: setuidgids _cscore.so
setuidgids: setuidgids.c
${COMPILER}gcc -O -Wall -D_GNU_SOURCE -o $@ $<
_cscore.so: ../robotpy-cscore/src/_cscore.cpp ../robotpy-cscore/src/ndarray_converter.cpp
${COMPILER}g++ -O -Wall -fvisibility=hidden -shared -fPIC -o $@ \
-I../03-build/opencv-build/install/include \
-I../allwpilib/wpiutil/src/main/native/include \
-I../allwpilib/cscore/src/main/native/include \
-I../02-extract/raspbian9/arm-raspbian9-linux-gnueabihf/usr/include/python3.5 \
-L../allwpilib/cscore/build/libs/cscore/shared/release \
-L../allwpilib/wpiutil/build/libs/wpiutil/shared/release \
-L../03-build/opencv-build/install/lib \
../robotpy-cscore/src/_cscore.cpp \
../robotpy-cscore/src/ndarray_converter.cpp \
-lcscore \
-lwpiutil \
-lopencv_highgui -lopencv_imgcodecs -lopencv_imgproc -lopencv_core

View File

@ -1,36 +0,0 @@
ALL: setuidgids _cscore.so
setuidgids: setuidgids.c
arm-raspbian9-linux-gnueabihf-gcc -O -Wall -D_GNU_SOURCE -o $@ $<
_cscore.so: ../../robotpy-cscore/src/_cscore.cpp ../../robotpy-cscore/src/ndarray_converter.cpp
arm-raspbian9-linux-gnueabihf-g++ -O -Wall -fvisibility=hidden -shared -fPIC -o $@ \
-I../../thirdparty-opencv/opencv/include \
-I../../thirdparty-opencv/opencv/modules/core/include \
-I../../thirdparty-opencv/opencv/modules/calib3d/include \
-I../../thirdparty-opencv/opencv/modules/features2d/include \
-I../../thirdparty-opencv/opencv/modules/flann/include \
-I../../thirdparty-opencv/opencv/modules/highgui/include \
-I../../thirdparty-opencv/opencv/modules/imgcodecs/include \
-I../../thirdparty-opencv/opencv/modules/imgproc/include \
-I../../thirdparty-opencv/opencv/modules/ml/include \
-I../../thirdparty-opencv/opencv/modules/objdetect/include \
-I../../thirdparty-opencv/opencv/modules/photo/include \
-I../../thirdparty-opencv/opencv/modules/shape/include \
-I../../thirdparty-opencv/opencv/modules/stitching/include \
-I../../thirdparty-opencv/opencv/modules/superres/include \
-I../../thirdparty-opencv/opencv/modules/video/include \
-I../../thirdparty-opencv/opencv/modules/videoio/include \
-I../../thirdparty-opencv/opencv/modules/videostab/include \
-I../../thirdparty-opencv/buildShared/linux-raspbian \
-I../../allwpilib/wpiutil/src/main/native/include \
-I../../allwpilib/cscore/src/main/native/include \
-I/usr/local/arm-raspbian9-linux-gnueabihf/usr/include/python3.5 \
-L../../allwpilib/cscore/build/libs/cscore/shared/raspbian/release \
-L../../allwpilib/wpiutil/build/libs/wpiutil/shared/raspbian/release \
-L../../thirdparty-opencv/buildShared/linux-raspbian/lib \
../../robotpy-cscore/src/_cscore.cpp \
../../robotpy-cscore/src/ndarray_converter.cpp \
-lcscore \
-lwpiutil \
-lopencv_highgui -lopencv_imgcodecs -lopencv_imgproc -lopencv_core