From 4ae8f48d20b3f706ef9d0365584f1bf1b13c5cac Mon Sep 17 00:00:00 2001 From: zador-blood-stained Date: Wed, 7 Sep 2016 19:52:31 +0300 Subject: [PATCH] Use distcc for cross-compiling --- chroot-buildpackages.sh | 76 +++++++++++++++++++++++++++++++++-------- common.sh | 6 ++-- general.sh | 16 +++------ 3 files changed, 71 insertions(+), 27 deletions(-) diff --git a/chroot-buildpackages.sh b/chroot-buildpackages.sh index 48b684610..331776687 100644 --- a/chroot-buildpackages.sh +++ b/chroot-buildpackages.sh @@ -10,7 +10,7 @@ # # Functions: # create_chroot -# update_chroot +# chroot_prepare_distccd # chroot_build_packages # chroot_installpackages_local # chroot_installpackages @@ -22,15 +22,16 @@ create_chroot() local target_dir="$1" local release=$2 local arch=$3 - declare -A qemu_binary + declare -A qemu_binary apt_mirror components qemu_binary['armhf']='qemu-arm-static' qemu_binary['arm64']='qemu-aarch64-static' - declare -A apt_mirror apt_mirror['jessie']='httpredir.debian.org/debian' apt_mirror['xenial']='ports.ubuntu.com' + components['jessie']='main,contrib' + components['xenial']='main,universe,multiverse' display_alert "Creating build chroot" "$release $arch" "info" - local includes="ccache,locales,git,ca-certificates,devscripts,libfile-fcntllock-perl,debhelper,rsync,python3" - debootstrap --variant=buildd --arch=$arch --foreign --include="$includes" $release $target_dir "http://localhost:3142/${apt_mirror[$release]}" + local includes="ccache,locales,git,ca-certificates,devscripts,libfile-fcntllock-perl,debhelper,rsync,python3,distcc" + debootstrap --variant=buildd --components=${components[$release]} --arch=$arch --foreign --include="$includes" $release $target_dir "http://localhost:3142/${apt_mirror[$release]}" [[ $? -ne 0 || ! -f $target_dir/debootstrap/debootstrap ]] && exit_with_error "Create chroot first stage failed" cp /usr/bin/${qemu_binary[$arch]} $target_dir/usr/bin/ [[ ! -f $target_dir/usr/share/keyrings/debian-archive-keyring.gpg ]] && \ @@ -57,10 +58,46 @@ create_chroot() rm -rf $target_dir/var/lock 2>/dev/null mkdir -p $target_dir/var/lock fi + chroot $target_dir /bin/bash -c "/usr/sbin/update-ccache-symlinks" touch $target_dir/root/.debootstrap-complete display_alert "Debootstrap complete" "$release $arch" "info" } ############################################################################# + +# chroot_prepare_distccd +# +chroot_prepare_distccd() +{ + local release=$1 + local arch=$2 + local dest=$DEST/buildpkg/distcc-wrappers/${release}-${arch} + declare -A gcc_version gcc_type + gcc_version['jessie']='4.9' + gcc_version['xenial']='5' + gcc_type['armhf']='arm-linux-gnueabihf' + gcc_type['arm64']='aarch64-linux-gnu' + mkdir -p $dest + rm -f $dest/cmdlist + for compiler in gcc cpp g++; do + echo "$dest/$compiler" >> $dest/cmdlist + ln -sf /usr/bin/${gcc_type[$arch]}-${compiler}-${gcc_version[$release]} $dest/$compiler + echo "$dest/${gcc_type[$arch]}-${compiler}" >> $dest/cmdlist + ln -sf /usr/bin/${gcc_type[$arch]}-${compiler}-${gcc_version[$release]} $dest/${gcc_type[$arch]}-${compiler} + done + ln -sf /usr/bin/${gcc_type[$arch]}-gcc-${gcc_version[$release]} $dest/cc + echo "$dest/cc" >> $dest/cmdlist + ln -sf /usr/bin/${gcc_type[$arch]}-g++-${gcc_version[$release]} $dest/c++ + echo "$dest/c++" >> $dest/cmdlist + mkdir -p /var/run/distcc/ + touch /var/run/distcc/${release}-${arch}.pid + touch /tmp/distcc-${release}-${arch}.log + mkdir -p /tmp/distcc + chown distccd /var/run/distcc/ + chown distccd /var/run/distcc/${release}-${arch}.pid + chown distccd /tmp/distcc-${release}-${arch}.log + chown distccd /tmp/distcc +} + # chroot_build_packages # chroot_build_packages() @@ -69,13 +106,18 @@ chroot_build_packages() for arch in armhf arm64; do display_alert "Starting package building process" "$release $arch" "info" - local target_dir=$DEST/buildpkg/${release}-${arch} - # to avoid conflicts between published and self-built packages higher pin-priority may be enough - # or may use hostname or other unique identifier or smth like builddate=$(date +"%Y%m%d") + local target_dir=$DEST/buildpkg/${release}-${arch}-v2 + local distcc_bindaddr="127.0.0.2" [[ ! -f $target_dir/root/.debootstrap-complete ]] && create_chroot "$target_dir" "$release" "$arch" [[ ! -f $target_dir/root/.debootstrap-complete ]] && exit_with_error "Creating chroot failed" "$release" + chroot_prepare_distccd $release $arch + + DISTCC_CMDLIST=$DEST/buildpkg/distcc-wrappers/${release}-${arch}/cmdlist TMPDIR=/tmp/distcc distccd --daemon \ + --pid-file /var/run/distcc/${release}-${arch}.pid --listen $distcc_bindaddr --allow 127.0.0.0/24 \ + --log-file /tmp/distcc-${release}-${arch}.log --user distccd + local t=$target_dir/root/.update-timestamp if [[ ! -f $t || $(( ($(date +%s) - $(<$t)) / 86400 )) -gt 7 ]]; then display_alert "Upgrading packages" "$release $arch" "info" @@ -114,9 +156,8 @@ chroot_build_packages() continue fi display_alert "Building packages" "$package_name $release $arch" "ext" - - [[ -v package_builddeps_${release} && -n $package_builddeps_${release} ]] && \ - package_builddeps="$package_builddeps $package_builddeps_${release}" + local dist_builddeps_name="package_builddeps_${release}" + [[ -v $dist_builddeps_name ]] && package_builddeps="$package_builddeps ${!dist_builddeps_name}" # create build script cat <<-EOF > $target_dir/root/build.sh @@ -124,8 +165,13 @@ chroot_build_packages() export PATH="/usr/lib/ccache:\$PATH" export HOME="/root" export DEBIAN_FRONTEND="noninteractive" - export DEB_BUILD_OPTIONS="ccache nocheck" + export DEB_BUILD_OPTIONS="nocheck" export CCACHE_TEMPDIR="/tmp" + export CCACHE_PREFIX="distcc" + # uncomment for debug + #export CCACHE_RECACHE="true" + export DISTCC_HOSTS="$distcc_bindaddr" + export DISTCC_TCP_CORK="0" export DEBFULLNAME="$MAINTAINER" export DEBEMAIL="$MAINTAINERMAIL" $(declare -f display_alert) @@ -135,7 +181,7 @@ chroot_build_packages() deps=() installed=\$(dpkg-query -W -f '\${db:Status-Abbrev}|\${binary:Package}\n' '*' 2>/dev/null | grep '^ii' | awk -F '|' '{print \$2}' | cut -d ':' -f 1) for packet in $package_builddeps; do grep -q -x -e "\$packet" <<< "\$installed" || deps+=("\$packet"); done - [[ \${#deps[@]} -gt 0 ]] && apt-get -y --no-install-recommends --show-progress -o DPKG::Progress-Fancy=1 install "\${deps[@]}" + [[ \${#deps[@]} -gt 0 ]] && apt-get -y -q --no-install-recommends --show-progress -o DPKG::Progress-Fancy=1 install "\${deps[@]}" fi display_alert "Copying sources" rsync -aq /root/sources/$package_name /root/build/ @@ -148,7 +194,7 @@ chroot_build_packages() # debchange -l~armbian${REVISION}-${builddate}+ "New Armbian release" debchange -l~armbian${REVISION}+ "New Armbian release" display_alert "Building package" - dpkg-buildpackage -b -uc -us -jauto + dpkg-buildpackage -b -uc -us -j2 if [[ \$? -eq 0 ]]; then cd /root/build # install in chroot if other libraries depend on them @@ -176,6 +222,8 @@ chroot_build_packages() ${PROGRESS_LOG_TO_FILE:+' | tee -a $DEST/debug/buildpkg.log'} mv $target_dir/root/*.deb $plugin_target_dir 2>/dev/null done + # cleanup for distcc + kill $(/dev/null | grep '^ii' | awk -F '|' '{print $2}' | cut -d ':' -f 1) @@ -540,6 +533,8 @@ prepare_host() { ${PROGRESS_LOG_TO_FILE:+' | tee -a $DEST/debug/output.log'} \ ${OUTPUT_DIALOG:+' | dialog --backtitle "$backtitle" --progressbox "Installing ${#deps[@]} host dependencies..." $TTY_Y $TTY_X'} \ ${OUTPUT_VERYSILENT:+' >/dev/null 2>/dev/null'} + # this is needed in case new compilers were installed + update-ccache-symlinks fi # install aptly separately @@ -548,7 +543,6 @@ prepare_host() { fi # TODO: Check for failed installation process - # test exit code propagation for commands in parentheses # enable arm binary format so that the cross-architecture chroot environment will work test -e /proc/sys/fs/binfmt_misc/qemu-arm || update-binfmts --enable qemu-arm