build/lib/build-all-ng.sh
Igor Pečovnik 98642b8581
Set image compression levels to maximum (#2491)
* Set image compression levels to maximum

This feature is used mainly by us or other distributors. An internal function. Compression time and CPU usage for this process is significantly increased but the size of the daily batch was decreased from 80Gb -> 50Gb. Mitigate upload speed bottleneck.

* Small improvement for internal build process

* Typo
2020-12-22 14:41:51 +01:00

463 lines
15 KiB
Bash

#!/bin/bash
#
# Copyright (c) Authors: http://www.armbian.com/authors
#
# This file is licensed under the terms of the GNU General Public
# License version 2. This program is licensed "as is" without any
# warranty of any kind, whether express or implied.
# This file is a part of the Armbian build script
# https://github.com/armbian/build/
# Functions:
# pack_upload
# build_main
# array_contains
# build_all
if [[ $BETA == "yes" ]]; then STABILITY="beta"; else STABILITY="stable"; fi
if [[ $MAKE_ALL_BETA == "yes" ]]; then STABILITY="stable"; fi
if [[ -z $KERNEL_ONLY ]]; then KERNEL_ONLY="yes"; fi
if [[ -z $MULTITHREAD ]]; then MULTITHREAD=0; fi
if [[ -z $START ]]; then START=0; fi
if [[ -z $KERNEL_CONFIGURE ]]; then KERNEL_CONFIGURE="no"; fi
if [[ -z $CLEAN_LEVEL ]]; then CLEAN_LEVEL="make,oldcache"; fi
MAINLINE_KERNEL_SOURCE='https://kernel.googlesource.com/pub/scm/linux/kernel/git/stable/linux-stable'
# cleanup
rm -f /run/armbian/*.pid
mkdir -p /run/armbian
# read user defined targets if exits
if [[ -f $USERPATCHES_PATH/targets.conf ]]; then
display_alert "Adding user provided targets configuration"
BUILD_TARGETS="${USERPATCHES_PATH}/targets.conf"
else
BUILD_TARGETS="${SRC}/config/targets.conf"
fi
unset_all ()
{
unset LINUXFAMILY LINUXCONFIG KERNELDIR KERNELSOURCE KERNELBRANCH BOOTDIR BOOTSOURCE BOOTBRANCH ARCH UBOOT_USE_GCC \
KERNEL_USE_GCC CPUMIN CPUMAX UBOOT_VER KERNEL_VER GOVERNOR BOOTSIZE BOOTFS_TYPE UBOOT_TOOLCHAIN KERNEL_TOOLCHAIN \
DEBOOTSTRAP_LIST PACKAGE_LIST_EXCLUDE KERNEL_IMAGE_TYPE write_uboot_platform family_tweaks family_tweaks_bsp \
setup_write_uboot_platform uboot_custom_postprocess atf_custom_postprocess family_tweaks_s BOOTSCRIPT \
UBOOT_TARGET_MAP LOCALVERSION UBOOT_COMPILER KERNEL_COMPILER BOOTCONFIG BOOTCONFIG_VAR_NAME INITRD_ARCH BOOTENV_FILE BOOTDELAY \
ATF_TOOLCHAIN2 MOUNT SDCARD BOOTPATCHDIR KERNELPATCHDIR RELEASE IMAGE_TYPE OVERLAY_PREFIX ASOUND_STATE ATF_COMPILER \
ATF_USE_GCC ATFSOURCE ATFDIR ATFBRANCH ATFSOURCEDIR PACKAGE_LIST_RM NM_IGNORE_DEVICES DISPLAY_MANAGER \
family_tweaks_bsp_s CRYPTROOT_ENABLE CRYPTROOT_PASSPHRASE CRYPTROOT_SSH_UNLOCK CRYPTROOT_SSH_UNLOCK_PORT \
CRYPTROOT_SSH_UNLOCK_KEY_NAME ROOT_MAPPER NETWORK HDMI USB WIRELESS ARMBIANMONITOR FORCE_BOOTSCRIPT_UPDATE \
UBOOT_TOOLCHAIN2 toolchain2 BUILD_REPOSITORY_URL BUILD_REPOSITORY_COMMIT BUILD_TARGET HOST BUILD_IMAGE \
DEB_STORAGE REPO_STORAGE REPO_CONFIG REPOSITORY_UPDATE PACKAGE_LIST_RELEASE LOCAL_MIRROR COMPILE_ATF \
PACKAGE_LIST_BOARD PACKAGE_LIST_FAMILY PACKAGE_LIST_DESKTOP_BOARD PACKAGE_LIST_DESKTOP_FAMILY ATF_COMPILE ATFPATCHDIR OFFSET BOOTSOURCEDIR BOOT_USE_BLOBS \
BOOT_SOC DDR_BLOB MINILOADER_BLOB BL31_BLOB BOOT_RK3328_USE_AYUFAN_ATF BOOT_USE_BLOBS BOOT_RK3399_LEGACY_HYBRID \
BOOT_USE_MAINLINE_ATF BOOT_USE_TPL_SPL_BLOB BOOT_SUPPORT_SPI OFFLINE_WORK IMAGE_PARTITION_TABLE BOOT_LOGO UPSTREM_VER FORCED_MONTH_OFFSET \
PACKAGE_LIST_BOARD_REMOVE PACKAGE_LIST_FAMILY_REMOVE PACKAGE_LIST_DESKTOP_BOARD_REMOVE PACKAGE_LIST_DESKTOP_FAMILY_REMOVE BOOTCONFIG_DEV USE_OVERLAYFS
}
pack_upload ()
{
# pack and upload to server or just pack
display_alert "Signing" "Please wait!" "info"
local version="Armbian_${REVISION}_${BOARD^}_${RELEASE}_${BRANCH}_${VER/-$LINUXFAMILY/}"
local subdir="archive"
compression_type=""
[[ $BUILD_DESKTOP == yes ]] && version=${version}_desktop
[[ $BUILD_MINIMAL == yes ]] && version=${version}_minimal
[[ $BETA == yes ]] && local subdir=nightly
cd "${DESTIMG}" || exit
if [[ $COMPRESS_OUTPUTIMAGE == yes ]]; then
COMPRESS_OUTPUTIMAGE="sha,gpg,7z"
fi
if [[ $COMPRESS_OUTPUTIMAGE == *gz* ]]; then
display_alert "Compressing" "$DEST/images/${version}.img.gz" "info"
pigz "$DESTIMG/${version}.img"
rm "${DESTIMG}/${version}.img"
compression_type=".gz"
fi
if [[ $COMPRESS_OUTPUTIMAGE == *xz* ]]; then
display_alert "Compressing" "$DEST/images/${version}.img.xz" "info"
pixz -9 < "$DESTIMG/${version}.img" > "${DESTIMG}/${version}.img.xz"
rm "${DESTIMG}/${version}.img"
compression_type=".xz"
fi
if [[ $COMPRESS_OUTPUTIMAGE == *sha* ]]; then
display_alert "SHA256 calculating" "${version}.img" "info"
sha256sum -b "${version}.img${compression_type}" > "${version}.img${compression_type}.sha"
fi
if [[ $COMPRESS_OUTPUTIMAGE == *gpg* ]]; then
if [[ -n $GPG_PASS ]]; then
display_alert "GPG signing" "${version}.img" "info"
echo "${GPG_PASS}" | gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes "${version}.img${compression_type}" || exit 1
else
display_alert "GPG signing skipped - no GPG_PASS" "${version}.img" "wrn"
fi
fi
fingerprint_image "${version}.img${compression_type}.txt" "${version}"
if [[ $COMPRESS_OUTPUTIMAGE == *7z* ]]; then
display_alert "Compressing" "${version}.7z" "info"
7za a -t7z -bd -m0=lzma2 -mx=3 -mfb=64 -md=32m -ms=on "${version}".7z "${version}".img* >/dev/null 2>&1
find . -type f -not -name '*.7z' -print0 | xargs -0 rm --
fi
if [[ -n "${SEND_TO_SERVER}" ]]; then
ssh "${SEND_TO_SERVER}" "mkdir -p ${SEND_TO_LOCATION}${BOARD}/{archive,nightly}" &
display_alert "Uploading" "Please wait!" "info"
nice -n 19 bash -c "rsync -arP --info=progress2 --prune-empty-dirs $DESTIMG/ -e 'ssh -T -c aes128-ctr -o Compression=no -x -p 22' ${SEND_TO_SERVER}:${SEND_TO_LOCATION}${BOARD}/${subdir}; rm -rf ${DESTIMG}/*" &
else
mv "${DESTIMG}"/*.* "${DEST}"/images/
fi
}
build_main ()
{
# shellcheck source=/dev/null
source "$USERPATCHES_PATH"/lib.config
# build images which we do pack or kernel
local upload_image upload_subdir
upload_image="Armbian_$(cat "${SRC}"/VERSION)_${BOARD^}_${RELEASE}_${BRANCH}_*${VER/-$LINUXFAMILY/}"
upload_subdir="archive"
[[ $BUILD_DESKTOP == yes ]] && upload_image=${upload_image}_desktop
[[ $BUILD_MINIMAL == yes ]] && upload_image=${upload_image}_minimal
[[ $BETA == yes ]] && local upload_subdir=nightly
touch "/run/armbian/Armbian_${BOARD^}_${BRANCH}_${RELEASE}_${BUILD_DESKTOP}_${BUILD_MINIMAL}.pid";
if [[ $KERNEL_ONLY != yes ]]; then
#if ssh ${SEND_TO_SERVER} stat ${SEND_TO_LOCATION}${BOARD}/${upload_subdir}/${upload_image}* \> /dev/null 2\>\&1; then
# echo "$n exists $upload_image"
#else
#shellcheck source=lib/main.sh
source "${SRC}"/lib/main.sh
[[ "$BSP_BUILD" != yes && -n "${SEND_TO_SERVER}" ]] && pack_upload
#fi
else
#shellcheck source=lib/main.sh
source "${SRC}"/lib/main.sh
fi
cd "${SRC}"
rm "/run/armbian/Armbian_${BOARD^}_${BRANCH}_${RELEASE}_${BUILD_DESKTOP}_${BUILD_MINIMAL}.pid"
}
array_contains ()
{
# utility snippet
local array="$1[@]"
local seeking=$2
local in=1
for element in "${!array}"; do
if [[ "${element}" == "${seeking}" ]]; then
in=0
break
fi
done
return $in
}
function check_hash()
{
local BOARDFAMILY ref_type ref_name
BOARDFAMILY=$(grep BOARDFAMILY "${SRC}/config/boards/${BOARD}".* | cut -d \" -f2)
# shellcheck source=/dev/null
source "${SRC}/config/sources/families/${BOARDFAMILY}.conf"
# shellcheck source=/dev/null
source "${SRC}/config/sources/${ARCH}.conf"
ref_type=${KERNELBRANCH%%:*}
if [[ $ref_type == head ]]; then
ref_name=HEAD
else
ref_name=${KERNELBRANCH##*:}
fi
[[ -z $LINUXFAMILY ]] && LINUXFAMILY=$BOARDFAMILY
[[ -z ${KERNELPATCHDIR} ]] && KERNELPATCHDIR=$LINUXFAMILY-$BRANCH
[[ -z ${LINUXCONFIG} ]] && LINUXCONFIG=linux-$LINUXFAMILY-$BRANCH
hash_watch_1=$(find "${SRC}/patch/kernel/${KERNELPATCHDIR}" -maxdepth 1 -printf '%s %P\n' 2> /dev/null | sort)
hash_watch_2=$(cat "${SRC}/config/kernel/${LINUXCONFIG}.config")
patch_hash=$(echo "${hash_watch_1}${hash_watch_2}" | improved_git hash-object --stdin)
case $ref_type in
branch) hash=$(improved_git ls-remote "${KERNELSOURCE}" refs/heads/"${ref_name}" 2> /dev/null | awk '{print $1}') ;;
tag) hash=$(improved_git ls-remote "${KERNELSOURCE}" tags/"${ref_name}" 2> /dev/null | awk '{print $1}') ;;
head) hash=$(improved_git ls-remote "${KERNELSOURCE}" HEAD 2> /dev/null | awk '{print $1}') ;;
commit) hash=$ref_name ;;
esac
# ignore diff checking in case of network errrors
local kernel_hash="${SRC}/cache/hash"$([[ ${BETA} == yes ]] && echo "-beta")"/linux-image-${BRANCH}-${LINUXFAMILY}.githash"
if [[ -f ${kernel_hash} ]]; then
[[ "$hash" == "$(head -1 "${kernel_hash}")" && "$patch_hash" == "$(tail -1 "${kernel_hash}")" || -z $hash ]] && echo "idential"
fi
}
function build_all()
{
# main routine
buildall_start=$(date +%s)
n=0
ARRAY=()
buildlist="cat "
# building selected ones
if [[ -n ${REBUILD_IMAGES} ]]; then
buildlist="grep -w '"
filter="'"
for build in $(tr ',' ' ' <<< "${REBUILD_IMAGES}"); do
buildlist=$buildlist"$build\|"
filter=$filter"$build\|"
done
buildlist=${buildlist::-2}"'"
filter=${filter::-2}"'"
fi
# find unique boards - we will build debs for all variants
sorted_unique_ids=($(echo "${ids[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
unique_boards=$(eval "${buildlist}" "${SRC}"/config/targets.conf | sed '/^#/ d' | awk '{print $1}')
read -r -a unique_boards <<< "$(echo "${unique_boards[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')"
while read -r line; do
[[ "${line}" =~ ^#.*$ ]] && continue
[[ -n "${REBUILD_IMAGES}" ]] && [[ -z $(echo "${line}" | eval grep -w "${filter}") ]] && continue
#[[ $n -lt $START ]] && ((n+=1)) && continue
unset_all
# unset also board related variables
unset BOARDFAMILY DESKTOP_AUTOLOGIN DEFAULT_CONSOLE FULL_DESKTOP MODULES MODULES_CURRENT MODULES_LEGACY MODULES_DEV \
BOOTCONFIG MODULES_BLACKLIST MODULES_BLACKLIST_LEGACY MODULES_BLACKLIST_CURRENT MODULES_BLACKLIST_DEV DEFAULT_OVERLAYS SERIALCON \
BUILD_MINIMAL RELEASE ATFBRANCH BOOT_FDT_FILE BOOTCONFIG_DEV
read -r BOARD BRANCH RELEASE BUILD_TARGET BUILD_STABILITY BUILD_IMAGE <<< "${line}"
# read all possible configurations
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".eos 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".tvb 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".csc 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".wip 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".conf 2> /dev/null
# override branch to build selected branches if defined
if [[ -n "${BROVER}" ]]; then
if [[ "${KERNEL_TARGET}" == *${BROVER}* ]]; then
BRANCH=${BROVER}
else
continue
fi
fi
# exceptions handling
[[ ${BOARDFAMILY} == sun*i ]] && BOARDFAMILY=sunxi
[[ ${BOARDFAMILY} == sun*iw* ]] && BOARDFAMILY=sunxi64
[[ ${BOARDFAMILY} == meson8b ]] && BOARDFAMILY=meson
[[ ${BOARDFAMILY} == meson-* ]] && BOARDFAMILY=meson64
# small optimisation. we only (try to) build needed kernels
if [[ $KERNEL_ONLY == yes ]]; then
LINUXFAMILY="${BOARDFAMILY}"
array_contains ARRAY "${LINUXFAMILY}${BRANCH}${BUILD_STABILITY}" && continue
elif [[ $BUILD_IMAGE == no ]] ; then
continue
fi
ARRAY+=("${LINUXFAMILY}${BRANCH}${BUILD_STABILITY}")
BUILD_DESKTOP="no"
BUILD_MINIMAL="no"
[[ ${BUILD_TARGET} == "desktop" ]] && BUILD_DESKTOP="yes"
[[ ${BUILD_TARGET} == "minimal" ]] && BUILD_MINIMAL="yes"
# create beta or stable
if [[ "${BUILD_STABILITY}" == "${STABILITY}" ]]; then
# check if currnt hash is the same as upstream
if [[ "$IGNORE_HASH" != yes ]]; then
local store_hash
store_hash=$(check_hash)
fi
if [[ "$store_hash" != idential ]]; then
if [[ $1 != "dryrun" ]] && [[ $n -ge $START ]]; then
((n+=1))
while :
do
if [[ $(find /run/armbian/*.pid 2>/dev/null | wc -l) -le ${MULTITHREAD} || ${MULTITHREAD} -eq 0 ]]; then
break
fi
sleep 5
done
display_alert "Building ${n}."
if [[ "$KERNEL_ONLY" == "no" && "${MULTITHREAD}" -gt 0 ]]; then
build_main &
sleep $((RANDOM % 5))
elif [[ "${MULTITHREAD}" -gt 0 ]]; then
build_main &
sleep $((RANDOM % 5))
else
build_main
fi
# create BSP for all boards
elif [[ "${BSP_BUILD}" == yes ]]; then
((n+=1))
for BOARD in "${unique_boards[@]}"
do
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".eos 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".tvb 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".csc 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".wip 2> /dev/null
# shellcheck source=/dev/null
source "${SRC}/config/boards/${BOARD}".conf 2> /dev/null
IFS=',' read -r -a RELBRANCH <<< "${KERNEL_TARGET}"
for BRANCH in "${RELBRANCH[@]}"
do
RELTARGETS=(xenial stretch buster bullseye bionic groovy focal)
for RELEASE in "${RELTARGETS[@]}"
do
display_alert "BSP for ${BOARD} ${BRANCH} ${RELEASE}."
if [[ "$IGNORE_HASH" == yes && "$KERNEL_ONLY" != "yes" && "${MULTITHREAD}" -gt 0 ]]; then
build_main &
sleep 0.02
elif [[ "${MULTITHREAD}" -gt 0 ]]; then
build_main &
sleep $((RANDOM % 5))
else
build_main
fi
# unset non board related stuff
unset_all
done
done
done
display_alert "Done building all BSP images"
exit
else
((n+=1))
# In dryrun it only prints out what will be build
printf "%s\t%-32s\t%-8s\t%-14s\t%-6s\t%-6s\t%-6s\n" "${n}." \
"$BOARD (${BOARDFAMILY})" "${BRANCH}" "${RELEASE}" "${BUILD_DESKTOP}" "${BUILD_MINIMAL}"
fi
fi
fi
# at which image to stop
[[ "$STOP" == "$n" ]] && [[ $1 != "dryrun" ]] && exit
done < "${BUILD_TARGETS}"
}
# display what will be build
echo ""
display_alert "Building all targets" "$STABILITY $(if [[ $KERNEL_ONLY == "yes" ]] ; then echo "kernels"; \
else echo "images"; fi)" "info"
printf "\n%s\t%-32s\t%-8s\t%-14s\t%-6s\t%-6s\t%-6s\n\n" "" "board" "branch" "release" "XFCE" "minimal"
# display what we will build
build_all "dryrun"
if [[ $BUILD_ALL != demo ]] ; then
echo ""
# build
build_all
fi
# wait until they are not finshed
sleep 5
while :
do
if [[ $(df | grep -c /.tmp) -lt 1 ]]; then
break
fi
sleep 5
done
while :
do
if [[ -z $(ps -uax | grep 7z | grep Armbian) ]]; then
break
fi
sleep 5
done
# bump version in case there was a change
if [[ $n -gt 0 && ${BUMP_VERSION} == yes ]]; then
cd "${SRC}" || exit
CURRENT_VERSION=$(cat VERSION)
NEW_VERSION="${CURRENT_VERSION%%-trunk}"
if [[ $CURRENT_VERSION == *trunk* ]]; then
NEW_VERSION=$(echo "${CURRENT_VERSION}" | cut -d. -f1-3)"."$((${NEW_VERSION##*.} + 1))
else
NEW_VERSION=$(echo "${CURRENT_VERSION}" | cut -d. -f1-2)"."$((${NEW_VERSION##*.} + 1))
fi
echo "${NEW_VERSION}" > VERSION
improved_git add "${SRC}"/VERSION
improved_git commit -m "Bumping to new version" -m "" -m "Adding following kernels:" -m "$(find output/debs/ -type f -name "linux-image*${CURRENT_VERSION}*.deb" -printf "%f\n" | sort)"
improved_git push
display_alert "Bumping to new version" "${NEW_VERSION}" "info"
fi
buildall_end=$(date +%s)
buildall_runtime=$(((buildall_end - buildall_start) / 60))
display_alert "Runtime in total" "${buildall_runtime} min" "info"
echo "${n}" > "${SRC}"/.tmp/n