mirror of
https://github.com/Fishwaldo/build.git
synced 2025-03-15 19:31:42 +00:00
Mitigate git server failures by retrying git commands several times. (#2251)
* Mitigate git server failures by retrying git commands several times. * Bugfix - commit type was not working at all. Now it works
This commit is contained in:
Igor Pečovnik
GitHub
parent
f89d9a1b46
commit
e8f961c9c7
5 changed files with 84 additions and 58 deletions
|
|
@ -210,12 +210,12 @@ function check_hash()
|
|||
[[ -z ${LINUXCONFIG} ]] && LINUXCONFIG=linux-$LINUXFAMILY-$BRANCH
|
||||
hash_watch_1=$(find "${SRC}/patch/kernel/${KERNELPATCHDIR}" -maxdepth 1 -printf '%s %P\n' 2> /dev/null | sort)
|
||||
hash_watch_2=$(cat "${SRC}/config/kernel/${LINUXCONFIG}.config")
|
||||
patch_hash=$(echo "${hash_watch_1}${hash_watch_2}" | git hash-object --stdin)
|
||||
patch_hash=$(echo "${hash_watch_1}${hash_watch_2}" | improved_git hash-object --stdin)
|
||||
|
||||
case $ref_type in
|
||||
branch) hash=$(git ls-remote "${KERNELSOURCE}" refs/heads/"${ref_name}" 2> /dev/null | awk '{print $1}') ;;
|
||||
tag) hash=$(git ls-remote "${KERNELSOURCE}" tags/"${ref_name}" 2> /dev/null | awk '{print $1}') ;;
|
||||
head) hash=$(git ls-remote "${KERNELSOURCE}" HEAD 2> /dev/null | awk '{print $1}') ;;
|
||||
branch) hash=$(improved_git ls-remote "${KERNELSOURCE}" refs/heads/"${ref_name}" 2> /dev/null | awk '{print $1}') ;;
|
||||
tag) hash=$(improved_git ls-remote "${KERNELSOURCE}" tags/"${ref_name}" 2> /dev/null | awk '{print $1}') ;;
|
||||
head) hash=$(improved_git ls-remote "${KERNELSOURCE}" HEAD 2> /dev/null | awk '{print $1}') ;;
|
||||
commit) hash=$ref_name ;;
|
||||
esac
|
||||
# ignore diff checking in case of network errrors
|
||||
|
|
@ -444,9 +444,9 @@ if [[ $n -gt 0 && ${BUMP_VERSION} == yes ]]; then
|
|||
fi
|
||||
|
||||
echo "${NEW_VERSION}" > VERSION
|
||||
git add "${SRC}"/VERSION
|
||||
git commit -m "Bumping to new version" -m "" -m "Adding following kernels:" -m "$(find output/debs/ -type f -name "linux-image*${CURRENT_VERSION}*.deb" -printf "%f\n" | sort)"
|
||||
git push
|
||||
improved_git add "${SRC}"/VERSION
|
||||
improved_git commit -m "Bumping to new version" -m "" -m "Adding following kernels:" -m "$(find output/debs/ -type f -name "linux-image*${CURRENT_VERSION}*.deb" -printf "%f\n" | sort)"
|
||||
improved_git push
|
||||
display_alert "Bumping to new version" "${NEW_VERSION}" "info"
|
||||
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -129,12 +129,12 @@ compilation_prepare()
|
|||
if linux-version compare "${version}" ge 5.4.3 && linux-version compare "${version}" le 5.5 ; then aufstag="5.4.3"; fi
|
||||
|
||||
# check if Mr. Okajima already made a branch for this version
|
||||
git ls-remote --exit-code --heads https://github.com/sfjro/aufs5-standalone "aufs${aufstag}" >/dev/null
|
||||
improved_git ls-remote --exit-code --heads https://github.com/sfjro/aufs5-standalone "aufs${aufstag}" >/dev/null
|
||||
|
||||
if [ "$?" -ne "0" ]; then
|
||||
# then use rc branch
|
||||
aufstag="5.x-rcN"
|
||||
git ls-remote --exit-code --heads https://github.com/sfjro/aufs5-standalone "aufs${aufstag}" >/dev/null
|
||||
improved_git ls-remote --exit-code --heads https://github.com/sfjro/aufs5-standalone "aufs${aufstag}" >/dev/null
|
||||
fi
|
||||
|
||||
if [ "$?" -eq "0" ]; then
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ compile_uboot()
|
|||
# read uboot version
|
||||
local version hash
|
||||
version=$(grab_version "$ubootdir")
|
||||
hash=$(git --git-dir="$ubootdir"/.git rev-parse HEAD)
|
||||
hash=$(improved_git --git-dir="$ubootdir"/.git rev-parse HEAD)
|
||||
|
||||
display_alert "Compiling u-boot" "$version" "info"
|
||||
|
||||
|
|
@ -157,7 +157,7 @@ compile_uboot()
|
|||
|
||||
# needed for multiple targets and for calling compile_uboot directly
|
||||
display_alert "Checking out to clean sources"
|
||||
git checkout -f -q HEAD
|
||||
improved_git checkout -f -q HEAD
|
||||
|
||||
if [[ $CLEAN_LEVEL == *make* ]]; then
|
||||
display_alert "Cleaning" "$BOOTSOURCEDIR" "info"
|
||||
|
|
@ -279,7 +279,7 @@ compile_uboot()
|
|||
|
||||
# store git hash to the file
|
||||
# echo $hash > ${SRC}/cache/hash/${CHOSEN_UBOOT}.githash
|
||||
# find "${SRC}/patch/u-boot/${BOOTPATCHDIR}" -maxdepth 1 -printf '%s %P\n' | git hash-object --stdin >> "${SRC}/cache/hash"$([[ ${BETA} == yes ]] && echo "-beta")"/${CHOSEN_UBOOT}.githash"
|
||||
# find "${SRC}/patch/u-boot/${BOOTPATCHDIR}" -maxdepth 1 -printf '%s %P\n' | improved_git hash-object --stdin >> "${SRC}/cache/hash"$([[ ${BETA} == yes ]] && echo "-beta")"/${CHOSEN_UBOOT}.githash"
|
||||
}
|
||||
|
||||
compile_kernel()
|
||||
|
|
@ -307,7 +307,7 @@ compile_kernel()
|
|||
version=$(grab_version "$kerneldir")
|
||||
|
||||
# read kernel git hash
|
||||
hash=$(git --git-dir="$kerneldir"/.git rev-parse HEAD)
|
||||
hash=$(improved_git --git-dir="$kerneldir"/.git rev-parse HEAD)
|
||||
|
||||
# build 3rd party drivers
|
||||
compilation_prepare
|
||||
|
|
@ -466,7 +466,7 @@ compile_kernel()
|
|||
[[ -z ${LINUXCONFIG} ]] && LINUXCONFIG=linux-$LINUXFAMILY-$BRANCH
|
||||
hash_watch_1=$(find "${SRC}/patch/kernel/${KERNELPATCHDIR}" -maxdepth 1 -printf '%s %P\n' 2> /dev/null | sort)
|
||||
hash_watch_2=$(cat "${SRC}/config/kernel/${LINUXCONFIG}.config")
|
||||
echo "${hash_watch_1}${hash_watch_2}" | git hash-object --stdin >> "${SRC}/cache/hash"$([[ ${BETA} == yes ]] && echo "-beta")"/linux-image-${BRANCH}-${LINUXFAMILY}.githash"
|
||||
echo "${hash_watch_1}${hash_watch_2}" | improved_git hash-object --stdin >> "${SRC}/cache/hash"$([[ ${BETA} == yes ]] && echo "-beta")"/linux-image-${BRANCH}-${LINUXFAMILY}.githash"
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -573,13 +573,13 @@ compile_sunxi_tools()
|
|||
# Compile and install only if git commit hash changed
|
||||
cd "${SRC}"/cache/sources/sunxi-tools || exit
|
||||
# need to check if /usr/local/bin/sunxi-fexc to detect new Docker containers with old cached sources
|
||||
if [[ ! -f .commit_id || $(git rev-parse @ 2>/dev/null) != $(<.commit_id) || ! -f /usr/local/bin/sunxi-fexc ]]; then
|
||||
if [[ ! -f .commit_id || $(improved_git rev-parse @ 2>/dev/null) != $(<.commit_id) || ! -f /usr/local/bin/sunxi-fexc ]]; then
|
||||
display_alert "Compiling" "sunxi-tools" "info"
|
||||
make -s clean >/dev/null
|
||||
make -s tools >/dev/null
|
||||
mkdir -p /usr/local/bin/
|
||||
make install-tools >/dev/null 2>&1
|
||||
git rev-parse @ 2>/dev/null > .commit_id
|
||||
improved_git rev-parse @ 2>/dev/null > .commit_id
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -588,12 +588,12 @@ install_rkbin_tools()
|
|||
# install only if git commit hash changed
|
||||
cd "${SRC}"/cache/sources/rkbin-tools || exit
|
||||
# need to check if /usr/local/bin/sunxi-fexc to detect new Docker containers with old cached sources
|
||||
if [[ ! -f .commit_id || $(git rev-parse @ 2>/dev/null) != $(<.commit_id) || ! -f /usr/local/bin/loaderimage ]]; then
|
||||
if [[ ! -f .commit_id || $(improved_git rev-parse @ 2>/dev/null) != $(<.commit_id) || ! -f /usr/local/bin/loaderimage ]]; then
|
||||
display_alert "Installing" "rkbin-tools" "info"
|
||||
mkdir -p /usr/local/bin/
|
||||
install -m 755 tools/loaderimage /usr/local/bin/
|
||||
install -m 755 tools/trust_merger /usr/local/bin/
|
||||
git rev-parse @ 2>/dev/null > .commit_id
|
||||
improved_git rev-parse @ 2>/dev/null > .commit_id
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -756,8 +756,8 @@ process_patch_file()
|
|||
userpatch_create()
|
||||
{
|
||||
# create commit to start from clean source
|
||||
git add .
|
||||
git -c user.name='Armbian User' -c user.email='user@example.org' commit -q -m "Cleaning working copy"
|
||||
improved_git add .
|
||||
improved_git -c user.name='Armbian User' -c user.email='user@example.org' commit -q -m "Cleaning working copy"
|
||||
|
||||
local patch="$DEST/patch/$1-$LINUXFAMILY-$BRANCH.patch"
|
||||
|
||||
|
|
@ -765,7 +765,7 @@ userpatch_create()
|
|||
if [[ -f $patch ]]; then
|
||||
display_alert "Applying existing $1 patch" "$patch" "wrn" && patch --batch --silent -p1 -N < "${patch}"
|
||||
# read title of a patch in case Git is configured
|
||||
if [[ -n $(git config user.email) ]]; then
|
||||
if [[ -n $(improved_git config user.email) ]]; then
|
||||
COMMIT_MESSAGE=$(cat "${patch}" | grep Subject | sed -n -e '0,/PATCH/s/.*PATCH]//p' | xargs)
|
||||
display_alert "Patch name extracted" "$COMMIT_MESSAGE" "wrn"
|
||||
fi
|
||||
|
|
@ -776,29 +776,29 @@ userpatch_create()
|
|||
display_alert "Press <Enter> after you are done" "waiting" "wrn"
|
||||
read -r </dev/tty
|
||||
tput cuu1
|
||||
git add .
|
||||
improved_git add .
|
||||
# create patch out of changes
|
||||
if ! git diff-index --quiet --cached HEAD; then
|
||||
if ! improved_git diff-index --quiet --cached HEAD; then
|
||||
# If Git is configured, create proper patch and ask for a name
|
||||
if [[ -n $(git config user.email) ]]; then
|
||||
if [[ -n $(improved_git config user.email) ]]; then
|
||||
display_alert "Add / change patch name" "$COMMIT_MESSAGE" "wrn"
|
||||
read -e -p "Patch description: " -i "$COMMIT_MESSAGE" COMMIT_MESSAGE
|
||||
[[ -z "$COMMIT_MESSAGE" ]] && COMMIT_MESSAGE="Patching something"
|
||||
git commit -s -m "$COMMIT_MESSAGE"
|
||||
git format-patch -1 HEAD --stdout --signature="Created with Armbian build tools https://github.com/armbian/build" > "${patch}"
|
||||
PATCHFILE=$(git format-patch -1 HEAD)
|
||||
improved_git commit -s -m "$COMMIT_MESSAGE"
|
||||
improved_git format-patch -1 HEAD --stdout --signature="Created with Armbian build tools https://github.com/armbian/build" > "${patch}"
|
||||
PATCHFILE=$(improved_git format-patch -1 HEAD)
|
||||
rm $PATCHFILE # delete the actual file
|
||||
# create a symlink to have a nice name ready
|
||||
find $DEST/patch/ -type l -delete # delete any existing
|
||||
ln -sf $patch $DEST/patch/$PATCHFILE
|
||||
else
|
||||
git diff --staged > "${patch}"
|
||||
improved_git diff --staged > "${patch}"
|
||||
fi
|
||||
display_alert "You will find your patch here:" "$patch" "info"
|
||||
else
|
||||
display_alert "No changes found, skipping patch creation" "" "wrn"
|
||||
fi
|
||||
git reset --soft HEAD~
|
||||
improved_git reset --soft HEAD~
|
||||
for i in {3..1..1}; do echo -n "$i." && sleep 1; done
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ EXIT_PATCHING_ERROR="" # exit patching if failed
|
|||
cd "${SRC}" || exit
|
||||
ROOTFSCACHE_VERSION=38
|
||||
CHROOT_CACHE_VERSION=7
|
||||
BUILD_REPOSITORY_URL=$(git remote get-url $(git remote 2>/dev/null) 2>/dev/null)
|
||||
BUILD_REPOSITORY_COMMIT=$(git describe --match=d_e_a_d_b_e_e_f --always --dirty 2>/dev/null)
|
||||
BUILD_REPOSITORY_URL=$(improved_git remote get-url $(improved_git remote 2>/dev/null) 2>/dev/null)
|
||||
BUILD_REPOSITORY_COMMIT=$(improved_git describe --match=d_e_a_d_b_e_e_f --always --dirty 2>/dev/null)
|
||||
ROOTFS_CACHE_MAX=42 # max number of rootfs cache, older ones will be cleaned up
|
||||
|
||||
if [[ $BETA == yes ]]; then
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
# get_package_list_hash
|
||||
# create_sources_list
|
||||
# fetch_from_repo
|
||||
# improved_git
|
||||
# display_alert
|
||||
# fingerprint_image
|
||||
# distro_menu
|
||||
|
|
@ -187,6 +188,31 @@ create_sources_list()
|
|||
rm "${SDCARD}"/armbian.key
|
||||
}
|
||||
|
||||
|
||||
#
|
||||
# This function retries Git operations to avoid failure in case remote is borked
|
||||
#
|
||||
improved_git()
|
||||
{
|
||||
|
||||
local realgit=$(which git)
|
||||
local retries=3
|
||||
local delay=10
|
||||
local count=1
|
||||
while [ $count -lt $retries ]; do
|
||||
$realgit $*
|
||||
if [ $? -eq 0 ]; then
|
||||
retries=0
|
||||
break
|
||||
fi
|
||||
let count=$count+1
|
||||
display_alert "Git not responding. Retrying in ${delay} seconds."
|
||||
sleep $delay
|
||||
done
|
||||
|
||||
}
|
||||
|
||||
|
||||
# fetch_from_repo <url> <directory> <ref> <ref_subdir>
|
||||
# <url>: remote repository URL
|
||||
# <directory>: local directory; subdir for branch/tag will be created
|
||||
|
|
@ -244,16 +270,16 @@ fetch_from_repo()
|
|||
# Check the folder as a git repository.
|
||||
# Then the target URL matches the local URL.
|
||||
|
||||
if [[ "$(git rev-parse --git-dir 2>/dev/null)" == ".git" && \
|
||||
"$url" != "$(git remote get-url origin 2>/dev/null)" ]]; then
|
||||
if [[ "$(improved_git rev-parse --git-dir 2>/dev/null)" == ".git" && \
|
||||
"$url" != "$(improved_git remote get-url origin 2>/dev/null)" ]]; then
|
||||
display_alert "Remote URL does not match, removing existing local copy"
|
||||
rm -rf .git ./*
|
||||
fi
|
||||
|
||||
if [[ "$(git rev-parse --git-dir 2>/dev/null)" != ".git" ]]; then
|
||||
if [[ "$(improved_git rev-parse --git-dir 2>/dev/null)" != ".git" ]]; then
|
||||
display_alert "Creating local copy"
|
||||
git init -q .
|
||||
git remote add origin "${url}"
|
||||
improved_git init -q .
|
||||
improved_git remote add origin "${url}"
|
||||
# Here you need to upload from a new address
|
||||
offline=false
|
||||
fi
|
||||
|
|
@ -269,22 +295,22 @@ fetch_from_repo()
|
|||
branch)
|
||||
# TODO: grep refs/heads/$name
|
||||
local remote_hash
|
||||
remote_hash=$(git ls-remote -h "${url}" "$ref_name" | head -1 | cut -f1)
|
||||
remote_hash=$(improved_git ls-remote -h "${url}" "$ref_name" | head -1 | cut -f1)
|
||||
[[ -z $local_hash || "${local_hash}" != "${remote_hash}" ]] && changed=true
|
||||
;;
|
||||
|
||||
tag)
|
||||
local remote_hash
|
||||
remote_hash=$(git ls-remote -t "${url}" "$ref_name" | cut -f1)
|
||||
remote_hash=$(improved_git ls-remote -t "${url}" "$ref_name" | cut -f1)
|
||||
if [[ -z $local_hash || "${local_hash}" != "${remote_hash}" ]]; then
|
||||
remote_hash=$(git ls-remote -t "${url}" "$ref_name^{}" | cut -f1)
|
||||
remote_hash=$(improved_git ls-remote -t "${url}" "$ref_name^{}" | cut -f1)
|
||||
[[ -z $remote_hash || "${local_hash}" != "${remote_hash}" ]] && changed=true
|
||||
fi
|
||||
;;
|
||||
|
||||
head)
|
||||
local remote_hash
|
||||
remote_hash=$(git ls-remote "${url}" HEAD | cut -f1)
|
||||
remote_hash=$(improved_git ls-remote "${url}" HEAD | cut -f1)
|
||||
[[ -z $local_hash || "${local_hash}" != "${remote_hash}" ]] && changed=true
|
||||
;;
|
||||
|
||||
|
|
@ -300,48 +326,48 @@ fetch_from_repo()
|
|||
# remote was updated, fetch and check out updates
|
||||
display_alert "Fetching updates"
|
||||
case $ref_type in
|
||||
branch) git fetch --depth 1 origin "${ref_name}" ;;
|
||||
tag) git fetch --depth 1 origin tags/"${ref_name}" ;;
|
||||
head) git fetch --depth 1 origin HEAD ;;
|
||||
branch) improved_git fetch --depth 1 origin "${ref_name}" ;;
|
||||
tag) improved_git fetch --depth 1 origin tags/"${ref_name}" ;;
|
||||
head) improved_git fetch --depth 1 origin HEAD ;;
|
||||
esac
|
||||
|
||||
# commit type needs support for older git servers that doesn't support fetching id directly
|
||||
if [[ $ref_type == commit ]]; then
|
||||
|
||||
git fetch --depth 1 origin "${ref_name}"
|
||||
improved_git fetch --depth 1 origin "${ref_name}"
|
||||
|
||||
# cover old type
|
||||
if [[ $? -ne 0 ]]; then
|
||||
|
||||
display_alert "Commit checkout not supported on this repository. Doing full clone." "" "wrn"
|
||||
git pull
|
||||
git checkout -fq "${ref_name}"
|
||||
display_alert "Checkout out to" "$(git --no-pager log -2 --pretty=format:"$ad%s [%an]" | head -1)" "info"
|
||||
improved_git pull
|
||||
improved_git checkout -fq "${ref_name}"
|
||||
display_alert "Checkout out to" "$(improved_git --no-pager log -2 --pretty=format:"$ad%s [%an]" | head -1)" "info"
|
||||
|
||||
else
|
||||
|
||||
display_alert "Checking out"
|
||||
git checkout -f -q FETCH_HEAD
|
||||
git clean -qdf
|
||||
improved_git checkout -f -q FETCH_HEAD
|
||||
improved_git clean -qdf
|
||||
|
||||
fi
|
||||
else
|
||||
|
||||
display_alert "Checking out"
|
||||
git checkout -f -q FETCH_HEAD
|
||||
git clean -qdf
|
||||
improved_git checkout -f -q FETCH_HEAD
|
||||
improved_git clean -qdf
|
||||
|
||||
fi
|
||||
elif [[ -n $(git status -uno --porcelain --ignore-submodules=all) ]]; then
|
||||
elif [[ -n $(improved_git status -uno --porcelain --ignore-submodules=all) ]]; then
|
||||
# working directory is not clean
|
||||
display_alert " Cleaning .... " "$(git status -s | wc -l) files"
|
||||
display_alert " Cleaning .... " "$(improved_git status -s | wc -l) files"
|
||||
|
||||
# Return the files that are tracked by git to the initial state.
|
||||
git checkout -f -q HEAD
|
||||
improved_git checkout -f -q HEAD
|
||||
|
||||
# Files that are not tracked by git and were added
|
||||
# when the patch was applied must be removed.
|
||||
git clean -qdf
|
||||
improved_git clean -qdf
|
||||
else
|
||||
# working directory is clean, nothing to do
|
||||
display_alert "Up to date"
|
||||
|
|
@ -350,11 +376,11 @@ fetch_from_repo()
|
|||
if [[ -f .gitmodules ]]; then
|
||||
display_alert "Updating submodules" "" "ext"
|
||||
# FML: http://stackoverflow.com/a/17692710
|
||||
for i in $(git config -f .gitmodules --get-regexp path | awk '{ print $2 }'); do
|
||||
for i in $(improved_git config -f .gitmodules --get-regexp path | awk '{ print $2 }'); do
|
||||
cd "${SRC}/cache/sources/${workdir}" || exit
|
||||
local surl sref
|
||||
surl=$(git config -f .gitmodules --get "submodule.$i.url")
|
||||
sref=$(git config -f .gitmodules --get "submodule.$i.branch")
|
||||
surl=$(improved_git config -f .gitmodules --get "submodule.$i.url")
|
||||
sref=$(improved_git config -f .gitmodules --get "submodule.$i.branch")
|
||||
if [[ -n $sref ]]; then
|
||||
sref="branch:$sref"
|
||||
else
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue