Commit 575ec676 authored by Baudouin Feildel's avatar Baudouin Feildel

Merge branch 'build-digest' into 'master'

Create digest and save it along with repo artifacts

See merge request !2
parents ad62c2c8 34cf181a
Pipeline #1039 failed with stages
in 2 minutes and 9 seconds
......@@ -11,6 +11,7 @@ stages:
artifacts:
paths:
- packages
- digest
expire_in: 2 days
before_script:
# We update just update the database and we don't upgrade packages
......@@ -119,6 +120,7 @@ generate-repository-database:
expire_in: 2 days
script:
- repo-add "./packages/${REPOSITORY_NAME}.db.tar.gz" "./packages/"*.pkg.tar.xz
- cat ./digest/package.*.txt > "./packages/latest_build_digest.txt"
tags:
- executor:docker
......@@ -144,4 +146,3 @@ deploy:
url: $REPOSITORY_URL
tags:
- executor:docker
......@@ -9,9 +9,6 @@ NO_FMT="\e[0m"
CURRENT_PACKAGE=""
CURRENT_DEPENDENCY_TREE=""
set -e
trap 'echo "Building $CURRENT_PACKAGE, dependency tree: $CURRENT_DEPENDENCY_TREE"' EXIT
invoke_pacman()
{
sudo pacman "$@"
......@@ -19,12 +16,14 @@ invoke_pacman()
download_package()
{
curl --location "https://aur.archlinux.org/cgit/aur.git/snapshot/$1.tar.gz" > "$1.tar.gz"
CURRENT_ACTION="Download https://aur.archlinux.org/cgit/aur.git/snapshot/$1.tar.gz"
curl --location "https://aur.archlinux.org/cgit/aur.git/snapshot/$1.tar.gz" > "$1.tar.gz" || log_error
}
extract_package()
{
tar -xf "$1.tar.gz"
CURRENT_ACTION="Extract $1.tar.gz in $PWD"
tar -xf "$1.tar.gz" || log_error
}
package_get_makedepends()
......@@ -44,32 +43,40 @@ package_get_depends()
install_package()
{
invoke_pacman -U --noconfirm "$WORKDIR/$1/"*.pkg.tar.xz
CURRENT_ACTION="Install package from disk"
invoke_pacman -U --noconfirm "$WORKDIR/$1/"*.pkg.tar.xz || log_error
}
save_package()
{
cp "$WORKDIR/$1/"*.pkg.tar.xz "$SRCDIR/packages"
CURRENT_ACTION="Save package as artifact"
cp "$WORKDIR/$1/"*.pkg.tar.xz "$SRCDIR/packages" || log_error
add_to_digest "Save OK"
}
save_package_from_cache()
{
CURRENT_ACTION="Cache Fetch"
if [[ ! -f current-repo/current ]]
then
pushd "$SRCDIR" > /dev/null
"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}"
"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}" || log_error
popd > /dev/null
fi
local filename=$(grep -P "^$1\t" "$SRCDIR/current-repo/current" | cut -f 3)
pushd "$SRCDIR/packages" > /dev/null
curl -O "${REPOSITORY_URL}/$filename"
curl -O "${REPOSITORY_URL}/$filename" || log_error
popd > /dev/null
add_to_digest "$CURRENT_ACTION OK"
}
install_dep_if_needed()
{
CURRENT_ACTION="Install dep $1"
# Search for package in current repositories
# If package is not available we need to build it first
local dep="$1"
......@@ -85,6 +92,7 @@ install_dep_if_needed()
local match=$(pacman -Sl | grep " ${target/[<>=]*/} ")
if [[ -z "$match" ]]
then
add_to_digest "Build dep $dep"
echo ", install it from AUR..."
build_package "${target/[<>=]*/}"
install_package "${target/[<>=]*/}"
......@@ -102,24 +110,28 @@ install_dep_if_needed()
recipe_deps()
{
CURRENT_ACTION="hook=deps"
export SRCDIR
export REPOSITORY_NAME
export REPOSITORY_URL
"$SRCDIR/recipes/apply.sh" "$1" deps
"$SRCDIR/recipes/apply.sh" "$1" deps || log_error
}
recipe_pre()
{
"$SRCDIR/recipes/apply.sh" "$1" pre
CURRENT_ACTION="hook=pre"
"$SRCDIR/recipes/apply.sh" "$1" pre || log_error
}
recipe_post()
{
"$SRCDIR/recipes/apply.sh" "$1" post
CURRENT_ACTION="hook=post"
"$SRCDIR/recipes/apply.sh" "$1" post || log_error
}
build_package()
{
CURRENT_ACTION="Start package $1 build"
local pkg="$1"
pushd "$WORKDIR" > /dev/null
......@@ -162,20 +174,23 @@ build_package()
CURRENT_DEPENDENCY_TREE="$old_deps_tree"
echo -e "${TITLE_FMT}==> Call makepkg...${NO_FMT}"
makepkg
CURRENT_ACTION="makepkg $pkg"
makepkg || log_error
recipe_post "$pkg"
add_to_digest "Build OK"
popd > /dev/null
popd > /dev/null
}
get_current_version()
{
CURRENT_ACTION="Get current version for $1"
if [[ ! -f current-repo/current ]]
then
pushd "$SRCDIR" > /dev/null
"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}"
"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}" || log_error
popd > /dev/null
fi
......@@ -184,6 +199,7 @@ get_current_version()
get_aur_version()
{
CURRENT_ACTION="Get AUR version for $1"
curl -s "https://aur.archlinux.org/rpc/?v=5&type=info&arg=${1}" | jq -r .results[0].Version
}
......@@ -198,6 +214,11 @@ create_dirs()
then
mkdir -p "$SRCDIR/packages"
fi
if [[ ! -d "$SRCDIR/digest" ]]
then
mkdir -p "$SRCDIR/digest"
fi
}
force_rebuild()
......@@ -206,6 +227,7 @@ force_rebuild()
if [[ "${FORCE_REBUILD}" = "yes" ]]
then
echo "Force update"
add_to_digest "force rebuild"
create_dirs
build_package "$CURRENT_PACKAGE"
save_package "$CURRENT_PACKAGE"
......@@ -214,6 +236,46 @@ force_rebuild()
fi
}
DIGEST="$1"
DIGEST_FILE="$SRCDIR/digest/package.$1.txt"
add_to_digest()
{
DIGEST="${DIGEST}\n\t$1"
}
add_status_to_digest()
{
if [[ $1 -eq 0 ]]
then
add_to_digest "$2$3"
else
add_to_digest "$2$4"
fi
}
save_digest()
{
if [[ -d "$SRCDIR/digest" ]]
then
echo -e "$DIGEST" > "$DIGEST_FILE"
else
echo "Can't save digest, digest directory doesn't exists." > /dev/stderr
fi
}
CURRENT_ACTION="unknown"
log_error()
{
echo "Error while building $CURRENT_PACKAGE, dependency tree: $CURRENT_DEPENDENCY_TREE" > /dev/stderr
echo "Current action was: '$CURRENT_ACTION'" > /dev/stderr
add_to_digest "$CURRENT_ACTION NOK"
# We use 0 here to allow GitLab to save artifacts even if failure happened
exit 0
}
trap save_digest EXIT
trap log_error ERR
if [[ -z "$1" ]]
then
echo "Error: package name not specified as first parameter."
......@@ -249,13 +311,18 @@ CURRENT_DEPENDENCY_TREE="$1"
current_ver=$(get_current_version ${1})
aur_ver=$(get_aur_version ${1})
ver_diff=$(vercmp "${current_ver}" "${aur_ver}")
create_dirs
if [[ ( "$current_ver" ) && ( $ver_diff -eq 0 ) ]]
then
echo "Package is already up to date, no build needed"
echo "AUR Version: '$aur_ver'"
echo "Repo Version: '$current_ver'"
echo ""
create_dirs
add_to_digest "no update needed"
save_package_from_cache "$1"
force_rebuild
elif [[ ( "$current_ver" ) && ( $ver_diff -gt 0 ) ]]
......@@ -264,7 +331,9 @@ then
echo "AUR Version: '$aur_ver'"
echo "Repo Version: '$current_ver'"
echo ""
create_dirs
add_to_digest "AUR out-of-date"
save_package_from_cache "$1"
force_rebuild
else
......@@ -274,16 +343,17 @@ else
echo "AUR Version: '$aur_ver'"
echo "Repo Version: '$current_ver'"
echo ""
add_to_digest "Update $aur_ver -> $current_ver"
else
echo "Package is new, let's build it"
echo "AUR Version: '$aur_ver'"
echo "Repo Version: '$current_ver'"
echo ""
add_to_digest "New Package"
fi
create_dirs
build_package "$1"
save_package "$1"
fi
trap - EXIT
#!/bin/bash
set -e
get_database()
{
local REPOSITORY_URL="$1"
......@@ -14,7 +16,7 @@ get_database()
generate()
{
pushd current-repo > /dev/null
rm current
rm -f current
for i in *
do
if [[ -d "$i" ]]
......
#!/bin/bash
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
if [[ -f "$DIR/$1.sh" ]]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment