Merge branch 'build-digest' into 'master'

Create digest and save it along with repo artifacts

See merge request !2

.gitlab-ci.yml

@@ -11,6 +11,7 @@ stages:
   artifacts:
     paths:
       - packages
+      - digest
     expire_in: 2 days
   before_script:
     # We update just update the database and we don't upgrade packages
@@ -119,6 +120,7 @@ generate-repository-database:
     expire_in: 2 days
   script:
     - repo-add "./packages/${REPOSITORY_NAME}.db.tar.gz" "./packages/"*.pkg.tar.xz
+    - cat ./digest/package.*.txt > "./packages/latest_build_digest.txt"
   tags:
     - executor:docker
 
@@ -144,4 +146,3 @@ deploy:
     url: $REPOSITORY_URL
   tags:
     - executor:docker
-

build.sh

@@ -9,9 +9,6 @@ NO_FMT="\e[0m"
 CURRENT_PACKAGE=""
 CURRENT_DEPENDENCY_TREE=""
 
-set -e
-trap 'echo "Building $CURRENT_PACKAGE, dependency tree: $CURRENT_DEPENDENCY_TREE"' EXIT
-
 invoke_pacman()
 {
 	sudo pacman "$@"
@@ -19,12 +16,14 @@ invoke_pacman()
 
 download_package()
 {
-	curl --location "https://aur.archlinux.org/cgit/aur.git/snapshot/$1.tar.gz" > "$1.tar.gz"
+	CURRENT_ACTION="Download https://aur.archlinux.org/cgit/aur.git/snapshot/$1.tar.gz"
+	curl --location "https://aur.archlinux.org/cgit/aur.git/snapshot/$1.tar.gz" > "$1.tar.gz" || log_error
 }
 
 extract_package()
 {
-	tar -xf "$1.tar.gz"
+	CURRENT_ACTION="Extract $1.tar.gz in $PWD"
+	tar -xf "$1.tar.gz" || log_error
 }
 
 package_get_makedepends()
@@ -44,32 +43,40 @@ package_get_depends()
 
 install_package()
 {
-	invoke_pacman -U --noconfirm "$WORKDIR/$1/"*.pkg.tar.xz
+	CURRENT_ACTION="Install package from disk"
+	invoke_pacman -U --noconfirm "$WORKDIR/$1/"*.pkg.tar.xz || log_error
 }
 
 save_package()
 {
-	cp "$WORKDIR/$1/"*.pkg.tar.xz "$SRCDIR/packages"
+	CURRENT_ACTION="Save package as artifact"
+	cp "$WORKDIR/$1/"*.pkg.tar.xz "$SRCDIR/packages" || log_error
+	add_to_digest "Save OK"
 }
 
 save_package_from_cache()
 {
+	CURRENT_ACTION="Cache Fetch"
+
 	if [[ ! -f current-repo/current ]]
 	then
 		pushd "$SRCDIR" > /dev/null
-		"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}"
+		"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}" || log_error
 		popd > /dev/null
 	fi
 
 	local filename=$(grep -P "^$1\t" "$SRCDIR/current-repo/current" | cut -f 3)
 
 	pushd "$SRCDIR/packages" > /dev/null
-	curl -O "${REPOSITORY_URL}/$filename"
+	curl -O "${REPOSITORY_URL}/$filename" || log_error
 	popd > /dev/null
+
+	add_to_digest "$CURRENT_ACTION OK"
 }
 
 install_dep_if_needed()
 {
+	CURRENT_ACTION="Install dep $1"
 	# Search for package in current repositories
 	# If package is not available we need to build it first
 	local dep="$1"
@@ -85,6 +92,7 @@ install_dep_if_needed()
 		local match=$(pacman -Sl | grep " ${target/[<>=]*/} ")
 		if [[ -z "$match" ]]
 		then
+			add_to_digest "Build dep $dep"
 			echo ", install it from AUR..."
 			build_package "${target/[<>=]*/}"
 			install_package "${target/[<>=]*/}"
@@ -102,24 +110,28 @@ install_dep_if_needed()
 
 recipe_deps()
 {
+	CURRENT_ACTION="hook=deps"
 	export SRCDIR
 	export REPOSITORY_NAME
 	export REPOSITORY_URL
-	"$SRCDIR/recipes/apply.sh" "$1" deps
+	"$SRCDIR/recipes/apply.sh" "$1" deps || log_error
 }
 
 recipe_pre()
 {
-	"$SRCDIR/recipes/apply.sh" "$1" pre
+	CURRENT_ACTION="hook=pre"
+	"$SRCDIR/recipes/apply.sh" "$1" pre || log_error
 }
 
 recipe_post()
 {
-	"$SRCDIR/recipes/apply.sh" "$1" post
+	CURRENT_ACTION="hook=post"
+	"$SRCDIR/recipes/apply.sh" "$1" post || log_error
 }
 
 build_package()
 {
+	CURRENT_ACTION="Start package $1 build"
 	local pkg="$1"
 
 	pushd "$WORKDIR" > /dev/null
@@ -162,20 +174,23 @@ build_package()
 	CURRENT_DEPENDENCY_TREE="$old_deps_tree"
 
 	echo -e "${TITLE_FMT}==> Call makepkg...${NO_FMT}"
-	makepkg
-
+	CURRENT_ACTION="makepkg $pkg"
+	makepkg || log_error
+	
 	recipe_post "$pkg"
-
+	add_to_digest "Build OK"
+	
 	popd > /dev/null
 	popd > /dev/null
 }
 
 get_current_version()
 {
+	CURRENT_ACTION="Get current version for $1"
 	if [[ ! -f current-repo/current ]]
 	then
 		pushd "$SRCDIR" > /dev/null
-		"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}"
+		"$SRCDIR/generate-current-status.sh" "${REPOSITORY_URL}" "${REPOSITORY_NAME}" || log_error
 		popd > /dev/null
 	fi
 
@@ -184,6 +199,7 @@ get_current_version()
 
 get_aur_version()
 {
+	CURRENT_ACTION="Get AUR version for $1"
 	curl -s "https://aur.archlinux.org/rpc/?v=5&type=info&arg=${1}" | jq -r .results[0].Version
 }
 
@@ -198,6 +214,11 @@ create_dirs()
 	then
 		mkdir -p "$SRCDIR/packages"
 	fi
+
+	if [[ ! -d "$SRCDIR/digest" ]]
+	then
+		mkdir -p "$SRCDIR/digest"
+	fi
 }
 
 force_rebuild()
@@ -206,6 +227,7 @@ force_rebuild()
 	if [[ "${FORCE_REBUILD}" = "yes" ]]
 	then
 		echo "Force update"
+		add_to_digest "force rebuild"
 		create_dirs
 		build_package "$CURRENT_PACKAGE"
 		save_package "$CURRENT_PACKAGE"
@@ -214,6 +236,46 @@ force_rebuild()
 	fi
 }
 
+DIGEST="$1"
+DIGEST_FILE="$SRCDIR/digest/package.$1.txt"
+add_to_digest()
+{
+	DIGEST="${DIGEST}\n\t$1"
+}
+
+add_status_to_digest()
+{
+	if [[ $1 -eq 0 ]]
+	then
+		add_to_digest "$2$3"
+	else
+		add_to_digest "$2$4"
+	fi
+}
+
+save_digest()
+{
+	if [[ -d "$SRCDIR/digest" ]]
+	then
+		echo -e "$DIGEST" > "$DIGEST_FILE"
+	else
+		echo "Can't save digest, digest directory doesn't exists." > /dev/stderr
+	fi
+}
+
+CURRENT_ACTION="unknown"
+log_error()
+{
+	echo "Error while building $CURRENT_PACKAGE, dependency tree: $CURRENT_DEPENDENCY_TREE" > /dev/stderr
+	echo "Current action was: '$CURRENT_ACTION'" > /dev/stderr
+	add_to_digest "$CURRENT_ACTION NOK"
+	# We use 0 here to allow GitLab to save artifacts even if failure happened
+	exit 0
+}
+
+trap save_digest EXIT
+trap log_error ERR
+
 if [[ -z "$1" ]]
 then
 	echo "Error: package name not specified as first parameter."
@@ -249,13 +311,18 @@ CURRENT_DEPENDENCY_TREE="$1"
 current_ver=$(get_current_version ${1})
 aur_ver=$(get_aur_version ${1})
 ver_diff=$(vercmp "${current_ver}" "${aur_ver}")
+
+create_dirs
+
 if [[ ( "$current_ver" ) && ( $ver_diff -eq 0 ) ]]
 then
 	echo "Package is already up to date, no build needed"
 	echo "AUR Version:  '$aur_ver'"
 	echo "Repo Version: '$current_ver'"
 	echo ""
-	create_dirs
+
+	add_to_digest "no update needed"
+
 	save_package_from_cache "$1"
 	force_rebuild
 elif [[ ( "$current_ver" ) && ( $ver_diff -gt 0 ) ]]
@@ -264,7 +331,9 @@ then
 	echo "AUR Version:  '$aur_ver'"
 	echo "Repo Version: '$current_ver'"
 	echo ""
-	create_dirs
+
+	add_to_digest "AUR out-of-date"
+
 	save_package_from_cache "$1"
 	force_rebuild
 else
@@ -274,16 +343,17 @@ else
 		echo "AUR Version:  '$aur_ver'"
 		echo "Repo Version: '$current_ver'"
 		echo ""
+
+		add_to_digest "Update $aur_ver -> $current_ver"
 	else
 		echo "Package is new, let's build it"
 		echo "AUR Version:  '$aur_ver'"
 		echo "Repo Version: '$current_ver'"
 		echo ""
+
+		add_to_digest "New Package"
 	fi
 
-	create_dirs
 	build_package "$1"
 	save_package "$1"
 fi
-
-trap - EXIT

generate-current-status.sh

 #!/bin/bash
 
+set -e
+
 get_database()
 {
 	local REPOSITORY_URL="$1"
@@ -14,7 +16,7 @@ get_database()
 generate()
 {
 	pushd current-repo > /dev/null
-	rm current
+	rm -f current
 	for i in *
 	do
 		if [[ -d "$i" ]]

recipes/apply.sh

 #!/bin/bash
 
+set -e
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 if [[ -f "$DIR/$1.sh" ]]