From ae34cd11c5db017d0cd108dee76c5b1c682fc3d2 Mon Sep 17 00:00:00 2001
From: Kristian Klausen <kristian@klausen.dk>
Date: Sun, 29 May 2022 20:11:05 +0200
Subject: [PATCH] arch_boxes_sync: Pull the artifacts from GitLab's package
registry
arch-boxes has decided to use GitLab's package registry instead of job
artifacts[1].
[1] https://gitlab.archlinux.org/archlinux/arch-boxes/-/commit/d04c827450880f60d015e910f53da65971635b01
---
.../arch_boxes_sync/files/arch-boxes-sync.sh | 57 ++++++++++++-------
1 file changed, 35 insertions(+), 22 deletions(-)
diff --git a/roles/arch_boxes_sync/files/arch-boxes-sync.sh b/roles/arch_boxes_sync/files/arch-boxes-sync.sh
index 8dc5ccc5a..4e589875a 100755
--- a/roles/arch_boxes_sync/files/arch-boxes-sync.sh
+++ b/roles/arch_boxes_sync/files/arch-boxes-sync.sh
@@ -2,41 +2,54 @@
set -o nounset -o errexit -o pipefail
# https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
readonly PROJECT_ID="archlinux%2Farch-boxes"
-readonly JOB_NAME="build:secure"
readonly ARCH_BOXES_PATH="/srv/ftp/images"
readonly MAX_RELEASES="6" # 3 months
-RELEASES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/releases")"
-LATEST_RELEASE_TAG="$(jq -r .[0].tag_name <<< "${RELEASES}")"
+PACKAGES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages?per_page=1&sort=desc")"
+LATEST_VERSION="$(jq -r .[0].version <<< "${PACKAGES}")"
-if [[ -d ${ARCH_BOXES_PATH}/${LATEST_RELEASE_TAG} ]]; then
+if [[ -d ${ARCH_BOXES_PATH}/${LATEST_VERSION} ]]; then
echo "Nothing to do"
exit
fi
-echo "Adding release: ${LATEST_RELEASE_TAG}"
+
+# The files aren't uploaded atomic, so avoid missing files by requiring every package to be at least 5 minutes old.
+if (( $(date -d "-5 min" +%s) < $(date -d "$(jq -r .[0].created_at <<< "${PACKAGES}")" +%s) )); then
+ echo "Skipping release: ${LATEST_VERSION}, too new"
+ exit
+fi
+
+echo "Adding release: ${LATEST_VERSION}"
+
+PACKAGE_ID="$(jq -r .[0].id <<< "${PACKAGES}")"
+PACKAGE_NAME="$(jq -r .[0].name <<< "${PACKAGES}")"
+PACKAGE_FILES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/${PACKAGE_ID}/package_files")"
readonly TMPDIR="$(mktemp --directory --tmpdir="/var/tmp")"
trap "rm -rf \"${TMPDIR}\"" EXIT
cd "${TMPDIR}"
-readonly HTTP_CODE="$(curl --silent --show-error --fail --output "output.zip" --write-out "%{http_code}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/jobs/artifacts/${LATEST_RELEASE_TAG}/download?job=${JOB_NAME}")"
-# The releases are released/tagged and then built, so the artifacts aren't necessarily ready (yet).
-if (( HTTP_CODE == 404 )); then
- echo "Skipping release: ${LATEST_RELEASE_TAG}, artifacts not ready (404)"
- exit
-fi
+mkdir "${LATEST_VERSION}"
+while IFS= read -r FILE; do
+ FILE_CREATED_AT="$(jq -r .created_at <<< "${FILE}")"
+ FILE_NAME="$(jq -r .file_name <<< "${FILE}")"
+ FILE_SHA256="$(jq -r .file_sha256 <<< "${FILE}")"
+
+ # People should download the vagrant images from Vagrant Cloud
+ if [[ $FILE_NAME =~ .*\.box(|\..*)$ ]]; then
+ continue
+ fi
-mkdir "${LATEST_RELEASE_TAG}"
-unzip output.zip
-# People should download the vagrant images from Vagrant Cloud
-rm output/*.box{,.*}
-mv output/* "${LATEST_RELEASE_TAG}"
+ curl --silent --show-error --fail --output "${LATEST_VERSION}/${FILE_NAME}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${LATEST_VERSION}/${FILE_NAME}"
+ sha256sum --quiet -c <<< "${FILE_SHA256} ${LATEST_VERSION}/${FILE_NAME}"
+ touch --no-create --date="@$(date -d "${FILE_CREATED_AT}" +%s)" "${LATEST_VERSION}/${FILE_NAME}"
+done < <(jq -c .[] <<< "${PACKAGE_FILES}")
-for FILE in "${LATEST_RELEASE_TAG}"/*; do
- if [[ $FILE == *${LATEST_RELEASE_TAG:1}* ]]; then
- DEST="${FILE//-${LATEST_RELEASE_TAG:1}}"
+for FILE in "${LATEST_VERSION}"/*; do
+ if [[ $FILE == *${LATEST_VERSION:1}* ]]; then
+ DEST="${FILE//-${LATEST_VERSION:1}}"
if [[ $FILE =~ .*\.SHA256$ ]]; then
- sed "s/-${LATEST_RELEASE_TAG:1}//" "${FILE}" > "${DEST}"
+ sed "s/-${LATEST_VERSION:1}//" "${FILE}" > "${DEST}"
touch --no-create --reference="${FILE}" "${DEST}"
# Don't create a symlink for the .SHA256.sig file, as we break the signature by fixing the checksum file.
elif [[ $FILE =~ .*\.SHA256.sig$ ]]; then
@@ -48,8 +61,8 @@ for FILE in "${LATEST_RELEASE_TAG}"/*; do
fi
done
-mv "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/"
-ln -nsf "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/latest"
+mv "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/"
+ln -nsf "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/latest"
echo "Removing old releases"
cd "${ARCH_BOXES_PATH}"
--
GitLab