Skip to content
Snippets Groups Projects
Verified Commit f34b16d3 authored by Kristian Klausen's avatar Kristian Klausen :tada:
Browse files

Merge branch 'arch-boxes-sha256' into 'master'

arch_boxes_sync: Pull the artifacts from GitLab's package registry

See merge request !580
parents e7aa193b 159ff401
No related branches found
No related tags found
1 merge request!580arch_boxes_sync: Pull the artifacts from GitLab's package registry
Pipeline #30079 passed
......@@ -6,4 +6,4 @@ Type=oneshot
ExecStart=/usr/local/bin/arch-boxes-sync.sh
ProtectSystem=strict
PrivateTmp=true
ReadWritePaths=/srv/ftp/images
ReadWritePaths=/srv/ftp/lastupdate /srv/ftp/images
......@@ -2,46 +2,72 @@
set -o nounset -o errexit -o pipefail
# https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
readonly PROJECT_ID="archlinux%2Farch-boxes"
readonly JOB_NAME="build:secure"
readonly ARCH_BOXES_PATH="/srv/ftp/images"
readonly LASTUPDATE_PATH="/srv/ftp/lastupdate"
readonly MAX_RELEASES="6" # 3 months
RELEASES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/releases")"
LATEST_RELEASE_TAG="$(jq -r .[0].tag_name <<< "${RELEASES}")"
PACKAGES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages?per_page=1&sort=desc")"
LATEST_VERSION="$(jq -r .[0].version <<< "${PACKAGES}")"
if [[ -d ${ARCH_BOXES_PATH}/${LATEST_RELEASE_TAG} ]]; then
if [[ -d ${ARCH_BOXES_PATH}/${LATEST_VERSION} ]]; then
echo "Nothing to do"
exit
fi
echo "Adding release: ${LATEST_RELEASE_TAG}"
# The files aren't uploaded atomic, so avoid missing files by requiring every package to be at least 5 minutes old.
if (( $(date -d "-5 min" +%s) < $(date -d "$(jq -r .[0].created_at <<< "${PACKAGES}")" +%s) )); then
echo "Skipping release: ${LATEST_VERSION}, too new"
exit
fi
echo "Adding release: ${LATEST_VERSION}"
PACKAGE_ID="$(jq -r .[0].id <<< "${PACKAGES}")"
PACKAGE_NAME="$(jq -r .[0].name <<< "${PACKAGES}")"
PACKAGE_FILES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/${PACKAGE_ID}/package_files")"
readonly TMPDIR="$(mktemp --directory --tmpdir="/var/tmp")"
trap "rm -rf \"${TMPDIR}\"" EXIT
cd "${TMPDIR}"
readonly HTTP_CODE="$(curl --silent --show-error --fail --output "output.zip" --write-out "%{http_code}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/jobs/artifacts/${LATEST_RELEASE_TAG}/download?job=${JOB_NAME}")"
# The releases are released/tagged and then built, so the artifacts aren't necessarily ready (yet).
if (( HTTP_CODE == 404 )); then
echo "Skipping release: ${LATEST_RELEASE_TAG}, artifacts not ready (404)"
exit
fi
mkdir "${LATEST_VERSION}"
while IFS= read -r FILE; do
FILE_CREATED_AT="$(jq -r .created_at <<< "${FILE}")"
FILE_NAME="$(jq -r .file_name <<< "${FILE}")"
FILE_SHA256="$(jq -r .file_sha256 <<< "${FILE}")"
mkdir "${LATEST_RELEASE_TAG}"
unzip output.zip
# People should download the vagrant images from Vagrant Cloud
rm output/*.box{,.*}
mv output/* "${LATEST_RELEASE_TAG}"
# People should download the vagrant images from Vagrant Cloud
if [[ $FILE_NAME =~ .*\.box(|\..*)$ ]]; then
continue
fi
for FILE in "${LATEST_RELEASE_TAG}"/*; do
if [[ $FILE == *${LATEST_RELEASE_TAG:1}* ]]; then
FILE="${FILE##*/}"
ln -s "${FILE}" "${LATEST_RELEASE_TAG}/${FILE//-${LATEST_RELEASE_TAG:1}}"
curl --silent --show-error --fail --output "${LATEST_VERSION}/${FILE_NAME}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${LATEST_VERSION}/${FILE_NAME}"
sha256sum --quiet -c <<< "${FILE_SHA256} ${LATEST_VERSION}/${FILE_NAME}"
touch --no-create --date="@$(date -d "${FILE_CREATED_AT}" +%s)" "${LATEST_VERSION}/${FILE_NAME}"
done < <(jq -c .[] <<< "${PACKAGE_FILES}")
for FILE in "${LATEST_VERSION}"/*; do
if [[ $FILE == *${LATEST_VERSION:1}* ]]; then
DEST="${FILE//-${LATEST_VERSION:1}}"
if [[ $FILE =~ .*\.SHA256$ ]]; then
sed "s/-${LATEST_VERSION:1}//" "${FILE}" > "${DEST}"
touch --no-create --reference="${FILE}" "${DEST}"
# Don't create a symlink for the .SHA256.sig file, as we break the signature by fixing the checksum file.
elif [[ $FILE =~ .*\.SHA256.sig$ ]]; then
continue
else
SYMLINK="${FILE##*/}"
ln -s "${SYMLINK}" "${DEST}"
touch --no-create --reference="${FILE}" --no-dereference "${DEST}"
fi
fi
done
mv "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/"
ln -nsf "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/latest"
mv "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/"
ln -nsf "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/latest"
echo "Removing old releases"
cd "${ARCH_BOXES_PATH}"
comm --output-delimiter="" -3 <({ ls | grep -v latest | sort -r | head -n "${MAX_RELEASES}"; echo latest; } | sort) <(ls | sort) | tr -d '\0' | xargs --no-run-if-empty rm -rvf
date +%s > "${LASTUPDATE_PATH}"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment