diff --git a/roles/arch_boxes_sync/files/arch-boxes-sync.service b/roles/arch_boxes_sync/files/arch-boxes-sync.service index 5965ae927d687f3cd535c15da6a024eaa78510cc..6b4d56fe8d2b3cb2726329c2d2c8f2ce99df0bda 100644 --- a/roles/arch_boxes_sync/files/arch-boxes-sync.service +++ b/roles/arch_boxes_sync/files/arch-boxes-sync.service @@ -6,4 +6,4 @@ Type=oneshot ExecStart=/usr/local/bin/arch-boxes-sync.sh ProtectSystem=strict PrivateTmp=true -ReadWritePaths=/srv/ftp/images +ReadWritePaths=/srv/ftp/lastupdate /srv/ftp/images diff --git a/roles/arch_boxes_sync/files/arch-boxes-sync.sh b/roles/arch_boxes_sync/files/arch-boxes-sync.sh index a81d458f5f830e1e9f9f7fee7ec62c214b7f746a..4a4b4a4eb243bd74a38a901e477cc44798866c9c 100755 --- a/roles/arch_boxes_sync/files/arch-boxes-sync.sh +++ b/roles/arch_boxes_sync/files/arch-boxes-sync.sh @@ -2,46 +2,72 @@ set -o nounset -o errexit -o pipefail # https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding readonly PROJECT_ID="archlinux%2Farch-boxes" -readonly JOB_NAME="build:secure" readonly ARCH_BOXES_PATH="/srv/ftp/images" +readonly LASTUPDATE_PATH="/srv/ftp/lastupdate" readonly MAX_RELEASES="6" # 3 months -RELEASES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/releases")" -LATEST_RELEASE_TAG="$(jq -r .[0].tag_name <<< "${RELEASES}")" +PACKAGES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages?per_page=1&sort=desc")" +LATEST_VERSION="$(jq -r .[0].version <<< "${PACKAGES}")" -if [[ -d ${ARCH_BOXES_PATH}/${LATEST_RELEASE_TAG} ]]; then +if [[ -d ${ARCH_BOXES_PATH}/${LATEST_VERSION} ]]; then echo "Nothing to do" exit fi -echo "Adding release: ${LATEST_RELEASE_TAG}" + +# The files aren't uploaded atomic, so avoid missing files by requiring every package to be at least 5 minutes old. +if (( $(date -d "-5 min" +%s) < $(date -d "$(jq -r .[0].created_at <<< "${PACKAGES}")" +%s) )); then + echo "Skipping release: ${LATEST_VERSION}, too new" + exit +fi + +echo "Adding release: ${LATEST_VERSION}" + +PACKAGE_ID="$(jq -r .[0].id <<< "${PACKAGES}")" +PACKAGE_NAME="$(jq -r .[0].name <<< "${PACKAGES}")" +PACKAGE_FILES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/${PACKAGE_ID}/package_files")" readonly TMPDIR="$(mktemp --directory --tmpdir="/var/tmp")" trap "rm -rf \"${TMPDIR}\"" EXIT cd "${TMPDIR}" -readonly HTTP_CODE="$(curl --silent --show-error --fail --output "output.zip" --write-out "%{http_code}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/jobs/artifacts/${LATEST_RELEASE_TAG}/download?job=${JOB_NAME}")" -# The releases are released/tagged and then built, so the artifacts aren't necessarily ready (yet). -if (( HTTP_CODE == 404 )); then - echo "Skipping release: ${LATEST_RELEASE_TAG}, artifacts not ready (404)" - exit -fi +mkdir "${LATEST_VERSION}" +while IFS= read -r FILE; do + FILE_CREATED_AT="$(jq -r .created_at <<< "${FILE}")" + FILE_NAME="$(jq -r .file_name <<< "${FILE}")" + FILE_SHA256="$(jq -r .file_sha256 <<< "${FILE}")" -mkdir "${LATEST_RELEASE_TAG}" -unzip output.zip -# People should download the vagrant images from Vagrant Cloud -rm output/*.box{,.*} -mv output/* "${LATEST_RELEASE_TAG}" + # People should download the vagrant images from Vagrant Cloud + if [[ $FILE_NAME =~ .*\.box(|\..*)$ ]]; then + continue + fi -for FILE in "${LATEST_RELEASE_TAG}"/*; do - if [[ $FILE == *${LATEST_RELEASE_TAG:1}* ]]; then - FILE="${FILE##*/}" - ln -s "${FILE}" "${LATEST_RELEASE_TAG}/${FILE//-${LATEST_RELEASE_TAG:1}}" + curl --silent --show-error --fail --output "${LATEST_VERSION}/${FILE_NAME}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${LATEST_VERSION}/${FILE_NAME}" + sha256sum --quiet -c <<< "${FILE_SHA256} ${LATEST_VERSION}/${FILE_NAME}" + touch --no-create --date="@$(date -d "${FILE_CREATED_AT}" +%s)" "${LATEST_VERSION}/${FILE_NAME}" +done < <(jq -c .[] <<< "${PACKAGE_FILES}") + +for FILE in "${LATEST_VERSION}"/*; do + if [[ $FILE == *${LATEST_VERSION:1}* ]]; then + DEST="${FILE//-${LATEST_VERSION:1}}" + if [[ $FILE =~ .*\.SHA256$ ]]; then + sed "s/-${LATEST_VERSION:1}//" "${FILE}" > "${DEST}" + touch --no-create --reference="${FILE}" "${DEST}" + # Don't create a symlink for the .SHA256.sig file, as we break the signature by fixing the checksum file. + elif [[ $FILE =~ .*\.SHA256.sig$ ]]; then + continue + else + SYMLINK="${FILE##*/}" + ln -s "${SYMLINK}" "${DEST}" + touch --no-create --reference="${FILE}" --no-dereference "${DEST}" + fi fi done -mv "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/" -ln -nsf "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/latest" +mv "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/" +ln -nsf "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/latest" echo "Removing old releases" cd "${ARCH_BOXES_PATH}" comm --output-delimiter="" -3 <({ ls | grep -v latest | sort -r | head -n "${MAX_RELEASES}"; echo latest; } | sort) <(ls | sort) | tr -d '\0' | xargs --no-run-if-empty rm -rvf + +date +%s > "${LASTUPDATE_PATH}"