Merge branch 'arch-boxes-sha256' into 'master'

arch_boxes_sync: Pull the artifacts from GitLab's package registry

See merge request !580

roles/arch_boxes_sync/files/arch-boxes-sync.service

@@ -6,4 +6,4 @@ Type=oneshot
 ExecStart=/usr/local/bin/arch-boxes-sync.sh
 ProtectSystem=strict
 PrivateTmp=true
-ReadWritePaths=/srv/ftp/images
+ReadWritePaths=/srv/ftp/lastupdate /srv/ftp/images

roles/arch_boxes_sync/files/arch-boxes-sync.sh

@@ -2,46 +2,72 @@
 set -o nounset -o errexit -o pipefail
 # https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding
 readonly PROJECT_ID="archlinux%2Farch-boxes"
-readonly JOB_NAME="build:secure"
 readonly ARCH_BOXES_PATH="/srv/ftp/images"
+readonly LASTUPDATE_PATH="/srv/ftp/lastupdate"
 readonly MAX_RELEASES="6" # 3 months
 
-RELEASES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/releases")"
-LATEST_RELEASE_TAG="$(jq -r .[0].tag_name <<< "${RELEASES}")"
+PACKAGES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages?per_page=1&sort=desc")"
+LATEST_VERSION="$(jq -r .[0].version <<< "${PACKAGES}")"
 
-if [[ -d ${ARCH_BOXES_PATH}/${LATEST_RELEASE_TAG} ]]; then
+if [[ -d ${ARCH_BOXES_PATH}/${LATEST_VERSION} ]]; then
   echo "Nothing to do"
   exit
 fi
-echo "Adding release: ${LATEST_RELEASE_TAG}"
+
+# The files aren't uploaded atomic, so avoid missing files by requiring every package to be at least 5 minutes old.
+if (( $(date -d "-5 min" +%s) < $(date -d "$(jq -r .[0].created_at <<< "${PACKAGES}")" +%s) )); then
+  echo "Skipping release: ${LATEST_VERSION}, too new"
+  exit
+fi
+
+echo "Adding release: ${LATEST_VERSION}"
+
+PACKAGE_ID="$(jq -r .[0].id <<< "${PACKAGES}")"
+PACKAGE_NAME="$(jq -r .[0].name <<< "${PACKAGES}")"
+PACKAGE_FILES="$(curl --silent --show-error --fail "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/${PACKAGE_ID}/package_files")"
 
 readonly TMPDIR="$(mktemp --directory --tmpdir="/var/tmp")"
 trap "rm -rf \"${TMPDIR}\"" EXIT
 cd "${TMPDIR}"
 
-readonly HTTP_CODE="$(curl --silent --show-error --fail --output "output.zip" --write-out "%{http_code}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/jobs/artifacts/${LATEST_RELEASE_TAG}/download?job=${JOB_NAME}")"
-# The releases are released/tagged and then built, so the artifacts aren't necessarily ready (yet).
-if (( HTTP_CODE == 404 )); then
-  echo "Skipping release: ${LATEST_RELEASE_TAG}, artifacts not ready (404)"
-  exit
-fi
+mkdir "${LATEST_VERSION}"
+while IFS= read -r FILE; do
+  FILE_CREATED_AT="$(jq -r .created_at <<< "${FILE}")"
+  FILE_NAME="$(jq -r .file_name <<< "${FILE}")"
+  FILE_SHA256="$(jq -r .file_sha256 <<< "${FILE}")"
 
-mkdir "${LATEST_RELEASE_TAG}"
-unzip output.zip
-# People should download the vagrant images from Vagrant Cloud
-rm output/*.box{,.*}
-mv output/* "${LATEST_RELEASE_TAG}"
+  # People should download the vagrant images from Vagrant Cloud
+  if [[ $FILE_NAME =~ .*\.box(|\..*)$ ]]; then
+    continue
+  fi
 
-for FILE in "${LATEST_RELEASE_TAG}"/*; do
-  if [[ $FILE == *${LATEST_RELEASE_TAG:1}* ]]; then
-    FILE="${FILE##*/}"
-    ln -s "${FILE}" "${LATEST_RELEASE_TAG}/${FILE//-${LATEST_RELEASE_TAG:1}}"
+  curl --silent --show-error --fail --output "${LATEST_VERSION}/${FILE_NAME}" "https://gitlab.archlinux.org/api/v4/projects/${PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${LATEST_VERSION}/${FILE_NAME}"
+  sha256sum --quiet -c <<< "${FILE_SHA256} ${LATEST_VERSION}/${FILE_NAME}"
+  touch --no-create --date="@$(date -d "${FILE_CREATED_AT}" +%s)" "${LATEST_VERSION}/${FILE_NAME}"
+done < <(jq -c .[] <<< "${PACKAGE_FILES}")
+
+for FILE in "${LATEST_VERSION}"/*; do
+  if [[ $FILE == *${LATEST_VERSION:1}* ]]; then
+    DEST="${FILE//-${LATEST_VERSION:1}}"
+    if [[ $FILE =~ .*\.SHA256$ ]]; then
+      sed "s/-${LATEST_VERSION:1}//" "${FILE}" > "${DEST}"
+      touch --no-create --reference="${FILE}" "${DEST}"
+    # Don't create a symlink for the .SHA256.sig file, as we break the signature by fixing the checksum file.
+    elif [[ $FILE =~ .*\.SHA256.sig$ ]]; then
+      continue
+    else
+      SYMLINK="${FILE##*/}"
+      ln -s "${SYMLINK}" "${DEST}"
+      touch --no-create --reference="${FILE}" --no-dereference "${DEST}"
+    fi
   fi
 done
 
-mv "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/"
-ln -nsf "${LATEST_RELEASE_TAG}" "${ARCH_BOXES_PATH}/latest"
+mv "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/"
+ln -nsf "${LATEST_VERSION}" "${ARCH_BOXES_PATH}/latest"
 
 echo "Removing old releases"
 cd "${ARCH_BOXES_PATH}"
 comm --output-delimiter="" -3 <({ ls | grep -v latest | sort -r | head -n "${MAX_RELEASES}"; echo latest; } | sort) <(ls | sort) | tr -d '\0' | xargs --no-run-if-empty rm -rvf
+
+date +%s > "${LASTUPDATE_PATH}"