From 0fdfac366e286c4b0b01b3e7651b0e589a101803 Mon Sep 17 00:00:00 2001 From: purian23 Date: Fri, 10 Jul 2026 17:57:28 -0400 Subject: [PATCH] ci(void): publish XBPS repository to R2 --- .github/scripts/publish-void-r2.sh | 170 +++++++++++++++++++++++++++++ .github/workflows/run-xbps.yml | 108 +++++++++++------- 2 files changed, 241 insertions(+), 37 deletions(-) create mode 100755 .github/scripts/publish-void-r2.sh diff --git a/.github/scripts/publish-void-r2.sh b/.github/scripts/publish-void-r2.sh new file mode 100755 index 000000000..4fc019710 --- /dev/null +++ b/.github/scripts/publish-void-r2.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + echo "usage: $0 download|publish" >&2 + exit 2 +} + +require_env() { + local name + for name in "$@"; do + if [[ -z "${!name:-}" ]]; then + echo "error: $name is required" >&2 + exit 2 + fi + done +} + +require_env R2_BUCKET R2_ENDPOINT R2_PREFIX REPOSITORY_DIR + +aws_r2() { + aws --endpoint-url "$R2_ENDPOINT" "$@" +} + +download() { + mkdir -p "$REPOSITORY_DIR/current" "$REPOSITORY_DIR/previous-current" + + aws_r2 s3 sync \ + "s3://${R2_BUCKET}/${R2_PREFIX}/current/" \ + "$REPOSITORY_DIR/current/" \ + --only-show-errors + + cp -a "$REPOSITORY_DIR/current/." "$REPOSITORY_DIR/previous-current/" +} + +build_manifest() { + local packages='[]' + local file filename pkgver name version revision sha size + + for file in "$REPOSITORY_DIR"/current/*.xbps; do + filename="$(basename "$file")" + pkgver="$(xbps-uhelper binpkgver "$filename")" + name="$(xbps-uhelper getpkgname "$pkgver")" + version="$(xbps-uhelper getpkgversion "$pkgver")" + revision="$(xbps-uhelper getpkgrevision "$pkgver")" + version="${version%_"${revision}"}" + sha="$(sha256sum "$file" | cut -d' ' -f1)" + size="$(stat -c '%s' "$file")" + packages="$(jq \ + --arg name "$name" \ + --arg version "$version" \ + --arg revision "$revision" \ + --arg filename "$filename" \ + --arg sha256 "$sha" \ + --argjson size "$size" \ + '. + [{name: $name, version: $version, revision: $revision, filename: $filename, sha256: $sha256, size: $size}]' \ + <<<"$packages")" + done + + jq -n \ + --arg repository "$R2_PREFIX" \ + --arg source_commit "$SOURCE_COMMIT" \ + --arg published_at "$PUBLISHED_AT" \ + --argjson packages "$packages" \ + '{schema: 1, repository: $repository, source_commit: $source_commit, published_at: $published_at, packages: $packages}' \ + > "$REPOSITORY_DIR/current/manifest.json" +} + +verify_immutable_packages() { + local file previous + + for file in "$REPOSITORY_DIR"/current/*.xbps "$REPOSITORY_DIR"/current/*.sig2; do + previous="$REPOSITORY_DIR/previous-current/$(basename "$file")" + if [[ -f "$previous" ]] && ! cmp -s "$previous" "$file"; then + echo "error: refusing to replace immutable object $(basename "$file")" >&2 + echo "bump the XBPS revision or version before publishing a changed build" >&2 + exit 1 + fi + done +} + +archive_retired() { + local old filename + local archive_prefix="archive/${R2_PREFIX}/${PUBLISHED_AT//:/-}" + + shopt -s nullglob + for old in "$REPOSITORY_DIR"/previous-current/*.xbps "$REPOSITORY_DIR"/previous-current/*.sig2; do + filename="$(basename "$old")" + if [[ ! -e "$REPOSITORY_DIR/current/$filename" ]]; then + aws_r2 s3 cp \ + "$old" \ + "s3://${R2_BUCKET}/${archive_prefix}/${filename}" \ + --cache-control 'private,no-store' \ + --only-show-errors + fi + done +} + +upload_current() { + local file filename old + + # Versioned package objects must exist before repodata can reference them. + for file in "$REPOSITORY_DIR"/current/*.xbps "$REPOSITORY_DIR"/current/*.sig2; do + filename="$(basename "$file")" + aws_r2 s3 cp \ + "$file" \ + "s3://${R2_BUCKET}/${R2_PREFIX}/current/${filename}" \ + --cache-control 'public,max-age=31536000,immutable' \ + --only-show-errors + done + + aws_r2 s3 cp \ + "$REPOSITORY_DIR/current/x86_64-repodata" \ + "s3://${R2_BUCKET}/${R2_PREFIX}/current/x86_64-repodata" \ + --cache-control 'no-cache' \ + --only-show-errors + + # The manifest is the publication marker and is always uploaded last. + aws_r2 s3 cp \ + "$REPOSITORY_DIR/current/manifest.json" \ + "s3://${R2_BUCKET}/${R2_PREFIX}/current/manifest.json" \ + --cache-control 'no-cache' \ + --only-show-errors + + # Once the new index and marker are live, remove objects no longer referenced. + shopt -s nullglob + for old in "$REPOSITORY_DIR"/previous-current/*; do + filename="$(basename "$old")" + if [[ ! -e "$REPOSITORY_DIR/current/$filename" ]]; then + aws_r2 s3 rm \ + "s3://${R2_BUCKET}/${R2_PREFIX}/current/${filename}" \ + --only-show-errors + fi + done +} + +publish() { + require_env SOURCE_COMMIT + PUBLISHED_AT="${PUBLISHED_AT:-$(date -u +'%Y-%m-%dT%H:%M:%SZ')}" + export PUBLISHED_AT + + shopt -s nullglob + local packages=("$REPOSITORY_DIR"/current/*.xbps) + if (( ${#packages[@]} == 0 )); then + echo "error: refusing to publish an empty XBPS repository" >&2 + exit 1 + fi + [[ -s "$REPOSITORY_DIR/current/x86_64-repodata" ]] || { + echo "error: x86_64-repodata is missing or empty" >&2 + exit 1 + } + for file in "${packages[@]}"; do + [[ -s "${file}.sig2" ]] || { + echo "error: signature is missing for $(basename "$file")" >&2 + exit 1 + } + done + + verify_immutable_packages + build_manifest + archive_retired + upload_current +} + +case "${1:-}" in + download) download ;; + publish) publish ;; + *) usage ;; +esac diff --git a/.github/workflows/run-xbps.yml b/.github/workflows/run-xbps.yml index 46dc2f506..ea813ddfc 100644 --- a/.github/workflows/run-xbps.yml +++ b/.github/workflows/run-xbps.yml @@ -34,13 +34,49 @@ on: default: true permissions: - contents: write + contents: read + +concurrency: + group: void-xbps-${{ github.ref }} + cancel-in-progress: false jobs: + preflight: + name: Check for unpublished changes + runs-on: ubuntu-latest + outputs: + should_build: ${{ steps.check.outputs.should_build }} + steps: + - name: Check published commit + id: check + env: + MANIFEST_URL: https://void.danklinux.com/dms/current/manifest.json + run: | + SHOULD_BUILD=true + if [ "${{ github.event_name }}" = "schedule" ]; then + PUBLISHED_COMMIT="$(curl -fsSL --retry 3 "$MANIFEST_URL" | jq -r '.source_commit // empty' 2>/dev/null || true)" + if [ "$PUBLISHED_COMMIT" = "${{ github.sha }}" ]; then + SHOULD_BUILD=false + echo "${{ github.sha }} is already published; skipping the scheduled build." + fi + fi + echo "should_build=$SHOULD_BUILD" >> "$GITHUB_OUTPUT" + build-and-deploy: name: Build & Deploy XBPS packages runs-on: ubuntu-latest - if: github.repository == 'AvengeMedia/DankMaterialShell' + needs: preflight + if: github.repository == 'AvengeMedia/DankMaterialShell' && needs.preflight.outputs.should_build == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_EC2_METADATA_DISABLED: true + R2_BUCKET: danklinux-void + R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com + R2_PREFIX: dms + REPOSITORY_DIR: ${{ github.workspace }}/r2-repo + SOURCE_COMMIT: ${{ github.sha }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -48,17 +84,13 @@ jobs: ref: ${{ github.event.release.tag_name || (github.event.inputs.version && (startsWith(github.event.inputs.version, 'v') && github.event.inputs.version || format('v{0}', github.event.inputs.version))) || github.ref }} fetch-depth: 0 - - name: Checkout gh-pages branch + - name: Record checked-out source commit + run: echo "SOURCE_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + + - name: Download current R2 repository run: | - git clone --branch gh-pages https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git gh-pages-repo || { - echo "⚠️ gh-pages branch not found or empty, initializing a new one..." - mkdir gh-pages-repo - cd gh-pages-repo - git init - git checkout -b gh-pages - git remote add origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git - cd .. - } + aws --version + bash .github/scripts/publish-void-r2.sh download - name: Install XBPS Static Utilities run: | @@ -132,19 +164,19 @@ jobs: - name: Configure repositories run: | # Append the repository to repos-remote templates so xbps-src translates it automatically - echo "repository=https://avengemedia.github.io/DankLinux/current" >> void-packages/etc/xbps.d/repos-remote.conf - echo "repository=https://avengemedia.github.io/DankLinux/current" >> void-packages/etc/xbps.d/repos-remote-x86_64-multilib.conf + echo "repository=https://void.danklinux.com/danklinux/current" >> void-packages/etc/xbps.d/repos-remote.conf + echo "repository=https://void.danklinux.com/danklinux/current" >> void-packages/etc/xbps.d/repos-remote-x86_64-multilib.conf # Add any existing compiled packages to the build cache directory to avoid rebuilds - if [ -d "gh-pages-repo/current" ]; then + if [ -d "r2-repo/current" ]; then mkdir -p void-packages/hostdir/binpkgs - cp -L gh-pages-repo/current/*.xbps void-packages/hostdir/binpkgs/ 2>/dev/null || true + cp -L r2-repo/current/*.xbps void-packages/hostdir/binpkgs/ 2>/dev/null || true xbps-rindex -a void-packages/hostdir/binpkgs/*.xbps 2>/dev/null || true fi - name: Build packages run: | - mkdir -p gh-pages-repo/current + mkdir -p r2-repo/current BUILD_DMS="${{ github.event.inputs.build_dms || 'true' }}" BUILD_GREETER="${{ github.event.inputs.build_greeter || 'true' }}" @@ -216,7 +248,7 @@ jobs: mkdir -p "$SRC_CACHE" tar -czf "${SRC_CACHE}/dms-git-${GIT_VER}.tar.gz" \ --exclude=void-packages \ - --exclude=gh-pages-repo \ + --exclude=r2-repo \ --exclude=.git \ --exclude=danklinux \ -C .. . @@ -230,13 +262,13 @@ jobs: EXPECTED_GIT_FILE="dms-git-${GIT_VER}_1.x86_64.xbps" - if [ -f "../gh-pages-repo/current/$EXPECTED_GIT_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then + if [ -f "../r2-repo/current/$EXPECTED_GIT_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then echo "✅ $EXPECTED_GIT_FILE already exists, skipping build." else echo "🔨 Compiling dms-git..." ./xbps-src pkg dms-git - rm -f "../gh-pages-repo/current/${EXPECTED_GIT_FILE}" - cp -L hostdir/binpkgs/dms-git-*.xbps ../gh-pages-repo/current/ + rm -f "../r2-repo/current/${EXPECTED_GIT_FILE}" + cp -L "hostdir/binpkgs/${EXPECTED_GIT_FILE}" ../r2-repo/current/ fi fi @@ -246,13 +278,13 @@ jobs: STABLE_REV=$(grep -E '^revision=' srcpkgs/dms/template | cut -d= -f2 | tr -d '"') EXPECTED_DMS_FILE="dms-${STABLE_VER}_${STABLE_REV}.x86_64.xbps" - if [ -f "../gh-pages-repo/current/$EXPECTED_DMS_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then + if [ -f "../r2-repo/current/$EXPECTED_DMS_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then echo "✅ $EXPECTED_DMS_FILE already exists, skipping build." else echo "🔨 Compiling dms ($STABLE_VER)..." ./xbps-src pkg dms - rm -f "../gh-pages-repo/current/${EXPECTED_DMS_FILE}" - cp -L hostdir/binpkgs/dms-${STABLE_VER}_${STABLE_REV}.x86_64.xbps ../gh-pages-repo/current/ + rm -f "../r2-repo/current/${EXPECTED_DMS_FILE}" + cp -L hostdir/binpkgs/dms-${STABLE_VER}_${STABLE_REV}.x86_64.xbps ../r2-repo/current/ fi fi @@ -262,22 +294,21 @@ jobs: GREETER_REV=$(grep -E '^revision=' srcpkgs/dms-greeter/template | cut -d= -f2 | tr -d '"') EXPECTED_GREETER_FILE="dms-greeter-${GREETER_VER}_${GREETER_REV}.x86_64.xbps" - if [ -f "../gh-pages-repo/current/$EXPECTED_GREETER_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then + if [ -f "../r2-repo/current/$EXPECTED_GREETER_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then echo "✅ $EXPECTED_GREETER_FILE already exists, skipping build." else echo "🔨 Compiling dms-greeter ($GREETER_VER)..." ./xbps-src pkg dms-greeter - rm -f "../gh-pages-repo/current/${EXPECTED_GREETER_FILE}" - cp -L hostdir/binpkgs/dms-greeter-${GREETER_VER}_${GREETER_REV}.x86_64.xbps ../gh-pages-repo/current/ + rm -f "../r2-repo/current/${EXPECTED_GREETER_FILE}" + cp -L hostdir/binpkgs/dms-greeter-${GREETER_VER}_${GREETER_REV}.x86_64.xbps ../r2-repo/current/ fi fi - name: Index and sign repository run: | - cd gh-pages-repo/current + cd r2-repo/current - # Clean up any stale or dangling signature files to prevent O_CREAT ENOENT errors - rm -f *.sig2 *.sig + rm -f *.sig # Guard: nothing to index if no .xbps files exist if ! ls *.xbps 1>/dev/null 2>&1; then @@ -287,6 +318,14 @@ jobs: # Regenerate repo index xbps-rindex -a $(pwd)/*.xbps + xbps-rindex --remove-obsoletes $(pwd) + + # Remove signatures left behind by obsolete packages. Signatures for + # unchanged immutable packages are retained byte-for-byte. + for sig in *.sig2; do + [ -e "$sig" ] || break + [ -f "${sig%.sig2}" ] || rm -f "$sig" + done # Sign repository echo "${{ secrets.XBPS_PRIVATE_KEY }}" > /tmp/xbps_privkey.pem @@ -297,10 +336,5 @@ jobs: rm -f /tmp/xbps_privkey.pem - - name: Deploy to gh-pages branch - run: | - cd gh-pages-repo - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add current/ - git diff --quiet && git diff --staged --quiet || (git commit -m "Update XBPS packages [skip ci]" && git push origin gh-pages) + - name: Publish repository to R2 + run: bash .github/scripts/publish-void-r2.sh publish