From fe1241bf9147472b00eaa805281c3f9100292092 Mon Sep 17 00:00:00 2001 From: purian23 Date: Sat, 24 Oct 2026 15:50:01 -0400 Subject: [PATCH] ci: dispatch distro builds from Release & guard shared Void index (cherry picked from commit 3388f789051b2f1518a878ab74d9d9212e3e4b7b) --- .github/scripts/publish-void-r2.sh | 77 ++++++++++++++++++++++++++--- .github/workflows/point-release.yml | 2 +- .github/workflows/release.yml | 35 +++++++++++++ .github/workflows/run-xbps.yml | 8 +-- 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/.github/scripts/publish-void-r2.sh b/.github/scripts/publish-void-r2.sh index 4fc019710..56f164468 100755 --- a/.github/scripts/publish-void-r2.sh +++ b/.github/scripts/publish-void-r2.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Download / publish an XBPS repository prefix on Cloudflare R2. +# Shared by dux-shell and dux-packaging (same prefix); keep both copies identical. +# ETag guard: publish aborts if another writer replaced x86_64-repodata mid-run. set -euo pipefail @@ -19,13 +22,54 @@ require_env() { require_env R2_BUCKET R2_ENDPOINT R2_PREFIX REPOSITORY_DIR +REPODATA_KEY="${R2_PREFIX}/current/x86_64-repodata" +ETAG_STATE="$REPOSITORY_DIR/.repodata-etag" + aws_r2() { aws --endpoint-url "$R2_ENDPOINT" "$@" } +# Print the remote repodata ETag, or "none" when the object does not exist yet. +remote_repodata_etag() { + local out + if out="$(aws_r2 s3api head-object --bucket "$R2_BUCKET" --key "$REPODATA_KEY" \ + --query ETag --output text 2>&1)"; then + printf '%s\n' "$out" + elif grep -qiE 'Not Found|404' <<<"$out"; then + echo none + else + printf '%s\n' "$out" >&2 + return 1 + fi +} + +record_repodata_etag() { + remote_repodata_etag > "$ETAG_STATE" +} + +# Abort when another publisher replaced the index since we recorded it. +assert_repodata_unchanged() { + local stage="$1" expected actual + if [[ ! -s "$ETAG_STATE" ]]; then + echo "error: $ETAG_STATE is missing; run '$0 download' before publishing" >&2 + exit 1 + fi + expected="$(<"$ETAG_STATE")" + actual="$(remote_repodata_etag)" + if [[ "$expected" != "$actual" ]]; then + echo "error: ${REPODATA_KEY} changed while this run was building ($stage)" >&2 + echo " recorded ETag: $expected" >&2 + echo " current ETag: $actual" >&2 + echo "another publisher won the race; nothing was overwritten. Re-run to retry." >&2 + exit 1 + fi +} + download() { mkdir -p "$REPOSITORY_DIR/current" "$REPOSITORY_DIR/previous-current" + # ETag must be taken before the sync so a mid-sync publish is detected. + record_repodata_etag aws_r2 s3 sync \ "s3://${R2_BUCKET}/${R2_PREFIX}/current/" \ "$REPOSITORY_DIR/current/" \ @@ -35,7 +79,7 @@ download() { } build_manifest() { - local packages='[]' + local packages='[]' previous_sources='{}' local file filename pkgver name version revision sha size for file in "$REPOSITORY_DIR"/current/*.xbps; do @@ -58,12 +102,21 @@ build_manifest() { <<<"$packages")" done + # `sources` maps publishing repo -> last published commit; `source_commit` kept for older consumers. + if [[ -s "$REPOSITORY_DIR/previous-current/manifest.json" ]]; then + previous_sources="$(jq -c '.sources // {}' "$REPOSITORY_DIR/previous-current/manifest.json" 2>/dev/null || echo '{}')" + fi + jq -n \ --arg repository "$R2_PREFIX" \ + --arg source_repository "$SOURCE_REPOSITORY" \ --arg source_commit "$SOURCE_COMMIT" \ --arg published_at "$PUBLISHED_AT" \ + --argjson previous_sources "$previous_sources" \ --argjson packages "$packages" \ - '{schema: 1, repository: $repository, source_commit: $source_commit, published_at: $published_at, packages: $packages}' \ + '{schema: 1, repository: $repository, source_repository: $source_repository, source_commit: $source_commit, + sources: ($previous_sources + {($source_repository): $source_commit}), + published_at: $published_at, packages: $packages}' \ > "$REPOSITORY_DIR/current/manifest.json" } @@ -110,11 +163,16 @@ upload_current() { --only-show-errors done - aws_r2 s3 cp \ - "$REPOSITORY_DIR/current/x86_64-repodata" \ - "s3://${R2_BUCKET}/${R2_PREFIX}/current/x86_64-repodata" \ + # Orphaned .xbps from an aborted run are re-indexed by the next run. + assert_repodata_unchanged "before uploading repodata" + + # Keep the ETag of our own PUT for the final check. + aws_r2 s3api put-object \ + --bucket "$R2_BUCKET" \ + --key "$REPODATA_KEY" \ + --body "$REPOSITORY_DIR/current/x86_64-repodata" \ --cache-control 'no-cache' \ - --only-show-errors + --query ETag --output text > "$ETAG_STATE" # The manifest is the publication marker and is always uploaded last. aws_r2 s3 cp \ @@ -123,7 +181,8 @@ upload_current() { --cache-control 'no-cache' \ --only-show-errors - # Once the new index and marker are live, remove objects no longer referenced. + # Remove retired objects only if the live index is still ours. + assert_repodata_unchanged "before removing retired objects" shopt -s nullglob for old in "$REPOSITORY_DIR"/previous-current/*; do filename="$(basename "$old")" @@ -137,8 +196,9 @@ upload_current() { publish() { require_env SOURCE_COMMIT + SOURCE_REPOSITORY="${SOURCE_REPOSITORY:-${GITHUB_REPOSITORY:-unknown}}" PUBLISHED_AT="${PUBLISHED_AT:-$(date -u +'%Y-%m-%dT%H:%M:%SZ')}" - export PUBLISHED_AT + export PUBLISHED_AT SOURCE_REPOSITORY shopt -s nullglob local packages=("$REPOSITORY_DIR"/current/*.xbps) @@ -157,6 +217,7 @@ publish() { } done + assert_repodata_unchanged "before publishing" verify_immutable_packages build_manifest archive_retired diff --git a/.github/workflows/point-release.yml b/.github/workflows/point-release.yml index 3f252af8b..25d573a0a 100644 --- a/.github/workflows/point-release.yml +++ b/.github/workflows/point-release.yml @@ -103,5 +103,5 @@ jobs: { echo "## ${{ steps.derive.outputs.tag }} tagged on ${{ steps.derive.outputs.branch }} — Release workflow dispatched" echo "" - echo "Distro builds are manual: run the per-distro workflows (COPR/OBS/PPA/XBPS) once the release is published." + echo "Distro builds (COPR/OBS/PPA/XBPS) are dispatched by the Release workflow once the release is published." } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 10b2a594e..7882b7f51 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -559,3 +559,38 @@ jobs: set -euo pipefail gh workflow run pin-release.yml --repo "$REPO" -f version="$TAG" echo "dispatched pin-release ${TAG} to ${REPO}" >> "$GITHUB_STEP_SUMMARY" + + # Build the stable dms package on every distro channel. Runs after the + # release job so every asset the packaging scripts download already exists. + distros: + name: Dispatch ${{ matrix.workflow }} + runs-on: ubuntu-latest + needs: [release] + if: ${{ !contains(inputs.tag, '-') }} + strategy: + fail-fast: false + matrix: + include: + - workflow: run-copr.yml + args: -f package=dms -f version=__VERSION__ -f release=1 + - workflow: run-obs.yml + args: -f package=dms + - workflow: run-ppa.yml + args: -f package=dms + - workflow: run-xbps.yml + args: -f version=__TAG__ -f build_dms=true -f build_git=false + steps: + - name: Dispatch packaging workflow + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WORKFLOW: ${{ matrix.workflow }} + ARGS: ${{ matrix.args }} + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + VERSION="${TAG#v}" + ARGS="${ARGS//__VERSION__/$VERSION}" + ARGS="${ARGS//__TAG__/$TAG}" + # shellcheck disable=SC2086 + gh workflow run "$WORKFLOW" --repo "${{ github.repository }}" $ARGS + echo "dispatched ${WORKFLOW} for ${TAG} (${ARGS})" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/run-xbps.yml b/.github/workflows/run-xbps.yml index 401ba3507..296ba0864 100644 --- a/.github/workflows/run-xbps.yml +++ b/.github/workflows/run-xbps.yml @@ -3,9 +3,7 @@ name: Void Linux XBPS Repository on: schedule: - cron: "0 2,5,14,17,20,23 * * *" # 9am, 12pm, 3pm, 6pm, 9pm, 12am EST (UTC times shown) - # release trigger disabled; dispatch manually after a release - # release: - # types: [published] + # Stable builds are dispatched by release.yml after the release assets exist. workflow_dispatch: inputs: version: @@ -49,7 +47,9 @@ jobs: run: | SHOULD_BUILD=true if [ "${{ github.event_name }}" = "schedule" ]; then - PUBLISHED_COMMIT="$(curl -fsSL --retry 3 "$MANIFEST_URL" | jq -r '.source_commit // empty' 2>/dev/null || true)" + # dms/current is shared with dux-packaging; read this repo's own entry + PUBLISHED_COMMIT="$(curl -fsSL --retry 3 "$MANIFEST_URL" \ + | jq -r --arg repo "${{ github.repository }}" '.sources[$repo] // .source_commit // empty' 2>/dev/null || true)" if [ "$PUBLISHED_COMMIT" = "${{ github.sha }}" ]; then SHOULD_BUILD=false echo "${{ github.sha }} is already published; skipping the scheduled build."