diff --git a/.github/scripts/prune-void-r2.sh b/.github/scripts/prune-void-r2.sh new file mode 100755 index 000000000..1df4bd249 --- /dev/null +++ b/.github/scripts/prune-void-r2.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# Prunes retired XBPS packages from the R2 bucket so it stays inside the +# Cloudflare free tier. Only objects under ARCHIVE_PREFIX are eligible; live +# repositories (/current/) are never touched. +# +# publish-void-r2.sh writes retired packages to archive///. +# Snapshots are dropped oldest first: +# 1. snapshots older than KEEP_DAYS, keeping KEEP_MIN_SNAPSHOTS per repo +# 2. then more until the whole bucket is <= MAX_BUCKET_GB + +set -euo pipefail + +require_env() { + local name + for name in "$@"; do + if [[ -z "${!name:-}" ]]; then + echo "error: $name is required" >&2 + exit 2 + fi + done +} + +require_env R2_BUCKET R2_ENDPOINT + +ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-archive/}" +MAX_BUCKET_GB="${MAX_BUCKET_GB:-8}" +KEEP_DAYS="${KEEP_DAYS:-30}" +KEEP_MIN_SNAPSHOTS="${KEEP_MIN_SNAPSHOTS:-2}" +DRY_RUN="${DRY_RUN:-false}" + +case "$ARCHIVE_PREFIX" in + */) ;; + *) echo "error: ARCHIVE_PREFIX must end with /" >&2; exit 2 ;; +esac + +aws_r2() { + aws --endpoint-url "$R2_ENDPOINT" "$@" +} + +human() { + numfmt --to=iec-i --suffix=B "$1" +} + +objects="$(aws_r2 s3api list-objects-v2 \ + --bucket "$R2_BUCKET" \ + --output json \ + --query 'Contents[].{key:Key,size:Size}')" +[[ -n "$objects" && "$objects" != null ]] || objects='[]' + +total="$(jq 'map(.size) | add // 0' <<<"$objects")" +budget="$(awk -v g="$MAX_BUCKET_GB" 'BEGIN { printf "%d", g * 1073741824 }')" +now="$(date -u +%s)" + +snapshots="$(jq -c --arg p "$ARCHIVE_PREFIX" ' + map(select(.key | startswith($p))) + | map(.rel = (.key[($p | length):] | split("/"))) + | map(select((.rel | length) >= 3)) + | group_by(.rel[0:2]) + | map({ + repo: .[0].rel[0], + stamp: .[0].rel[1], + prefix: ($p + (.[0].rel[0:2] | join("/")) + "/"), + size: (map(.size) | add), + count: length + }) + | sort_by(.stamp) +' <<<"$objects")" + +to_delete="$(jq -c \ + --argjson now "$now" \ + --argjson keepDays "$KEEP_DAYS" \ + --argjson keepMin "$KEEP_MIN_SNAPSHOTS" \ + --argjson total "$total" \ + --argjson budget "$budget" ' + def epoch: + .stamp + | sub("T(?[0-9]{2})-(?[0-9]{2})-(?[0-9]{2})Z$"; "T\(.h):\(.m):\(.s)Z") + | try fromdateiso8601 catch null; + (group_by(.repo) | map({key: .[0].repo, value: length}) | from_entries) as $perRepo + | reduce .[] as $s ({kept: $perRepo, total: $total, delete: []}; + ($s | epoch) as $e + | (.kept[$s.repo] > $keepMin and $e != null and ($now - $e) > $keepDays * 86400) as $expired + | (.total > $budget) as $over + | if $expired or $over then + .delete += [$s + {reason: (if $over then "budget" else "age" end)}] + | .total -= $s.size + | .kept[$s.repo] -= 1 + else . end) + | .delete +' <<<"$snapshots")" + +echo "bucket: $R2_BUCKET ($(human "$total") in $(jq length <<<"$objects") objects)" +echo "budget: $(human "$budget") (MAX_BUCKET_GB=$MAX_BUCKET_GB)" +echo "archive: $(jq length <<<"$snapshots") snapshots, $(human "$(jq 'map(.size) | add // 0' <<<"$snapshots")")" +echo "policy: KEEP_DAYS=$KEEP_DAYS KEEP_MIN_SNAPSHOTS=$KEEP_MIN_SNAPSHOTS DRY_RUN=$DRY_RUN" + +deleted=0 +freed=0 +while IFS=$'\t' read -r prefix size count reason; do + [[ -n "$prefix" ]] || continue + # Belt and braces: never delete outside the archive or anything live. + case "$prefix" in + "$ARCHIVE_PREFIX"*) ;; + *) echo "error: refusing to delete $prefix (outside $ARCHIVE_PREFIX)" >&2; exit 1 ;; + esac + case "$prefix" in + */current/*) echo "error: refusing to delete live prefix $prefix" >&2; exit 1 ;; + esac + + if [[ "$DRY_RUN" == "true" ]]; then + echo "would delete $prefix ($count objects, $(human "$size"), $reason)" + else + echo "deleting $prefix ($count objects, $(human "$size"), $reason)" + aws_r2 s3 rm "s3://${R2_BUCKET}/${prefix}" --recursive --only-show-errors + fi + deleted=$((deleted + 1)) + freed=$((freed + size)) +done < <(jq -r '.[] | [.prefix, .size, .count, .reason] | @tsv' <<<"$to_delete") + +remaining=$((total - freed)) +echo "pruned: $deleted snapshots, $(human "$freed") freed, $(human "$remaining") remaining" + +if (( remaining > budget )); then + echo "::warning::bucket is still $(human "$remaining") after pruning (budget $(human "$budget")); live repositories exceed the budget" +fi + +note="" +if [[ "$DRY_RUN" == "true" ]]; then + note=" (dry run)" +fi + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + echo "### R2 prune: \`$R2_BUCKET\`" + echo + echo "| | |" + echo "|---|---|" + echo "| Before | $(human "$total") |" + echo "| Budget | $(human "$budget") |" + echo "| Snapshots pruned | $deleted$note |" + echo "| Freed | $(human "$freed") |" + echo "| After | $(human "$remaining") |" + if (( deleted > 0 )); then + echo + echo "| Snapshot | Objects | Bytes | Reason |" + echo "|---|---|---|---|" + jq -r '.[] | "| `\(.prefix)` | \(.count) | \(.size) | \(.reason) |"' <<<"$to_delete" + fi + } >> "$GITHUB_STEP_SUMMARY" +fi diff --git a/.github/workflows/prune-r2.yml b/.github/workflows/prune-r2.yml new file mode 100644 index 000000000..21969e8cf --- /dev/null +++ b/.github/workflows/prune-r2.yml @@ -0,0 +1,66 @@ +name: Prune Void R2 archive + +# Keeps the danklinux-void bucket inside the Cloudflare R2 free tier by +# deleting the oldest archive/// snapshots written by +# publish-void-r2.sh. Live repositories (/current/) are never touched. + +on: + schedule: + - cron: "30 6 * * *" + workflow_dispatch: + inputs: + dry_run: + description: 'Report what would be deleted without deleting' + type: boolean + required: false + default: false + max_bucket_gb: + description: 'Total bucket size to prune down to (GB)' + type: string + required: false + default: '8' + keep_days: + description: 'Delete archive snapshots older than this many days' + type: string + required: false + default: '30' + keep_min_snapshots: + description: 'Always keep at least this many snapshots per repo (age rule only)' + type: string + required: false + default: '2' + +permissions: + contents: read + +# Serialize with the XBPS publisher so a snapshot is never pruned mid-publish. +concurrency: + group: void-xbps-${{ github.ref }} + cancel-in-progress: false + +jobs: + prune: + name: Prune archive snapshots + runs-on: ubuntu-latest + if: github.repository == 'AvengeMedia/DankMaterialShell' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_EC2_METADATA_DISABLED: true + R2_BUCKET: danklinux-void + R2_ENDPOINT: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com + ARCHIVE_PREFIX: archive/ + MAX_BUCKET_GB: ${{ github.event.inputs.max_bucket_gb || '8' }} + KEEP_DAYS: ${{ github.event.inputs.keep_days || '30' }} + KEEP_MIN_SNAPSHOTS: ${{ github.event.inputs.keep_min_snapshots || '2' }} + DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Prune archive + run: bash .github/scripts/prune-void-r2.sh