diff --git a/services/nomad/build/build-rsyncd.nomad b/services/nomad/build/build-rsyncd.nomad index 0235dd19..0a8f2621 100644 --- a/services/nomad/build/build-rsyncd.nomad +++ b/services/nomad/build/build-rsyncd.nomad @@ -65,8 +65,8 @@ job "build-rsyncd" { } template { - data = file("rsync-post-xfer") - destination = "local/rsync-post-xfer" + data = file("rsync-update-repo") + destination = "local/rsync-update-repo" perms = "0755" } @@ -114,8 +114,13 @@ EOF [incoming-${template.value}] path = /incoming/${template.value} auth users = buildsync:rw -filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .* -post-xfer exec = /local/rsync-post-xfer +filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .* - *.log + +[updaterepo-${template.value}] +path = /incoming/${template.value} +auth users = buildsync:r +filter = + update-repo.log - * - .* +early exec = /local/rsync-update-repo EOF destination = "local/rsyncd.conf.d/${template.value}.conf.inc" } diff --git a/services/nomad/build/buildbot.cfg b/services/nomad/build/buildbot.cfg index cc3391fe..cb12745f 100644 --- a/services/nomad/build/buildbot.cfg +++ b/services/nomad/build/buildbot.cfg @@ -10,7 +10,6 @@ from pathlib import Path from twisted.internet import defer from twisted.python import log -from buildbot.process.results import SUCCESS from buildbot.plugins import util, secrets, reporters, worker, schedulers from buildbot.plugins import steps @@ -222,20 +221,45 @@ def make_rsync_cmd(props): return ['bash', '-e', '-c', util.Interpolate(""" echo "=> Syncing packages to the shadow repository..." -rsync -vurk --delete-after \ +rsync -vurk --delete-after --remove-source-files \ --filter='+ */' --filter='+ *.%(prop:target)s.xbps' \ --filter='- .*' --filter='- *' \ --password-file=/secrets/rsync/password /hostdir/binpkgs/ \ {{ range nomadService 1 $allocID "build-rsyncd" -}} rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s {{ end -}} -case "$?" in +RSYNC_EXIT="$?" +echo "=> Cleaning repodatas..." +find /hostdir/binpkgs -name '%(prop:target)s-repodata' -print -delete +case "$RSYNC_EXIT" in 0|23|24) exit 0 ;; *) exit 1 ;; esac """)] +@util.renderer +def make_update_repo_cmd(props): + return ['bash', '-e', '-c', + util.Interpolate(f""" +echo "=> Moving packages to the shadow repository..." +rsync -vurk --delete-after --password-file=/secrets/rsync/password \ +{{ range nomadService 1 $allocID "build-rsyncd" -}} +rsync://buildsync@{{ .Address }}:{{ .Port }}/updaterepo-%(prop:target)s/update-repo.log ./update-repo.log +{{ end -}} +case "$?" in + 0|23|24) + cat ./update-repo.log + exit 0 + ;; + *) + cat ./update-repo.log + exit 1 + ;; +esac +""")] + + @util.renderer def make_clean_cmd(props): return [ @@ -259,6 +283,11 @@ def build_packages(props): logname=f'sync:{p}', haltOnFailure=True, )) + cmds.append(util.ShellArg( + command=make_update_repo_cmd, + logname=f'update-repo:{p}', + haltOnFailure=True, + )) if cmds: cmds.append(util.ShellArg( command=['make', 'clean'], diff --git a/services/nomad/build/rsync-post-xfer b/services/nomad/build/rsync-post-xfer deleted file mode 100644 index 7d70ceef..00000000 --- a/services/nomad/build/rsync-post-xfer +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh - -set -e - -TGT_BASE="/mirror/current" - -process_repo() { - repo="$1" - if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "$TGT_BASE/$TGT/$repo" ]; then - return - fi - # add to repodata - find "$RSYNC_MODULE_PATH"/"$repo" -maxdepth 1 -type f -printf "$TGT_BASE/$TGT/$repo/%f\0" | \ - xargs -r0 xbps-rindex -a - # clean old packages from repodata - xbps-rindex -c "$TGT_BASE/$TGT/$repo" - # remove old packages - xbps-rindex -r "$TGT_BASE/$TGT/$repo" - # Remove signatures that don't have a corresponding package - find "$TGT_BASE/$TGT/$repo" -maxdepth 1 \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) \ - -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} + -} - -export XBPS_TARGET_ARCH="${RSYNC_MODULE_NAME#*-}" - -case "$XBPS_TARGET_ARCH" in - aarch64*) TGT="aarch64" ;; - *-musl) TGT="musl" ;; - *) TGT="" ;; -esac - -# copy files to repo -rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/" - -for repo in / /bootstrap /debug /nonfree; do - process_repo "$repo" -done - -if [ "$XBPS_TARGET_ARCH" = 'i686' ]; then - for repo in /multilib /multilib/bootstrap /multilib/nonfree; do - process_repo "$repo" - done -fi - -# clean up incoming -rm -r "${RSYNC_MODULE_PATH:?}"/* - -exit 0 diff --git a/services/nomad/build/rsync-update-repo b/services/nomad/build/rsync-update-repo new file mode 100644 index 00000000..9d70f4ef --- /dev/null +++ b/services/nomad/build/rsync-update-repo @@ -0,0 +1,58 @@ +#!/bin/sh + +msg() { + printf '\033[32m=> %s\033[0m\n' "$*" +} + +process_repo() { + repo="$1" + msg "Processing repository: $repo" + if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "$TGT_BASE/$TGT/$repo" ]; then + msg "Nothing to do" + return + fi + # add to repodata + find "$RSYNC_MODULE_PATH"/"$repo" -mindepth 1 -maxdepth 1 -name '*.xbps' -printf "$TGT_BASE/$TGT/$repo/%f\0" | \ + xargs -r0 env XBPS_TARGET_ARCH="$ARCH" xbps-rindex -a + # clean old packages from repodata + XBPS_TARGET_ARCH="$ARCH" xbps-rindex -c "$TGT_BASE/$TGT/$repo" + # remove old packages + XBPS_TARGET_ARCH="$ARCH" xbps-rindex -r "$TGT_BASE/$TGT/$repo" +} + +run() { + set -e + TGT_BASE="/mirror/current" + ARCH="${RSYNC_MODULE_NAME#*-}" + + case "$ARCH" in + aarch64*) TGT="aarch64" ;; + *-musl) TGT="musl" ;; + *) TGT="" ;; + esac + + # copy files to repo + msg "Copying new packages to the shadow mirror" + rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/" + + for repo in / /bootstrap /debug /nonfree; do + process_repo "$repo" + done + + case "$ARCH" in + x86_64|i686) + ARCH=x86_64 + for repo in /multilib /multilib/bootstrap /multilib/nonfree; do + process_repo "$repo" + done + ;; + esac + + # clean up incoming + msg "Cleaning up" + rm -rvf "${RSYNC_MODULE_PATH:?}"/*.xbps + set +e +} + +# if this exits non-zero, rsync will fail and the log will be lost +run >"$RSYNC_MODULE_PATH"/update-repo.log 2>&1 || true