buildbot sync fixes

services/nomad/build/build-rsyncd.nomad

@@ -65,8 +65,8 @@ job "build-rsyncd" {
       }
 
       template {
-        data        = file("rsync-post-xfer")
-        destination = "local/rsync-post-xfer"
+        data        = file("rsync-update-repo")
+        destination = "local/rsync-update-repo"
         perms       = "0755"
       }
 
@@ -114,8 +114,13 @@ EOF
 [incoming-${template.value}]
 path = /incoming/${template.value}
 auth users = buildsync:rw
-filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .*
-post-xfer exec = /local/rsync-post-xfer
+filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .* - *.log
+
+[updaterepo-${template.value}]
+path = /incoming/${template.value}
+auth users = buildsync:r
+filter = + update-repo.log - * - .*
+early exec = /local/rsync-update-repo
 EOF
           destination = "local/rsyncd.conf.d/${template.value}.conf.inc"
         }

services/nomad/build/buildbot.cfg

@@ -10,7 +10,6 @@ from pathlib import Path
 from twisted.internet import defer
 from twisted.python import log
 
-from buildbot.process.results import SUCCESS
 from buildbot.plugins import util, secrets, reporters, worker, schedulers
 from buildbot.plugins import steps
 
@@ -219,10 +218,17 @@ def make_xbps_bulk_cmd(props):
 
 @util.renderer
 def make_rsync_cmd(props):
+    target = props.getProperty('target')
+    if target.startswith("aarch64"):
+        final_dest = "/mirror/current/aarch64"
+    elif target.endswith("musl"):
+        final_dest = "/mirror/current/musl"
+    else:
+        final_dest = "/mirror/current"
     return ['bash', '-e', '-c',
-            util.Interpolate("""
+            util.Interpolate(f"""
 echo "=> Syncing packages to the shadow repository..."
-rsync -vurk --delete-after \
+rsync -vurk --delete-after --compare-dest={final_dest} \
 --filter='+ */' --filter='+ *.%(prop:target)s.xbps' \
 --filter='- .*' --filter='- *' \
 --password-file=/secrets/rsync/password /hostdir/binpkgs/ \
@@ -236,6 +242,28 @@ esac
 """)]
 
 
+@util.renderer
+def make_update_repo_cmd(props):
+    return ['bash', '-e', '-c',
+            util.Interpolate(f"""
+echo "=> Moving packages to the shadow repository..."
+rsync -vurk --delete-after --password-file=/secrets/rsync/password \
+{{ range nomadService 1 $allocID "build-rsyncd" -}}
+rsync://buildsync@{{ .Address }}:{{ .Port }}/updaterepo-%(prop:target)s/update-repo.log ./update-repo.log
+{{ end -}}
+case "$?" in
+    0|23|24)
+        cat ./update-repo.log
+        exit 0
+        ;;
+    *)
+        cat ./update-repo.log
+        exit 1
+        ;;
+esac
+""")]
+
+
 @util.renderer
 def make_clean_cmd(props):
     return [
@@ -259,6 +287,11 @@ def build_packages(props):
             logname=f'sync:{p}',
             haltOnFailure=True,
         ))
+        cmds.append(util.ShellArg(
+            command=make_update_repo_cmd,
+            logname=f'update-repo:{p}',
+            haltOnFailure=True,
+        ))
     if cmds:
         cmds.append(util.ShellArg(
             command=['make', 'clean'],
@@ -361,6 +394,30 @@ factory.addStep(steps.ShellSequence(
     timeout=14400,
 ))
 
+factory.addStep(steps.ShellCommand(
+    command=make_rsync_cmd,
+    name='sync_packages',
+    description='syncing packages from hostdir',
+    descriptionDone='synced packages from hostdir',
+    alwaysRun=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir='.',
+    timeout=14400,
+))
+
+factory.addStep(steps.ShellCommand(
+    command=make_update_repo_cmd,
+    name='sync_packages',
+    description='updating shadow repository with built packages',
+    descriptionDone='updated shadow repository with built packages',
+    alwaysRun=True,
+    logEnviron=False,
+    usePTY=True,
+    workdir=builddir(),
+    timeout=14400,
+))
+
 factory.addStep(steps.ShellCommand(
     command=make_clean_cmd,
     name='clean_packages',

services/nomad/build/rsync-update-repo

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+set -ex
+
+exec >"$RSYNC_MODULE_PATH"/update-repo.log 2>&1
+
+TGT_BASE="/mirror/current"
+
+process_repo() {
+    repo="$1"
+    if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "$TGT_BASE/$TGT/$repo" ]; then
+        return
+    fi
+    # add to repodata
+    find "$RSYNC_MODULE_PATH"/"$repo" -maxdepth 1 -type f -printf "$TGT_BASE/$TGT/$repo/%f\0" | \
+        xargs -r0 env XBPS_TARGET_ARCH="$ARCH" xbps-rindex -a
+    # clean old packages from repodata
+    XBPS_TARGET_ARCH="$ARCH" xbps-rindex -c "$TGT_BASE/$TGT/$repo"
+    # remove old packages
+    XBPS_TARGET_ARCH="$ARCH" xbps-rindex -r "$TGT_BASE/$TGT/$repo"
+}
+
+ARCH="${RSYNC_MODULE_NAME#*-}"
+
+case "$ARCH" in
+    aarch64*) TGT="aarch64" ;;
+    *-musl) TGT="musl" ;;
+    *) TGT="" ;;
+esac
+
+# copy files to repo
+rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/"
+
+for repo in / /bootstrap /debug /nonfree; do
+    process_repo "$repo"
+done
+
+case "$ARCH" in
+    x86_64|i686)
+        ARCH=x86_64
+        for repo in /multilib /multilib/bootstrap /multilib/nonfree; do
+            process_repo "$repo"
+        done
+        ;;
+esac
+
+# clean up incoming
+rm -r "${RSYNC_MODULE_PATH:?}"/*.xbps
+
+exit 0