Datanommer daily exports

Export daily 24h data in addition to the full pg_dump to allow
downstream replicas to stay in sync without requiring full PG
replication.

We call this "incremental" in some places because perhaps later we will
decide on a different level of granularity, e.g. hourly or weekly.

Signed-off-by: Michael Winters <fedora@mwinters.net>

scripts/iso-file-retention

@@ -0,0 +1,69 @@
+#!/bin/bash
+# vim: ts=4:sw=4:expandtab:tw=100
+
+# Deletes files prefixed with ISO dates (YYYY-MM-DD-) older than RETENTION_DAYS days,
+# but always retains the most-recent MIN_KEEP files even if they're "old".
+#
+# Usage: ./retain.sh --dry-run <directory>
+
+set -euo pipefail
+
+DRY_RUN=false
+DIR=""
+
+for arg in "$@"; do
+    case "$arg" in
+        --dry-run) DRY_RUN=true ;;
+        *) DIR="$arg" ;;
+    esac
+done
+
+DIR="${DIR:-.}"
+RETENTION_DAYS="${RETENTION_DAYS:-31}"
+MIN_KEEP="${MIN_KEEP:-31}"
+CUTOFF=$(date -d "${RETENTION_DAYS} days ago" +%Y-%m-%d)
+
+# Collect all dated files (basename matches YYYY-MM-DD-*)
+mapfile -t DATED_FILES < <(
+    find "$DIR" -maxdepth 1 -type f \
+        | grep -E '.*/[0-9]{4}-[0-9]{2}-[0-9]{2}-' \
+        | sort -t'/' -k2 -r  # sort descending by filename (date first)
+)
+
+total=${#DATED_FILES[@]}
+
+$DRY_RUN && echo "*** DRY RUN — no files will be deleted ***"
+echo ""
+
+if [[ $total -eq 0 ]]; then
+    echo "No dated files found in '$DIR'."
+    exit 0
+fi
+
+deleted=0
+skipped_retention=0
+
+for i in "${!DATED_FILES[@]}"; do
+    filepath="${DATED_FILES[$i]}"
+    filename=$(basename "$filepath")
+    file_date="${filename:0:10}"  # extract YYYY-MM-DD
+
+    # Always keep the MIN_KEEP most-recent files (indices 0..MIN_KEEP-1)
+    if [[ $i -lt $MIN_KEEP ]]; then
+        echo "KEEP (newest ${MIN_KEEP}): $filename"
+        continue
+    fi
+
+    # Delete if the file's date is before the cutoff
+    if [[ "$file_date" < "$CUTOFF" ]]; then
+        echo "DELETE (expired): $filename"
+        $DRY_RUN || rm -- "$filepath"
+        deleted="$((deleted + 1))"
+    else
+        echo "KEEP (within retention): $filename"
+        skipped_retention="$((skipped_retention + 1))"
+    fi
+done
+
+echo ""
+echo "Deleted: $deleted files (dry-run=$DRY_RUN).  Kept by min-count: $(( total < MIN_KEEP ? total : MIN_KEEP )).  Kept by date window: $skipped_retention."

scripts/public-db-copy

@@ -10,6 +10,9 @@ bkup_domain=rdu3.fedoraproject.org
 
 scp db-datanommer02.${bkup_domain}:/backups/datanommer2-$(date +%F).dump.xz \
     /srv/web/infra/db-dumps/datanommer2.dump.xz
+# Always yesterday because the export needs to ensure a full day's worth of data
+scp db-datanommer02.${bkup_domain}:/backups/datanommer-incremental/$(date -d 'yesterday' +%F)-datanommer-incremental-csv.tar \
+    /srv/web/infra/db-dumps/datanommer-incremental/
 
 scp db-koji01.${bkup_domain}:/backups/koji-$(date +%F).dump.xz \
     /srv/web/infra/db-dumps/koji.dump.xz