forked from infra/ansible
Datanommer daily exports
Export daily 24h data in addition to the full pg_dump to allow downstream replicas to stay in sync without requiring full PG replication. We call this "incremental" in some places because perhaps later we will decide on a different level of granularity, e.g. hourly or weekly. Signed-off-by: Michael Winters <fedora@mwinters.net>
This commit is contained in:
parent
b69ce1c8d8
commit
7f493a042a
7 changed files with 253 additions and 0 deletions
|
|
@ -2,4 +2,9 @@
|
|||
mailto='admin@fedoraproject.org'
|
||||
source /root/sshagent >>/dev/null
|
||||
export ANSIBLE_HOST_KEY_CHECKING=False
|
||||
|
||||
# Copy all public database backups from the database servers to our public directory
|
||||
/srv/web/infra/ansible/scripts/public-db-copy >& /dev/null
|
||||
|
||||
# Delete old datanommer-incremental backups
|
||||
/srv/web/infra/ansible/scripts/iso-file-retention /srv/web/infra/db-dumps/datanommer-incremental >& /dev/null
|
||||
|
|
@ -475,6 +475,16 @@
|
|||
- httpd
|
||||
- httpd/website
|
||||
|
||||
#
|
||||
# Setup public-db-copy target directory for datanommer-incremental.
|
||||
#
|
||||
- name: Create db-dumps/datanommer-incremental directory
|
||||
ansible.builtin.file: dest=/srv/web/infra/db-dumps/datanommer-incremental mode=2755 state=directory owner=root group=sysadmin-main setype=httpd_sys_content_t
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# set selinux context for public git repos
|
||||
#
|
||||
|
|
|
|||
142
roles/postgresql_server/files/datanommer-export-day
Normal file
142
roles/postgresql_server/files/datanommer-export-day
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
#!/bin/bash
|
||||
# vim: ts=4:sw=4:expandtab
|
||||
|
||||
set -eu
|
||||
|
||||
# Exports one day of messages from datanommer as a single .tar file, using
|
||||
# psql and xz.
|
||||
#
|
||||
# Usage:
|
||||
# $0
|
||||
# - defaults to "yesterday"
|
||||
# $0 2025-08-30
|
||||
|
||||
|
||||
# --- Config ---
|
||||
|
||||
export PGDATABASE='datanommer2'
|
||||
export DESTDIR='/backups/datanommer-incremental'
|
||||
# tmp directory
|
||||
export WORKDIR='/tmp/datanommer-incremental'
|
||||
|
||||
# --- Init ---
|
||||
|
||||
if [[ ! -w "$WORKDIR" ]]; then
|
||||
mkdir "$WORKDIR"
|
||||
if [[ ! -w "$WORKDIR" ]]; then
|
||||
echo "error: '$WORKDIR' is not writable"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ ! -w "$DESTDIR" ]]; then
|
||||
echo "error: '$DESTDIR' is not writable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
QUERY_DATE=""
|
||||
|
||||
set_default_date() {
|
||||
QUERY_DATE=$(date -d "yesterday" +%Y-%m-%d)
|
||||
}
|
||||
|
||||
usage () {
|
||||
echo "Usage:"
|
||||
echo
|
||||
echo "$0"
|
||||
echo "$0 2025-08-01"
|
||||
exit 1
|
||||
}
|
||||
case $# in
|
||||
0)
|
||||
set_default_date
|
||||
;;
|
||||
1)
|
||||
QUERY_DATE="$1"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
|
||||
# Calculate the end range of our query, which will simultaneously
|
||||
# validate the input.
|
||||
QUERY_DATE_END=$(date -d "${QUERY_DATE} + 1 day" +%Y-%m-%d)
|
||||
|
||||
# ON_ERROR_STOP turns SQL errors into bash errors
|
||||
PSQL="psql --set=ON_ERROR_STOP=on"
|
||||
|
||||
# --- Export Tables ---
|
||||
|
||||
echo "Exporting '$QUERY_DATE' to '$DESTDIR'..."
|
||||
echo "Query dates: \">= '$QUERY_DATE 00:00:00' AND < '$QUERY_DATE_END 00:00:00'\""
|
||||
|
||||
export_table() {
|
||||
local SELECT="$1"
|
||||
# globals used:
|
||||
# TABLE
|
||||
# WORKDIR
|
||||
# QUERY_DATE
|
||||
|
||||
echo "$(date -Iseconds): $TABLE..."
|
||||
# compression level 6 (out of 9)
|
||||
# 4 threads
|
||||
$PSQL --command="\COPY ( $SELECT ) TO STDOUT WITH (FORMAT csv, NULL '\N', HEADER true)" \
|
||||
| xz -6 -T4 \
|
||||
> "${WORKDIR}/${QUERY_DATE}-${TABLE}.csv.xz"
|
||||
}
|
||||
|
||||
TABLE=messages
|
||||
export_table "
|
||||
SELECT * FROM ${TABLE}
|
||||
WHERE timestamp >= '${QUERY_DATE}'
|
||||
AND timestamp < '${QUERY_DATE_END}'
|
||||
"
|
||||
|
||||
TABLE=users_messages
|
||||
export_table "
|
||||
SELECT * FROM ${TABLE}
|
||||
WHERE msg_timestamp >= '${QUERY_DATE}'
|
||||
AND msg_timestamp < '${QUERY_DATE_END}'
|
||||
"
|
||||
|
||||
TABLE=packages_messages
|
||||
export_table "
|
||||
SELECT * FROM ${TABLE}
|
||||
WHERE msg_timestamp >= '${QUERY_DATE}'
|
||||
AND msg_timestamp < '${QUERY_DATE_END}'
|
||||
"
|
||||
|
||||
# Since the 'users' and 'packages' tables are not timestamped, we export
|
||||
# both of them in their entirety. Fortunately, they're both tiny.
|
||||
TABLE=users
|
||||
export_table "SELECT * FROM ${TABLE}"
|
||||
|
||||
TABLE=packages
|
||||
export_table "SELECT * FROM ${TABLE}"
|
||||
|
||||
# This gives us a historical record of when schema changes were applied
|
||||
ALEMBIC_FILE="$(date +%Y-%m-%d)-alembic_version.txt"
|
||||
echo "$(date -Iseconds): alembic_version..."
|
||||
# - "tuples-only" = data only, no ASCII box art
|
||||
# - "no-align" = no whitespace padding
|
||||
$PSQL --tuples-only --no-align --command="SELECT version_num FROM alembic_version;" > "${WORKDIR}/${ALEMBIC_FILE}"
|
||||
|
||||
# --- tar ---
|
||||
|
||||
TARFILE="${QUERY_DATE}-datanommer-incremental-csv.tar"
|
||||
echo "Combining into '${DESTDIR}/${TARFILE}'..."
|
||||
# weird subshell glob is so that this script location is irrelevant to tar invocation
|
||||
tar \
|
||||
-cf "${DESTDIR}/${TARFILE}" \
|
||||
-C "$WORKDIR" \
|
||||
$(cd "$WORKDIR" && echo "${QUERY_DATE}"*.csv.xz) "${ALEMBIC_FILE}"
|
||||
# rm intermediate files
|
||||
rm "${WORKDIR}/${QUERY_DATE}"*.csv.xz
|
||||
rm "${WORKDIR}/${ALEMBIC_FILE}"
|
||||
echo
|
||||
|
||||
echo "$(date -Iseconds): export done!"
|
||||
|
||||
echo "Running retention script for '$DESTDIR'..."
|
||||
/usr/local/bin/iso-file-retention "$DESTDIR"
|
||||
echo
|
||||
|
|
@ -1,4 +1,27 @@
|
|||
---
|
||||
- name: Ensure datanommer incremental backups have a destination directory
|
||||
ansible.builtin.file: dest=/backups/datanommer-incremental state=directory owner=postgres
|
||||
when: inventory_hostname.startswith('db-datanommer02.rdu3')
|
||||
tags:
|
||||
- datanommer
|
||||
- postgresql
|
||||
|
||||
|
||||
- name: Copy datanommer-export-day backup script
|
||||
ansible.builtin.copy: src=datanommer-export-day dest=/usr/local/bin/datanommer-export-day mode=0755
|
||||
when: inventory_hostname.startswith('db-datanommer02.rdu3')
|
||||
tags:
|
||||
- datanommer
|
||||
- postgresql
|
||||
|
||||
|
||||
- name: Copy iso-file-retention script
|
||||
ansible.builtin.copy: src=scripts/iso-file-retention dest=/usr/local/bin/iso-file-retention mode=0755
|
||||
tags:
|
||||
- datanommer
|
||||
- postgresql
|
||||
|
||||
|
||||
- name: Install timescaledb
|
||||
dnf:
|
||||
name:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
{% if inventory_hostname == 'db-datanommer02.rdu3.fedoraproject.org' %}
|
||||
0 0 * * * postgres /usr/local/bin/lock-wrapper backup-database "/usr/local/bin/backup-database {{ item }} |& grep -Ev 'warning:|hypertable|chunk|restore|data-only|continuous_agg'"
|
||||
0 3 * * * postgres /usr/local/bin/datanommer-export-day
|
||||
{% else %}
|
||||
0 0 * * * postgres /usr/local/bin/backup-database {{ item }}
|
||||
{% endif %}
|
||||
|
|
|
|||
69
scripts/iso-file-retention
Executable file
69
scripts/iso-file-retention
Executable file
|
|
@ -0,0 +1,69 @@
|
|||
#!/bin/bash
|
||||
# vim: ts=4:sw=4:expandtab:tw=100
|
||||
|
||||
# Deletes files prefixed with ISO dates (YYYY-MM-DD-) older than RETENTION_DAYS days,
|
||||
# but always retains the most-recent MIN_KEEP files even if they're "old".
|
||||
#
|
||||
# Usage: ./retain.sh --dry-run <directory>
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DRY_RUN=false
|
||||
DIR=""
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
*) DIR="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
DIR="${DIR:-.}"
|
||||
RETENTION_DAYS="${RETENTION_DAYS:-31}"
|
||||
MIN_KEEP="${MIN_KEEP:-31}"
|
||||
CUTOFF=$(date -d "${RETENTION_DAYS} days ago" +%Y-%m-%d)
|
||||
|
||||
# Collect all dated files (basename matches YYYY-MM-DD-*)
|
||||
mapfile -t DATED_FILES < <(
|
||||
find "$DIR" -maxdepth 1 -type f \
|
||||
| grep -E '.*/[0-9]{4}-[0-9]{2}-[0-9]{2}-' \
|
||||
| sort -t'/' -k2 -r # sort descending by filename (date first)
|
||||
)
|
||||
|
||||
total=${#DATED_FILES[@]}
|
||||
|
||||
$DRY_RUN && echo "*** DRY RUN — no files will be deleted ***"
|
||||
echo ""
|
||||
|
||||
if [[ $total -eq 0 ]]; then
|
||||
echo "No dated files found in '$DIR'."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
deleted=0
|
||||
skipped_retention=0
|
||||
|
||||
for i in "${!DATED_FILES[@]}"; do
|
||||
filepath="${DATED_FILES[$i]}"
|
||||
filename=$(basename "$filepath")
|
||||
file_date="${filename:0:10}" # extract YYYY-MM-DD
|
||||
|
||||
# Always keep the MIN_KEEP most-recent files (indices 0..MIN_KEEP-1)
|
||||
if [[ $i -lt $MIN_KEEP ]]; then
|
||||
echo "KEEP (newest ${MIN_KEEP}): $filename"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Delete if the file's date is before the cutoff
|
||||
if [[ "$file_date" < "$CUTOFF" ]]; then
|
||||
echo "DELETE (expired): $filename"
|
||||
$DRY_RUN || rm -- "$filepath"
|
||||
deleted="$((deleted + 1))"
|
||||
else
|
||||
echo "KEEP (within retention): $filename"
|
||||
skipped_retention="$((skipped_retention + 1))"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Deleted: $deleted files (dry-run=$DRY_RUN). Kept by min-count: $(( total < MIN_KEEP ? total : MIN_KEEP )). Kept by date window: $skipped_retention."
|
||||
|
|
@ -10,6 +10,9 @@ bkup_domain=rdu3.fedoraproject.org
|
|||
|
||||
scp db-datanommer02.${bkup_domain}:/backups/datanommer2-$(date +%F).dump.xz \
|
||||
/srv/web/infra/db-dumps/datanommer2.dump.xz
|
||||
# Always yesterday because the export needs to ensure a full day's worth of data
|
||||
scp db-datanommer02.${bkup_domain}:/backups/datanommer-incremental/$(date -d 'yesterday' +%F)-datanommer-incremental-csv.tar \
|
||||
/srv/web/infra/db-dumps/datanommer-incremental/
|
||||
|
||||
scp db-koji01.${bkup_domain}:/backups/koji-$(date +%F).dump.xz \
|
||||
/srv/web/infra/db-dumps/koji.dump.xz
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue