1
0
Fork 0
forked from infra/ansible

database migration: db-datanommer, db-fas, db-koji, db-openqa, db01, db03

This deletes the temporary rhel10 instances, since we have migrated to
them and renamed them back (to avoid acl issues).

The exception is db-datanommer, which we just are keeping the 01
name now instead of 02.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>
This commit is contained in:
Kevin Fenzi 2026-10-01 14:45:24 -07:00
commit 14661d27ee
22 changed files with 23 additions and 363 deletions

View file

@ -6,7 +6,7 @@ copr-fe.aws.fedoraproject.org
copr-keygen.aws.fedoraproject.org
db01.rdu3.fedoraproject.org
db03.rdu3.fedoraproject.org
db-datanommer02.rdu3.fedoraproject.org
db-datanommer01.rdu3.fedoraproject.org
db-fas01.rdu3.fedoraproject.org
batcave01.rdu3.fedoraproject.org
people01.fedoraproject.org

View file

@ -103,7 +103,7 @@ custom_rules: []
datacenter: rdu3
# Datanommer
datanommer_db_hostname: db-datanommer02
datanommer_db_hostname: db-datanommer01
# By default, nodes don't backup any dbs on them unless they declare it.
dbs_to_backup: []
# dnf-automatic-install.service mode default: security-only

View file

@ -1,46 +0,0 @@
---
# This is a generic list
databases:
- datanommer
datacenter: rdu3
# Group vars beg:
dns1: 10.16.163.33
dns2: 10.16.163.34
dns_search1: "rdu3.fedoraproject.org"
dns_search2: "fedoraproject.org"
# end
db_backup_dir: ['/backups']
# This is a more strict list, to be made publicly available
dbs_to_backup:
- datanommer2
effective_cache_size: "12GB"
eth0_ipv4: "{{eth0_ipv4_ip}}"
eth0_ipv4_gw: 10.16.163.254
eth0_ipv4_ip: 10.16.163.111
# kernel SHMMAX value
kernel_shmmax: 68719476736
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL9-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel
# These are normally group variables, but in this case db servers are often different
lvm_size: 2500g
max_cpu: "{{ num_cpus * 2 }}"
max_mem_size: "{{ mem_size * 2 }}"
max_parallel_workers: 6
max_parallel_workers_per_gather: 4
max_worker_processes: 8
mem_size: 65536
nagios_Check_Services:
swap: false
num_cpus: 16
sar_huge: true
sar_output_file: datagrepper.csv
# GDPR SAR variables - datanommer/datagrepper
sar_script: /usr/local/bin/datagrepper_sar.py
sar_script_user: root
shared_buffers: "4GB"
tcp_ports: [5432, 443]
vmhost: vmhost-x86-04.rdu3.fedoraproject.org
volgroup: /dev/vg_guests
zabbix_macros:
'PG.LOCKS.MAX.WARN:"mirrormanager2"': 3000
'PG.LOCKS.MAX.WARN:"datanommer2"': 1000

View file

@ -34,8 +34,8 @@ dbs_to_backup:
effective_cache_size: "6GB"
eth0_ipv4_gw: 10.16.163.254
eth0_ipv4_ip: 10.16.163.42
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL9-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 100000
max_stack_depth: "4MB"

View file

@ -1,52 +0,0 @@
---
#
# Only allow postgresql access from the frontend nodes and hosted.
#
nft_custom_rules:
# Openshift nodes (egress policy will block connection from non-authorized projects)
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.69 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.70 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.71 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.72 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.73 tcp dport 5432 counter accept'
# noc01 needs to connect to check the db
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.10 tcp dport 5432 counter accept'
# Ipsilon VMs
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.105 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.106 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.117 tcp dport 5432 counter accept'
# This is a generic list
databases:
- fas2
- ipsilon
datacenter: rdu3
# Group vars beg:
dns1: 10.16.163.33
dns2: 10.16.163.34
dns_search1: "rdu3.fedoraproject.org"
dns_search2: "fedoraproject.org"
# end
db_backup_dir: ['/backups']
# This is a more strict list, to be made publicly available
dbs_to_backup:
- fas2
- ipsilon
effective_cache_size: "6GB"
eth0_ipv4_gw: 10.16.163.254
eth0_ipv4_ip: 10.16.163.115
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 100000
max_stack_depth: "4MB"
mem_size: 16384
nrpe_procs_crit: 800
#
# Large updates pushes cause lots of db threads doing the tag moves, so up this from default.
#
nrpe_procs_warn: 600
num_cpus: 8
shared_buffers: "2GB"
temp_buffers: "8MB"
vmhost: vmhost-x86-04.rdu3.fedoraproject.org
volgroup: /dev/vg_guests

View file

@ -27,8 +27,8 @@ eth0_ipv4_gw: 10.16.169.254
eth0_ipv4_ip: 10.16.169.103
# kernel SHMMAX value
kernel_shmmax: 68719476736
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL9-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 2500g
max_cpu: 96

View file

@ -1,56 +0,0 @@
---
#
# Only allow postgresql access from the frontend node.
#
nft_custom_rules:
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.169.104 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.169.105 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.169.10 tcp dport 5432 counter accept'
- 'add rule {{nft_table_filter}} INPUT ip saddr 10.16.163.10 tcp dport 5432 counter accept'
# This is a generic list
databases:
- koji
datacenter: rdu3
# Group vars beg:
dns1: 10.16.163.33
dns2: 10.16.163.34
dns_search1: "rdu3.fedoraproject.org"
dns_search2: "fedoraproject.org"
# end
db_backup_dir: ['/backups']
# This is a more strict list, to be made publicly available
dbs_to_backup:
- koji
effective_cache_size: "100GB"
effective_io_concurrency: 20
eth0_ipv4_gw: 10.16.169.254
eth0_ipv4_ip: 10.16.169.25
# kernel SHMMAX value
kernel_shmmax: 68719476736
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 3000g
max_cpu: 96
max_mem_size: 327680
max_parallel_maintenance_workers: 4
max_parallel_workers: 40
max_parallel_workers_per_gather: 10
max_stack_depth: "6MB"
max_worker_processes: 60
mem_size: 262140
nagios_Check_Services:
swap: false
nrpe_procs_crit: 700
#
# Large updates pushes cause lots of db threads doing the tag moves, so up this from default.
#
nrpe_procs_warn: 600
num_cpus: 64
shared_buffers: "64GB"
swap: false
temp_buffers: "192MB"
vmhost: bvmhost-x86-01.rdu3.fedoraproject.org
volgroup: /dev/vg_guests
zabbix_macros:
'PG.LOCKS.MAX.WARN:"koji"': 500

View file

@ -18,8 +18,8 @@ eth0_ipv4_ip: 10.16.174.51
host_backup_targets: ['/backups']
# kernel SHMMAX value
kernel_shmmax: 68719476736
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL9-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 300000
mem_size: 16384

View file

@ -1,34 +0,0 @@
---
# This is a generic list
databases:
- postgres
- openqa
datacenter: rdu3
db_backup_dir: ['/backups']
# This is a more strict list, to be made publicly available
dbs_to_backup:
- postgres
# these names are also stored as host vars 'openqa_dbname',
# make sure to keep in sync
- openqa
- openqa-stg
effective_cache_size: "6GB"
eth0_ipv4_gw: 10.16.174.254
eth0_ipv4_ip: 10.16.174.56
host_backup_targets: ['/backups']
# kernel SHMMAX value
kernel_shmmax: 68719476736
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 300000
mem_size: 16384
num_cpus: 10
shared_buffers: "2GB"
sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
tcp_ports: [5432, 443, 3306]
vmhost: qvmhost-x86-01.rdu3.fedoraproject.org
volgroup: /dev/vg_guests
zabbix_macros:
'PG.LOCKS.MAX.WARN:"openqa-stg"': 250
'PG.LOCKS.MAX.WARN:"openqa"': 250

View file

@ -59,8 +59,8 @@ dbs_to_backup:
effective_cache_size: "50GB"
eth0_ipv4_gw: 10.16.163.254
eth0_ipv4_ip: 10.16.163.41
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL9-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 1500000
max_cpu: "{{ num_cpus }}"

View file

@ -1,90 +0,0 @@
---
#
# We should narrow this down at some point
#
nft_custom_rules: ['add rule {{nft_table_filter}} INPUT tcp dport 5432 counter accept']
# This is a generic list
databases:
- anitya
- blockerbugs
- bodhi2
- elections
- fedocal
- hyperkitty
- kerneltest
- koschei
- mailman
- mirrormanager2
- notifications
- pagure
- resultsdb
- tahrir
- testdays
- testdays_resultsdb
- transtats
- waiverdb
- webhook2fedmsg
- zezere
datacenter: rdu3
# Group vars beg:
dns1: 10.16.163.33
dns2: 10.16.163.34
dns_search1: "rdu3.fedoraproject.org"
dns_search2: "fedoraproject.org"
# end
db_backup_dir: ['/backups']
# This is a more strict list of databases to backup every day
dbs_to_backup:
- anitya
- blockerbugs
- bodhi2
- elections
- fedocal
- hyperkitty
- kerneltest
- koschei
- mailman
- mirrormanager2
- notifications
- pagure
- resultsdb
- tahrir
- testdays
- testdays_resultsdb
- transtats
- waiverdb
- webhook2fedmsg
- zezere
# Should be 0.80 of memory
effective_cache_size: "50GB"
eth0_ipv4_gw: 10.16.163.254
eth0_ipv4_ip: 10.16.163.114
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 1500000
max_cpu: "{{ num_cpus }}"
max_mem_size: "{{ mem_size * 2 }}"
max_stack_depth: "6MB"
max_worker_processes: 12
mem_size: 65536
nagios_Check_Services:
swap: false
nrpe_procs_crit: 900
#
# db01 handles lots of apps, could have many procs if they are busy.
#
nrpe_procs_warn: 800
num_cpus: 48
# Should be 0.25 of memory
shared_buffers: "24GB"
vmhost: vmhost-x86-04.rdu3.fedoraproject.org
volgroup: /dev/vg_guests
zabbix_macros:
'PG.LOCKS.MAX.WARN:"anitya"': 500
'PG.LOCKS.MAX.WARN:"bodhi2"': 500
'PG.LOCKS.MAX.WARN:"datanommer2"': 1000
'PG.LOCKS.MAX.WARN:"koschei"': 300
'PG.LOCKS.MAX.WARN:"hyperkitty"': 500
'PG.LOCKS.MAX.WARN:"mailman"': 300
'PG.LOCKS.MAX.WARN:"resultsdb"': 200

View file

@ -19,8 +19,8 @@ eth0_ipv4_ip: 10.16.163.44
extra_enablerepos: ''
# kernel SHMMAX value
kernel_shmmax: 68719476736
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL9-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 500000
mariadb_root_password: "{{ db03_mysql_root_password }}"

View file

@ -1,32 +0,0 @@
---
# This is a generic list
databases:
- mysql
- fpo-mediawiki
datacenter: rdu3
# Group vars beg:
dns1: 10.16.163.33
dns2: 10.16.163.34
dns_search1: "rdu3.fedoraproject.org"
dns_search2: "fedoraproject.org"
# end
db_backup_dir: ['/backups']
# This is a more strict list of db to backup to /backups
dbs_to_backup:
- fpo-mediawiki
eth0_ipv4_gw: 10.16.163.254
eth0_ipv4_ip: 10.16.163.116
extra_enablerepos: ''
# kernel SHMMAX value
kernel_shmmax: 68719476736
ks_repo: https://infrastructure.fedoraproject.org/repo/rhel/RHEL10-x86_64/
ks_url: https://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-10
# These are normally group variables, but in this case db servers are often different
lvm_size: 600000
mariadb_root_password: "{{ db03_mysql_root_password }}"
mem_size: 16384
num_cpus: 8
shared_buffers: "4GB"
tcp_ports: [5432, 443, 3306]
vmhost: vmhost-x86-04.rdu3.fedoraproject.org
volgroup: /dev/vg_guests

View file

@ -206,35 +206,21 @@ autosign01.stg.rdu3.fedoraproject.org
[dbserver]
db-riscv-koji01.rdu3.fedoraproject.org
db-riscv-koji02.rdu3.fedoraproject.org
db-datanommer01.rdu3.fedoraproject.org
db-datanommer02.rdu3.fedoraproject.org
db-fas01.rdu3.fedoraproject.org
db-fas02.rdu3.fedoraproject.org
db-koji01.rdu3.fedoraproject.org
db-koji02.rdu3.fedoraproject.org
db-openqa01.rdu3.fedoraproject.org
db-openqa02.rdu3.fedoraproject.org
db01.rdu3.fedoraproject.org
db02.rdu3.fedoraproject.org
db03.rdu3.fedoraproject.org
db04.rdu3.fedoraproject.org
[dbserver_rdu3]
db-datanommer01.rdu3.fedoraproject.org
db-datanommer02.rdu3.fedoraproject.org
db-fas01.rdu3.fedoraproject.org
db-fas02.rdu3.fedoraproject.org
db-koji01.rdu3.fedoraproject.org
db-koji02.rdu3.fedoraproject.org
db01.rdu3.fedoraproject.org
db02.rdu3.fedoraproject.org
db03.rdu3.fedoraproject.org
db04.rdu3.fedoraproject.org
db-riscv-koji01.rdu3.fedoraproject.org
db-riscv-koji02.rdu3.fedoraproject.org
db-openqa01.rdu3.fedoraproject.org
db-openqa02.rdu3.fedoraproject.org
[dbserver_stg]
db.stg.aws.fedoraproject.org
@ -253,14 +239,12 @@ db03.stg.rdu3.fedoraproject.org
[apps_dbserver]
db01.rdu3.fedoraproject.org
db02.rdu3.fedoraproject.org
[apps_dbserver_stg]
db01.stg.rdu3.fedoraproject.org
[datanommer_dbserver]
db-datanommer01.rdu3.fedoraproject.org
db-datanommer02.rdu3.fedoraproject.org
[datanommer_dbserver_stg]
db-datanommer01.stg.rdu3.fedoraproject.org

View file

@ -7,14 +7,12 @@
vars:
# See: https://yaml-multiline.info/
myhosts: "db03.rdu3.fedoraproject.org:
db04.rdu3.fedoraproject.org:
db03.stg.rdu3.fedoraproject.org"
# Once the instance exists, configure it.
- name: Configure mariadb server system
hosts: "db03.rdu3.fedoraproject.org:
db04.rdu3.fedoraproject.org:
db03.stg.rdu3.fedoraproject.org"
user: root
gather_facts: true

View file

@ -7,41 +7,29 @@
vars:
# See: https://yaml-multiline.info/
myhosts: "db-openqa01.rdu3.fedoraproject.org:
db-openqa02.rdu3.fedoraproject.org:
db-datanommer01.rdu3.fedoraproject.org:
db-datanommer02.rdu3.fedoraproject.org:
db-fas01.rdu3.fedoraproject.org:
db-fas02.rdu3.fedoraproject.org:
db-koji01.rdu3.fedoraproject.org:
db-koji02.rdu3.fedoraproject.org:
db01.rdu3.fedoraproject.org:
db02.rdu3.fedoraproject.org:
db-datanommer01.stg.rdu3.fedoraproject.org:
db-fas01.stg.rdu3.fedoraproject.org:
db-koji01.stg.rdu3.fedoraproject.org:
db01.stg.rdu3.fedoraproject.org:
db-riscv-koji01.rdu3.fedoraproject.org:
db-riscv-koji02.rdu3.fedoraproject.org"
db-riscv-koji01.rdu3.fedoraproject.org"
# Once the instance exists, configure it.
- name: Configure postgresql server system
hosts: "db-openqa01.rdu3.fedoraproject.org:
db-openqa02.rdu3.fedoraproject.org:
db-datanommer01.rdu3.fedoraproject.org:
db-datanommer02.rdu3.fedoraproject.org:
db-fas01.rdu3.fedoraproject.org:
db-fas02.rdu3.fedoraproject.org:
db-koji01.rdu3.fedoraproject.org:
db-koji02.rdu3.fedoraproject.org:
db01.rdu3.fedoraproject.org:
db02.rdu3.fedoraproject.org:
db-datanommer01.stg.rdu3.fedoraproject.org:
db-fas01.stg.rdu3.fedoraproject.org:
db-koji01.stg.rdu3.fedoraproject.org:
db01.stg.rdu3.fedoraproject.org:
db-riscv-koji01.rdu3.fedoraproject.org:
db-riscv-koji02.rdu3.fedoraproject.org"
db-riscv-koji01.rdu3.fedoraproject.org"
user: root
gather_facts: true

View file

@ -652,7 +652,7 @@
- private
#
# install psql to allow for some ro queries against db-datanommer02
# install psql to allow for some ro queries against db-datanommer01
#
- name: Enable the postgresql 12 module for psql on batcave

View file

@ -5,7 +5,7 @@
name:
- moreutils # provides `chronic` for use with cron emails
- tar
when: inventory_hostname.startswith('db-datanommer02.rdu3')
when: inventory_hostname.startswith('db-datanommer01.rdu3')
tags:
- datanommer
- postgresql
@ -13,7 +13,7 @@
- name: Ensure datanommer incremental backups have a destination directory
ansible.builtin.file: dest=/backups/datanommer-incremental state=directory owner=postgres
when: inventory_hostname.startswith('db-datanommer02.rdu3')
when: inventory_hostname.startswith('db-datanommer01.rdu3')
tags:
- datanommer
- postgresql
@ -21,7 +21,7 @@
- name: Copy datanommer-export-day backup script
ansible.builtin.copy: src=datanommer-export-day dest=/usr/local/bin/datanommer-export-day mode=0755
when: inventory_hostname.startswith('db-datanommer02.rdu3')
when: inventory_hostname.startswith('db-datanommer01.rdu3')
tags:
- datanommer
- postgresql

View file

@ -173,8 +173,8 @@
- postgresql
- name: Copy over backup scriplet
ansible.builtin.copy: src=backup-database.db-datanommer02 dest=/usr/local/bin/backup-database mode=0755
when: inventory_hostname.startswith('db-datanommer02.rdu3')
ansible.builtin.copy: src=backup-database.db-datanommer01 dest=/usr/local/bin/backup-database mode=0755
when: inventory_hostname.startswith('db-datanommer01.rdu3')
tags:
- postgresql

View file

@ -1,4 +1,4 @@
{% if inventory_hostname == 'db-datanommer02.rdu3.fedoraproject.org' %}
{% if inventory_hostname == 'db-datanommer01.rdu3.fedoraproject.org' %}
0 0 * * * postgres /usr/local/bin/lock-wrapper backup-database "/usr/local/bin/backup-database {{ item }} |& grep -Ev 'warning:|hypertable|chunk|restore|data-only|continuous_agg'"
0 3 * * * postgres chronic /usr/local/bin/datanommer-export-day
{% else %}

View file

@ -12,10 +12,10 @@ set -e
# This changes really often ;)
bkup_domain=rdu3.fedoraproject.org
scp db-datanommer02.${bkup_domain}:/backups/datanommer2-$(date +%F).dump.xz \
scp db-datanommer01.${bkup_domain}:/backups/datanommer2-$(date +%F).dump.xz \
/srv/web/infra/db-dumps/datanommer2.dump.xz
# Always yesterday because the export needs to ensure a full day's worth of data
scp db-datanommer02.${bkup_domain}:/backups/datanommer-incremental/$(date -d 'yesterday' +%F)-datanommer-incremental-csv.tar \
scp db-datanommer01.${bkup_domain}:/backups/datanommer-incremental/$(date -d 'yesterday' +%F)-datanommer-incremental-csv.tar \
/srv/web/infra/db-dumps/datanommer-incremental/
scp db-koji01.${bkup_domain}:/backups/koji-$(date +%F).dump.xz \