dkirwan/infra-ansible
1
0
Fork 0
forked from infra/ansible
Code Pull requests Activity

copr: better divide the ipv6 range we have

Browse source 
Now, all VMs on stage are 0x900+, and 0x100+ are in prod.

Relates: https://github.com/fedora-copr/copr/issues/3984
This commit is contained in:
Pavel Raiskup 2026-03-09 17:37:12 +01:00
commit 14b89a2a67
15 changed files with 111 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

8
inventory/group_vars/copr_aws
View file

@ -46,6 +46,14 @@ builders:
ppc64le: [15, 5, 15]
p09_hypervisor_04:
ppc64le: [15, 5, 15]
hp_p09_hypervisor_01:
ppc64le: [1, 1, 1]
hp_p09_hypervisor_02:
ppc64le: [1, 1, 1]
hp_p09_hypervisor_03:
ppc64le: [1, 1, 1]
hp_p09_hypervisor_04:
ppc64le: [1, 1, 1]
x86_hypervisor_01:
x86_64: [20, 4, 20]
x86_hypervisor_02:

8
inventory/group_vars/copr_dev_aws
View file

@ -45,6 +45,14 @@ builders:
ppc64le: [1, 1, 1]
p09_hypervisor_04:
ppc64le: [1, 1, 1]
hp_p09_hypervisor_01:
ppc64le: [1, 1, 1]
hp_p09_hypervisor_02:
ppc64le: [1, 1, 1]
hp_p09_hypervisor_03:
ppc64le: [1, 1, 1]
hp_p09_hypervisor_04:
ppc64le: [1, 1, 1]
x86_hypervisor_01:
x86_64: [2, 1, 1]
x86_hypervisor_02:

5
inventory/host_vars/vmhost-p09-copr01.rdu3.fedoraproject.org
View file

@ -19,8 +19,9 @@ mac3: "b8:ce:f6:c6:00:c7"
mac4: "b8:ce:f6:c6:00:d0"
mac5: "b8:ce:f6:c6:00:d1"
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_p09_01
libvirt_pool_order_id: 4
libvirt_pools:
- name: vmhost_p09_01
- name: vmhost_p_p09_01
libvirt_arch: ppc64le
network_connections:
# Bridge profile

5
inventory/host_vars/vmhost-p09-copr02.rdu3.fedoraproject.org
View file

@ -22,8 +22,9 @@ mac7: ac:1f:6b:a5:4e:f5
mac8: ac:1f:6b:a5:4e:f6
mac9: ac:1f:6b:a5:4e:f7
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_p09_02
libvirt_pool_order_id: 0
libvirt_pools:
- name: vmhost_p09_02
- name: vmhost_p_p09_02
libvirt_arch: ppc64le
network_connections:
# Bridge profile

5
inventory/host_vars/vmhost-p09-copr03.rdu3.fedoraproject.org
View file

@ -22,8 +22,9 @@ mac7: ac:1f:6b:8a:9a:31
mac8: ac:1f:6b:8a:9a:32
mac9: ac:1f:6b:8a:9a:33
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_p09_03
libvirt_pool_order_id: 1
libvirt_pools:
- name: vmhost_p09_03
- name: vmhost_p_p09_03
libvirt_arch: ppc64le
network_connections:
# Bridge profile

5
inventory/host_vars/vmhost-p09-copr04.rdu3.fedoraproject.org
View file

@ -22,8 +22,9 @@ mac7: ac:1f:6b:a4:e3:b1
mac8: ac:1f:6b:a4:e3:b2
mac9: ac:1f:6b:a4:e3:b3
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_p09_04
libvirt_pool_order_id: 2
libvirt_pools:
- name: vmhost_p09_04
- name: vmhost_p_p09_04
libvirt_arch: ppc64le
network_connections:
# Bridge profile

4
inventory/host_vars/vmhost-x86-copr01.rdu3.fedoraproject.org
View file

@ -20,8 +20,8 @@ mac6: b4:96:91:63:3b:ea
mac7: b4:96:91:63:3b:eb
mac8: f4:02:70:d3:15:95
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_x86_01
libvirt_pool_order_id: 7
libvirt_pools:
- name: vmhost_x86_01
libvirt_arch: x86_64
network_connections:
# Bridge profile

4
inventory/host_vars/vmhost-x86-copr02.rdu3.fedoraproject.org
View file

@ -19,8 +19,8 @@ mac5: 84:16:0c:bc:24:e0
mac6: b4:96:91:63:3b:9e
mac7: b4:96:91:63:3b:9f
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_x86_02
libvirt_pool_order_id: 8
libvirt_pools:
- name: vmhost_x86_02
libvirt_arch: x86_64
network_connections:
# Bridge profile

4
inventory/host_vars/vmhost-x86-copr03.rdu3.fedoraproject.org
View file

@ -19,8 +19,8 @@ mac5: "b4:96:91:63:3b:51"
mac6: "b4:96:91:63:3b:52"
mac7: "b4:96:91:63:3b:53"
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_x86_03
libvirt_pool_order_id: 9
libvirt_pools:
- name: vmhost_x86_03
libvirt_arch: x86_64
network_connections:
# Bridge profile

4
inventory/host_vars/vmhost-x86-copr04.rdu3.fedoraproject.org
View file

@ -19,8 +19,8 @@ mac5: "b4:96:91:63:3a:a1"
mac6: "b4:96:91:63:3a:a2"
mac7: "b4:96:91:63:3a:a3"
libvirt_host: "[{{ br0_ipv6_ip }}]"
libvirt_pool: vmhost_x86_04
libvirt_pool_order_id: 3
libvirt_pools:
- name: vmhost_x86_04
libvirt_arch: x86_64
network_connections:
# Bridge profile

2
roles/copr/backend/files/provision/libvirt-delete
View file

@ -54,7 +54,7 @@ def _main():
logging.basicConfig(level=logging.INFO)
args = _get_parser().parse_args()
pool_id = os.getenv("RESALLOC_POOL_ID")
connection = get_hv_identification_from_pool_id(pool_id)[1]
connection, *_ = get_hv_identification_from_pool_id(pool_id)
conn = repeat(libvirt.open, (connection,))
try:
domain = repeat(conn.lookupByName, (args.domainname,))

5
roles/copr/backend/files/provision/libvirt-list
View file

@ -28,8 +28,9 @@ def _main():
sys.stderr.write("Specify pool ID by --pool or $RESALLOC_POOL_ID\n")
sys.exit(1)
connection = args.connection if args.connection else \
get_hv_identification_from_pool_id(pool_id)[1]
connection = args.connection
if not connection:
connection, *_ = get_hv_identification_from_pool_id(pool_id)
try:
conn = libvirt.openReadOnly(connection)

15
roles/copr/backend/templates/provision/helpers.py.j2
View file

@ -1,15 +1,24 @@
def get_hv_identification_from_pool_id(pool_id):
""" Get unique ID of the hypervisor """
"""
Get unique ID of the hypervisor
# rdu3 https://github.com/fedora-copr/copr/issues/3786#issuecomment-3412337856
# starting with offset 0x100, that's reserved for hypervisor machines
# we have 16^3 - 16^2 IPs = 3840, dedicate the last 256 to staging.
"""
{% for host in groups["copr_hypervisor"] %}
{% set hostinfo = hostvars[host] %}
if pool_id.startswith("{{ hostinfo['libvirt_pool'] }}"):
{% for pool in hostinfo["libvirt_pools"] %}
if pool_id.startswith("{{ pool["name"] }}"):
return (
{{ hostinfo["libvirt_pool_order_id"] }},
"qemu+ssh://copr@{{ hostinfo['libvirt_host'] }}/system",
"{{ hostinfo['libvirt_arch'] }}",
# devel uses 0x900+
{% if devel %}"2620:52:6:1161:dead:beef:cafe:c900"{% else %}"2620:52:6:1161:dead:beef:cafe:c100"{% endif %},
"2620:52:6:1161::1",
)
{% endfor %}
{% endfor %}
raise Exception("can't convert pool_id to hv ID")

57
roles/copr/backend/templates/provision/libvirt-new
View file

@ -16,6 +16,8 @@ import shutil
import shlex
import time
import argparse
import ipaddress
from helpers import get_hv_identification_from_pool_id
@ -51,7 +53,7 @@ class LibvirtSpawner:
def __init__(self, resalloc_pool_id, log, args):
self.args = args
self.vm_name = args.name
host_id, self.connection, self.arch = get_hv_identification_from_pool_id(
self.connection, self.arch, *_ = get_hv_identification_from_pool_id(
resalloc_pool_id)
self.config_files.append(ConfigFile(
"resalloc-vars.sh",
@ -382,46 +384,20 @@ def get_arg_parser():
parser.add_argument('--ram-size', metavar='MB', default=4096)
parser.add_argument('--name')
parser.add_argument('--resalloc-pool-id')
parser.add_argument('--resalloc-id-in-pool')
parser.add_argument('--resalloc-ip-offset')
return parser
def get_fedora_ipv6_address(pool_id, id_in_pool, dev, log):
def get_fedora_ipv6_address(pool_id, ip_offset, log):
"""
Statically assign IPv6 + Gateway based on id_in_pool.
"""
hv_id, _, _ = get_hv_identification_from_pool_id(pool_id)
hv_id = int(hv_id)
# rdu-cc
# gateway = "2620:52:3:1:ffff:ffff:ffff:fffe"
# base = "2620:52:3:1:dead:beef:cafe:c"
# rdu3 https://github.com/fedora-copr/copr/issues/3786#issuecomment-3412337856
gateway = "2620:52:6:1161::1"
base = "2620:52:6:1161:dead:beef:cafe:c"
# The initial 256 addresses (:c0XX) is reserved for hypervisor machines
# itself (not really in rdu3, per copr issue/3786 at least, but it doesn't
# hurt to wait those in practice so we keep the pattern).
start = 0x100
# each hypervisor has block of 64 IPs
block = 64
log.info("Hypervisor ID: %s", hv_id)
# give 48 IPs to each hv (32 prod, some dev), currently 4*48=192 ips
offset = hv_id * block
if not dev:
# give the first 8 addresses to Copr dev instance
offset += 8
addr_number = start + offset + int(id_in_pool)
addr_number = "{0:#05x}".format(addr_number).replace("0x", "")
log.info("Using an IPv6 ending with ':c%s'", addr_number)
return base + addr_number, gateway
_, _, ipv6_start_string, gw = get_hv_identification_from_pool_id(pool_id)
log.info("Getting IP in %s + 64 range", ipv6_start_string)
ip_start = ipaddress.IPv6Address(ipv6_start_string)
ip = ip_start + ip_offset
log.info("IP %s", ip)
return str(ip), gw
def _main():
@ -440,16 +416,11 @@ def _main():
_arange_default("name", "RESALLOC_NAME")
_arange_default("resalloc_pool_id", "RESALLOC_POOL_ID")
_arange_default("resalloc_id_in_pool", "RESALLOC_ID_IN_POOL")
devel = True
if "prod" in args.name:
devel = False
_arange_default("resalloc_ip_offset", "NAMED_COUNTER_VALUE_FEDORA_IPV6_POOL")
ip6_a, ip6_g = get_fedora_ipv6_address(args.resalloc_pool_id,
args.resalloc_id_in_pool,
devel, log)
args.resalloc_ip_offset,
log)
spawner = LibvirtSpawner(args.resalloc_pool_id, log, args)
spawner.add_nat_network()

47
roles/copr/backend/templates/resalloc/pools.yaml.j2
View file

@ -201,6 +201,8 @@ vmhost_x86_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}:
max: {{ builders["x86_hypervisor_" + hv]["x86_64"][0] }}
max_starting: {{ builders["x86_hypervisor_" + hv]["x86_64"][1] }}
max_prealloc: {{ builders["x86_hypervisor_" + hv]["x86_64"][2] }}
named_counters:
- FEDORA_IPV6_POOL
tags:
- copr_builder
- arch_noarch
@ -276,7 +278,6 @@ copr_hv_ppc64le_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}:
{% endif %}
{% endfor %}
# Power9 hypervisors
{% for hv in ["01", "02", "03", "04"] %}
{% if "p09_hypervisor_" + hv in builders %}
@ -285,11 +286,14 @@ vmhost_p09_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}:
max: {{ builders["p09_hypervisor_" + hv]["ppc64le"][0] }}
max_starting: {{ builders["p09_hypervisor_" + hv]["ppc64le"][1] }}
max_prealloc: {{ builders["p09_hypervisor_" + hv]["ppc64le"][2] }}
named_counters:
- FEDORA_IPV6_POOL
tags:
- copr_builder
- name: arch_noarch
priority: -8
- arch_ppc64le
- name: arch_ppc64le
priority: 10
- arch_ppc64le_native
- hypervisor
- hypervisor_p09
@ -311,6 +315,45 @@ vmhost_p09_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}:
These machines have POWER9 processors and are located in RDU (N Carolina).
Thank you Fedora Infrastructure team for maintaining the hypervisors.
{% endif %}
{% if "hp_p09_hypervisor_" + hv in builders %}
# Each hypervisor has one High-perf machine.
vmhost_p_p09_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}:
max: 1
max_starting: 1
named_counters:
- FEDORA_IPV6_POOL
tags:
- copr_builder
- name: arch_noarch
priority: -25
- name: arch_ppc64le
priority: 10
- arch_ppc64le_native
- hypervisor
- hypervisor_p09
- hypervisor_p09_{{ hv }}
- arch_power9
tags_on_demand:
- on_demand_powerful
# The Power9 machine has 160 threads. The bottleneck is certainly small
# disk, so try to waste the CPUs appropriately.
cmd_new: "copr-resalloc-vm-ip-to-yaml /var/lib/resallocserver/provision/libvirt-new --cpu-count 24 --ram-size 65536 --swap-vol-size 320"
cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-delete"
cmd_livecheck: "resalloc-check-vm-ip"
cmd_release: "/var/lib/resallocserver/resalloc_provision/vm-release"
cmd_list: "/var/lib/resallocserver/provision/libvirt-list"
livecheck_period: 180
reuse_opportunity_time: 90
reuse_max_count: 8
reuse_max_time: 1800
description: >
A pool that provides one high-perf ppc64le machine on p09 {{ hv }} machine
in the Fedora Community Cage. These machines have POWER9 processors and
are located in RDU (N Carolina). Thank you Fedora Infrastructure team for
maintaining the hypervisors.
{% endif %}
{% endfor %}