Add support for Google Cloud Platform
Introduce a handler to upload the images we're building for Google's cloud.
This commit is contained in:
Jeremy Cline
parent
2483eb9f80
commit
e5c1107601
12 changed files with 3972 additions and 3 deletions
|
|
@ -1,7 +1,8 @@
|
|||
__version__ = "1.1.1"
|
||||
__version__ = "1.2.0"
|
||||
|
||||
from .publish import ( # noqa: F401
|
||||
AwsPublishedV1,
|
||||
AzurePublishedV1,
|
||||
ContainerPublishedV1,
|
||||
GcpPublishedV1,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -250,3 +250,79 @@ class ContainerPublishedV1(_PublishedV1):
|
|||
f"\tTags: {', '.join(self.body['tags'])}\n"
|
||||
f"\tArchitectures: {', '.join(self.body['architectures'])}\n"
|
||||
)
|
||||
|
||||
|
||||
class GcpPublishedV1(_PublishedV1):
|
||||
topic = ".".join([_PublishedV1.topic, "gcp"])
|
||||
body_schema = {
|
||||
"id": f"{SCHEMA_URL}/v1/{'.'.join([_PublishedV1.topic, 'gcp'])}",
|
||||
"$schema": "https://json-schema.org/draft/2019-09/schema",
|
||||
"description": (
|
||||
"Schema for messages sent by fedora-image-uploader when a "
|
||||
"new Google Cloud Platform image is published."
|
||||
),
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"architecture": {
|
||||
"type": "string",
|
||||
"description": "The machine architecture of the image (x86_64, aarch64, etc).",
|
||||
},
|
||||
"compose_id": {
|
||||
"type": "string",
|
||||
"description": "The compose ID this image was created from.",
|
||||
},
|
||||
"release": {
|
||||
"type": "integer",
|
||||
"description": "The release number associated with the image.",
|
||||
},
|
||||
"subvariant": {
|
||||
"type": "string",
|
||||
"description": "The subvariant of the image (e.g. Cloud_Base).",
|
||||
},
|
||||
"family": {
|
||||
"type": "string",
|
||||
"description": "The Google Compute Engine OS family for the image.",
|
||||
},
|
||||
"image_name": {
|
||||
"type": "string",
|
||||
"description": "The name of the image.",
|
||||
},
|
||||
"image_url": {
|
||||
"type": "string",
|
||||
"description": "The URL of the image in Google Cloud Engine.",
|
||||
},
|
||||
"storage_locations": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "The geographic location codes where the image is available.",
|
||||
},
|
||||
},
|
||||
"required": [
|
||||
"architecture",
|
||||
"compose_id",
|
||||
"release",
|
||||
"subvariant",
|
||||
"family",
|
||||
"image_name",
|
||||
"image_url",
|
||||
"storage_locations",
|
||||
],
|
||||
}
|
||||
|
||||
@property
|
||||
def summary(self):
|
||||
return (
|
||||
f"{self.app_name} published the {self.body['architecture']} image from compose"
|
||||
f" {self.body['compose_id']} to the {self.body['family']} family in Google"
|
||||
" Cloud Platform"
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return (
|
||||
"A new image has been published to Google Cloud Platform:\n\n"
|
||||
f"\tArchitecture: {self.body['architecture']}\n"
|
||||
f"\tCompose ID: {self.body['compose_id']}\n"
|
||||
f"\tFamily: {self.body['family']}\n"
|
||||
f"\tImage Name: {self.body['image_name']}\n"
|
||||
f"\tImage URL: {self.body['image_url']}\n"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ test = [
|
|||
"fedora_image_uploader.published.v1.aws" = "fedora_image_uploader_messages.publish:AwsPublishedV1"
|
||||
"fedora_image_uploader.published.v1.azure" = "fedora_image_uploader_messages.publish:AzurePublishedV1"
|
||||
"fedora_image_uploader.published.v1.container" = "fedora_image_uploader_messages.publish:ContainerPublishedV1"
|
||||
"fedora_image_uploader.published.v1.gcp" = "fedora_image_uploader_messages.publish:GcpPublishedV1"
|
||||
|
||||
[tool.hatch.version]
|
||||
path = "fedora_image_uploader_messages/__init__.py"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from fedora_image_uploader_messages import (
|
|||
AwsPublishedV1,
|
||||
AzurePublishedV1,
|
||||
ContainerPublishedV1,
|
||||
GcpPublishedV1,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -192,3 +193,34 @@ def test_container_str():
|
|||
message = ContainerPublishedV1(body=body)
|
||||
assert expected_summary == message.summary
|
||||
assert expected_str == str(message)
|
||||
|
||||
|
||||
def test_gcp_str():
|
||||
message = GcpPublishedV1(
|
||||
topic="fedora_image_uploader.published.v1.gcp.rc.Cloud_Base.aarch64",
|
||||
body={
|
||||
"architecture": "x86_64",
|
||||
"compose_id": "Fedora-40-20240414.0",
|
||||
"release": 40,
|
||||
"subvariant": "Cloud_Base",
|
||||
"family": "fedora-cloud-40",
|
||||
"image_name": "fedora-cloud-40-1-14-x86-64",
|
||||
"image_url": "https://example.com/link",
|
||||
"storage_locations": ["us"],
|
||||
},
|
||||
)
|
||||
expected_summary = (
|
||||
"fedora-image-uploader published the x86_64 image from compose Fedora-40-20240414.0 to "
|
||||
"the fedora-cloud-40 family in Google Cloud Platform"
|
||||
)
|
||||
expected_str = (
|
||||
"A new image has been published to Google Cloud Platform:\n\n"
|
||||
"\tArchitecture: x86_64\n"
|
||||
"\tCompose ID: Fedora-40-20240414.0\n"
|
||||
"\tFamily: fedora-cloud-40\n"
|
||||
"\tImage Name: fedora-cloud-40-1-14-x86-64\n"
|
||||
"\tImage URL: https://example.com/link\n"
|
||||
)
|
||||
|
||||
assert message.summary == expected_summary
|
||||
assert str(message) == expected_str
|
||||
|
|
|
|||
435
fedora-image-uploader/fedora_image_uploader/gcp.py
Normal file
435
fedora-image-uploader/fedora_image_uploader/gcp.py
Normal file
|
|
@ -0,0 +1,435 @@
|
|||
import datetime
|
||||
import hashlib
|
||||
import logging
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
from fedfind import release as ff_release
|
||||
from fedora_image_uploader_messages import GcpPublishedV1
|
||||
from fedora_messaging import config
|
||||
from fedora_messaging import exceptions as fm_exceptions
|
||||
from google.api_core import exceptions as google_exceptions
|
||||
from google.cloud import compute_v1, storage
|
||||
|
||||
from .utils import (
|
||||
download_image,
|
||||
fallible_publish,
|
||||
get_eol,
|
||||
get_milestone,
|
||||
parse_release,
|
||||
)
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Gcp:
|
||||
"""
|
||||
Uploader for Google Cloud Platform.
|
||||
|
||||
This handler uses the following configuration keys under the "gcp" section of "consumer_config":
|
||||
- project: The GCP project to upload images to.
|
||||
- bucket_name: The name of the GCP storage bucket to upload images to.
|
||||
- storage_locations: A list of GCP storage locations to store the image in. Currently this
|
||||
list should contain only one location since GCP rejects it otherwise.
|
||||
- publish_amqp_messages: Whether to publish messages to the AMQP broker.
|
||||
"""
|
||||
|
||||
SUPPORTED_ARCHES = {
|
||||
"x86_64": compute_v1.Image.Architecture.X86_64,
|
||||
"aarch64": compute_v1.Image.Architecture.ARM64,
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.conf = config.conf["consumer_config"]["gcp"]
|
||||
self.storage_client = storage.Client(project=self.conf["project"])
|
||||
self.images_client = compute_v1.ImagesClient()
|
||||
|
||||
def __call__(self, image: dict, ffrel: ff_release.Release):
|
||||
if image.get("arch") not in self.SUPPORTED_ARCHES.keys():
|
||||
_log.debug(f"Skipping unsupported arch '{image['arch']}' for GCP")
|
||||
return
|
||||
if image.get("format") != "tar.gz":
|
||||
_log.debug(f"Skipping unsupported format '{image['format']}' (need tar.gz) for GCP")
|
||||
return
|
||||
if image.get("subvariant") not in ("Cloud_Base", "BaseOS"):
|
||||
_log.debug(
|
||||
"Skipping %s for GCP: subvariant is %s", image.get("path"), image.get("arch")
|
||||
)
|
||||
return
|
||||
if ffrel.relnum < 40 and ffrel.release.lower() != "eln":
|
||||
# The format changed with F40 and I can't be bothered to support the old format for
|
||||
# a month
|
||||
_log.debug("Skipping %s for GCP: F{ffrel.relnum} not supported", image.get("path"))
|
||||
return
|
||||
|
||||
blob = self.upload_disk_image(image)
|
||||
try:
|
||||
uploaded_image = self.import_image(image, ffrel, blob)
|
||||
except google_exceptions.Conflict as e:
|
||||
# The name conflicts; requests are idempotent if and only if the image sha256 is the
|
||||
# same. Something's gone quite wrong here so log at error level, but don't nack the
|
||||
# message since there's not much we can programmatically do to fix this.
|
||||
_log.error(
|
||||
"Unable to import image from %s as it conflicts with an existing image (%s)",
|
||||
blob.self_link,
|
||||
str(e),
|
||||
)
|
||||
return
|
||||
|
||||
message = GcpPublishedV1(
|
||||
topic=".".join(
|
||||
[GcpPublishedV1.topic, get_milestone(ffrel), image["subvariant"], image["arch"]]
|
||||
),
|
||||
body={
|
||||
"architecture": image["arch"],
|
||||
"compose_id": ffrel.cid,
|
||||
"release": ffrel.relnum,
|
||||
"subvariant": image["subvariant"],
|
||||
"family": uploaded_image.family,
|
||||
"image_name": uploaded_image.name,
|
||||
"image_url": uploaded_image.self_link,
|
||||
"storage_locations": uploaded_image.storage_locations,
|
||||
},
|
||||
)
|
||||
|
||||
if self.conf.get("publish_amqp_messages", False):
|
||||
fallible_publish(message)
|
||||
|
||||
try:
|
||||
self.promote_image(uploaded_image)
|
||||
except google_exceptions.GoogleAPICallError as e:
|
||||
# We'll try again next time
|
||||
_log.error("Failed to promote images: %s", str(e))
|
||||
|
||||
try:
|
||||
self.cleanup_old_images()
|
||||
except google_exceptions.GoogleAPICallError as e:
|
||||
# We'll try again next time
|
||||
_log.error("Failed to clean up images: %s", str(e))
|
||||
|
||||
def import_image(
|
||||
self, image: dict, ffrel: ff_release.Release, blob: storage.Blob
|
||||
) -> compute_v1.Image:
|
||||
image_suffix, y_release, z_release = parse_release(ffrel)
|
||||
image_version = f"{ffrel.relnum}.{y_release}.{z_release}"
|
||||
image_name = (
|
||||
f"fedora-cloud-{ffrel.relnum}-{y_release}-{z_release}-{image['arch']}".lower().replace(
|
||||
"_", "-"
|
||||
)
|
||||
)
|
||||
family = f"fedora-cloud-{image_suffix.lower()}"
|
||||
architecture = self.SUPPORTED_ARCHES[image["arch"]]
|
||||
eol = get_eol(ffrel)
|
||||
if eol:
|
||||
eol = eol.strftime("%Y-%m-%d")
|
||||
else:
|
||||
eol = "none"
|
||||
|
||||
# This list may not be complete...
|
||||
features = [
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.UEFI_COMPATIBLE.name),
|
||||
compute_v1.GuestOsFeature(
|
||||
type_=compute_v1.GuestOsFeature.Type.VIRTIO_SCSI_MULTIQUEUE.name
|
||||
),
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.IDPF.name),
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.GVNIC.name),
|
||||
]
|
||||
# Once aarch64 is signed for Secure Boot, we can include this unconditionally
|
||||
if architecture == compute_v1.Image.Architecture.X86_64:
|
||||
features.append(
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.SECURE_BOOT.name),
|
||||
)
|
||||
|
||||
# With the exception of Rawhide and ELN, we initially upload the image as deprecated so
|
||||
# thatit doesn't became the default for the image family. Images can then be selectively
|
||||
# promoted to be "ACTIVE".
|
||||
#
|
||||
# https://cloud.google.com/compute/docs/images/deprecate-custom#deprecation-states
|
||||
deprecated = compute_v1.DeprecationStatus.State.DEPRECATED.name
|
||||
if ffrel.release.lower() in ("rawhide", "eln"):
|
||||
deprecated = compute_v1.DeprecationStatus.State.ACTIVE.name
|
||||
# Also, if this is the first image of a family, make it active
|
||||
request = compute_v1.ListImagesRequest(
|
||||
project=self.conf["project"],
|
||||
filter=f"(family = {family}) AND (architecture = {architecture.name})",
|
||||
max_results=1,
|
||||
)
|
||||
if len(list(self.images_client.list(request=request, timeout=60))) == 0:
|
||||
deprecated = compute_v1.DeprecationStatus.State.ACTIVE.name
|
||||
|
||||
disk = compute_v1.RawDisk(source=blob.self_link)
|
||||
image_resource = compute_v1.Image(
|
||||
architecture=architecture.name,
|
||||
family=family,
|
||||
description=f"Fedora Cloud base image version {image_version}",
|
||||
# Despite being able to set this, it seems to be ignored when adding the image,
|
||||
# so there's a call below to set it again.
|
||||
deprecated=compute_v1.DeprecationStatus(state=deprecated),
|
||||
guest_os_features=features,
|
||||
labels={
|
||||
# Sadly only ASCII lowercase, numeric, underscore and dash are allowed
|
||||
"fedora-compose-id": ffrel.cid.lower().replace(".", "-"),
|
||||
"fedora-subvariant": image["subvariant"].lower(),
|
||||
"fedora-release": ffrel.release.lower(),
|
||||
"fedora-version": str(ffrel.relnum),
|
||||
"end-of-life": eol,
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
# The API rejects this as an "invalid URL".
|
||||
# licenses=["https://docs.fedoraproject.org/en-US/legal/fedora-linux-license/"],
|
||||
name=image_name,
|
||||
raw_disk=disk,
|
||||
# Despite the name, it seems to only accept a list of length 1.
|
||||
storage_locations=self.conf["storage_locations"],
|
||||
)
|
||||
image_request = compute_v1.InsertImageRequest(
|
||||
project=self.conf["project"],
|
||||
image_resource=image_resource,
|
||||
request_id=str(uuid.UUID(hex=image["checksums"]["sha256"][:32])),
|
||||
)
|
||||
_log.info(
|
||||
"Requesting import of image %s to the %s family in GCP as %s",
|
||||
blob.self_link,
|
||||
family,
|
||||
image_name,
|
||||
)
|
||||
try:
|
||||
response = self.images_client.insert(image_request, timeout=60)
|
||||
response.result(timeout=60 * 30)
|
||||
response.exception(timeout=5)
|
||||
if response.warnings:
|
||||
for warning in response.warnings:
|
||||
_log.warning("Warning during image import: %s", warning.message)
|
||||
except google_exceptions.Conflict:
|
||||
# This specific error needs to be handled by this function's caller since retrying is
|
||||
# not going to fix it.
|
||||
raise
|
||||
except google_exceptions.GoogleAPICallError as e:
|
||||
_log.error("Failed to import image %s: %s", image_name, str(e))
|
||||
raise fm_exceptions.Nack()
|
||||
|
||||
# GCE doesn't actually mark the image deprecated despite our request (maybe they'll
|
||||
# fix that some day?), so do it a second time. This obviously introduces a race where
|
||||
# booting from the family during this time gets the new image, but there's not much we
|
||||
# can do about that.
|
||||
if deprecated == compute_v1.DeprecationStatus.State.DEPRECATED.name:
|
||||
try:
|
||||
_log.info(
|
||||
"Requesting new image %s in the %s family be marked DEPRECATED",
|
||||
image_name,
|
||||
family,
|
||||
)
|
||||
request = compute_v1.DeprecateImageRequest(
|
||||
project=self.conf["project"],
|
||||
image=image_name,
|
||||
deprecation_status_resource=compute_v1.DeprecationStatus(
|
||||
state=compute_v1.DeprecationStatus.State.DEPRECATED.name,
|
||||
),
|
||||
request_id=str(uuid.UUID(hex=image["checksums"]["sha256"][:32])),
|
||||
)
|
||||
response = self.images_client.deprecate(request=request, timeout=60)
|
||||
response.result(timeout=60 * 10)
|
||||
response.exception(timeout=5)
|
||||
except google_exceptions.GoogleAPICallError as e:
|
||||
_log.error("Failed to deprecate image %s: %s", image_name, str(e))
|
||||
raise fm_exceptions.Nack()
|
||||
|
||||
try:
|
||||
image_request = compute_v1.GetImageRequest(
|
||||
project=self.conf["project"], image=image_name
|
||||
)
|
||||
image_resource = self.images_client.get(image_request, timeout=30)
|
||||
except google_exceptions.GoogleAPICallError as e:
|
||||
_log.error("Failed to read image details for %s: %s", image_name, str(e))
|
||||
raise fm_exceptions.Nack()
|
||||
|
||||
_log.info(
|
||||
"Successfully uploaded image %s to GCP under the %s family",
|
||||
image_resource.name,
|
||||
image_resource.family,
|
||||
)
|
||||
return image_resource
|
||||
|
||||
def upload_disk_image(self, image: dict) -> storage.Blob:
|
||||
"""
|
||||
Upload the disk image to a GCP storage bucket.
|
||||
|
||||
This is a pre-requisite for importing the image as a machine image.
|
||||
"""
|
||||
bucket = self.storage_client.bucket(self.conf["bucket_name"])
|
||||
blob_name = f"{image['checksums']['sha256']}.tar.gz"
|
||||
blob = bucket.blob(blob_name)
|
||||
|
||||
if blob.exists():
|
||||
blob.reload()
|
||||
_log.info("Skipping upload of GCP image to %s as it already exists", blob.self_link)
|
||||
else:
|
||||
with tempfile.TemporaryDirectory() as workdir:
|
||||
image_path = download_image(image, workdir, decompress=False)
|
||||
_log.info("Starting upload of GCP image to %s", bucket.name)
|
||||
try:
|
||||
blob.upload_from_filename(image_path, if_generation_match=0, timeout=60 * 30)
|
||||
_log.info("Finished upload of GCP image as %s", blob.self_link)
|
||||
except google_exceptions.GoogleAPICallError as e:
|
||||
_log.error("Failed to upload image to bucket: %s", str(e))
|
||||
raise fm_exceptions.Nack()
|
||||
|
||||
return blob
|
||||
|
||||
def promote_image(self, new_image: compute_v1.Image):
|
||||
"""
|
||||
Promote an image from deprecated to active state if it's been 2 weeks since the last
|
||||
promotion.
|
||||
"""
|
||||
# These are always active
|
||||
if new_image.labels["fedora-release"] in ("rawhide", "eln"):
|
||||
return
|
||||
|
||||
# We can't filter on deprecated.state = ACTIVE since that's not set initially and no value
|
||||
# is the same thing as ACTIVE, apparently.
|
||||
request = compute_v1.ListImagesRequest(
|
||||
project=self.conf["project"],
|
||||
filter=(
|
||||
f"(family = {new_image.family}) AND " f"(architecture = {new_image.architecture})"
|
||||
),
|
||||
)
|
||||
response = self.images_client.list(request=request, timeout=60)
|
||||
|
||||
today = datetime.datetime.now(tz=datetime.UTC)
|
||||
two_weeks_ago = today - datetime.timedelta(days=14)
|
||||
active_images = [
|
||||
image
|
||||
for image in response
|
||||
if image.deprecated != compute_v1.DeprecationStatus.State.DEPRECATED.name
|
||||
]
|
||||
active_images.sort(
|
||||
key=lambda image: datetime.datetime.fromisoformat(image.creation_timestamp),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
try:
|
||||
latest_active = active_images[0]
|
||||
if two_weeks_ago > datetime.datetime.fromisoformat(latest_active.creation_timestamp):
|
||||
_log.info(
|
||||
"The latest active image in the %s family, %s, is more than two weeks old",
|
||||
latest_active.family,
|
||||
latest_active.name,
|
||||
)
|
||||
else:
|
||||
_log.info(
|
||||
"The latest active image in the %s family, %s, is less than two weeks old (%s)",
|
||||
latest_active.family,
|
||||
latest_active.name,
|
||||
str(latest_active.creation_timestamp),
|
||||
)
|
||||
return
|
||||
except IndexError:
|
||||
_log.error(
|
||||
"No active images found in the %s family, but the first image should have been"
|
||||
" active!",
|
||||
new_image.family,
|
||||
)
|
||||
return
|
||||
|
||||
request = compute_v1.DeprecateImageRequest(
|
||||
project=self.conf["project"],
|
||||
image=new_image.name,
|
||||
deprecation_status_resource=compute_v1.DeprecationStatus(
|
||||
state=compute_v1.DeprecationStatus.State.ACTIVE.name,
|
||||
),
|
||||
)
|
||||
response = self.images_client.deprecate(request=request, timeout=60)
|
||||
_log.info("Requested image %s in the %s be marked ACTIVE", new_image.name)
|
||||
response.result(timeout=60 * 10)
|
||||
response.exception(timeout=5)
|
||||
if response.warnings:
|
||||
for warning in response.warnings:
|
||||
_log.warning("Warning while marking image active: %s", warning.message)
|
||||
|
||||
def cleanup_old_images(self):
|
||||
"""
|
||||
Remove old images from GCP.
|
||||
|
||||
The retention policy implemented here is as follows:
|
||||
- Images for stable releases marked as active are retained until their EOL date.
|
||||
- Images for stable releases marked as deprecated are deleted after 14 days.
|
||||
- Rawhide and ELN images are retained for 14 days.
|
||||
"""
|
||||
# Start by cleaning up deprecated and Rawhide/ELN images
|
||||
request = compute_v1.ListImagesRequest(
|
||||
project=self.conf["project"],
|
||||
filter=(
|
||||
"(deprecated.state = DEPRECATED) OR "
|
||||
"(labels.fedora-release = rawhide) OR "
|
||||
"(labels.fedora-release = eln)"
|
||||
),
|
||||
)
|
||||
today = datetime.datetime.today().replace(tzinfo=datetime.UTC)
|
||||
two_weeks_ago = today - datetime.timedelta(days=14)
|
||||
delete_responses = []
|
||||
response = self.images_client.list(request=request, timeout=60)
|
||||
for image in response:
|
||||
if image.labels.get("fedora-image-uploader-managed") != "true":
|
||||
# We didn't upload this image, so we shouldn't delete it
|
||||
continue
|
||||
image_creation = datetime.datetime.fromisoformat(image.creation_timestamp)
|
||||
if two_weeks_ago > image_creation:
|
||||
_log.info(
|
||||
"Requesting delete for image %s since it's older than %s",
|
||||
image.name,
|
||||
str(two_weeks_ago),
|
||||
)
|
||||
request = compute_v1.DeleteImageRequest(
|
||||
project=self.conf["project"],
|
||||
image=image.name,
|
||||
request_id=str(
|
||||
uuid.UUID(
|
||||
hex=hashlib.sha256(image.name.encode(errors="ignore")).hexdigest()[:32]
|
||||
)
|
||||
),
|
||||
)
|
||||
delete_responses.append(self.images_client.delete(request, timeout=60 * 10))
|
||||
|
||||
# Next, remove any images with an EOL date that has passed. There's probably a fancy filter
|
||||
# to do this, but I don't know the right syntax and we're going to be dealing with no more
|
||||
# than a couple hundred images at a time.
|
||||
request = compute_v1.ListImagesRequest(project=self.conf["project"])
|
||||
response = self.images_client.list(request=request, timeout=60)
|
||||
for image in response:
|
||||
if image.labels.get("fedora-image-uploader-managed") != "true":
|
||||
# We didn't upload this image, so we shouldn't delete it
|
||||
continue
|
||||
|
||||
try:
|
||||
eol = datetime.datetime.fromisoformat(image.labels["end-of-life"]).replace(
|
||||
tzinfo=datetime.UTC
|
||||
)
|
||||
except ValueError:
|
||||
# Pre-release, Rawhide, and ELN have an EOL of "none"
|
||||
_log.debug(
|
||||
"Skipping image %s with EOL of %s", image.name, image.labels["end-of-life"]
|
||||
)
|
||||
continue
|
||||
except KeyError:
|
||||
_log.error("Image %s is missing an EOL label", image.name)
|
||||
continue
|
||||
|
||||
if today > eol:
|
||||
_log.info("Requesting delete for image %s since it's past EOL", image.name)
|
||||
request = compute_v1.DeleteImageRequest(
|
||||
project=self.conf["project"],
|
||||
image=image.name,
|
||||
request_id=str(
|
||||
uuid.UUID(
|
||||
hex=hashlib.sha256(image.name.encode(errors="ignore")).hexdigest()[:32]
|
||||
)
|
||||
),
|
||||
)
|
||||
delete_responses.append(self.images_client.delete(request, timeout=60 * 10))
|
||||
|
||||
for response in delete_responses:
|
||||
response.result(timeout=60 * 30)
|
||||
response.exception(timeout=5)
|
||||
if response.warnings:
|
||||
for warning in response.warnings:
|
||||
_log.warning("Warning during image import: %s", warning.message)
|
||||
|
|
@ -30,6 +30,11 @@ try:
|
|||
except ImportError:
|
||||
_log.info("Install the 'azure' extra for Azure support")
|
||||
|
||||
try:
|
||||
from . import gcp
|
||||
except ImportError:
|
||||
_log.info("Install the 'gcp' extra for Google Cloud Platform support")
|
||||
|
||||
|
||||
DOCKER_ARCHES = {"amd64": "x86_64", "arm64": "aarch64", "ppc64le": "ppc64le", "s390x": "s390x"}
|
||||
|
||||
|
|
@ -76,6 +81,8 @@ class Uploader:
|
|||
self.handlers["azure"] = azure.Azure()
|
||||
if "aws" in self.conf.keys():
|
||||
self.handlers["aws"] = aws.Aws()
|
||||
if "gcp" in self.conf.keys():
|
||||
self.handlers["gcp"] = gcp.Gcp()
|
||||
|
||||
# tracks the container repos we got images for, for manifest
|
||||
# creation purposes
|
||||
|
|
|
|||
|
|
@ -30,8 +30,12 @@ azure = [
|
|||
"azure-mgmt-compute",
|
||||
"azure-storage-blob",
|
||||
]
|
||||
gcp = [
|
||||
"google-cloud-compute",
|
||||
"google-cloud-storage",
|
||||
]
|
||||
dev = [
|
||||
"fedora-image-uploader[aws,azure]",
|
||||
"fedora-image-uploader[aws,azure,gcp]",
|
||||
"black",
|
||||
"isort",
|
||||
"flake8",
|
||||
|
|
|
|||
1446
fedora-image-uploader/tests/fixtures/cassettes/test_import_image.yaml
vendored
Normal file
1446
fedora-image-uploader/tests/fixtures/cassettes/test_import_image.yaml
vendored
Normal file
File diff suppressed because it is too large
Load diff
1387
fedora-image-uploader/tests/fixtures/cassettes/test_messages.yaml
vendored
Normal file
1387
fedora-image-uploader/tests/fixtures/cassettes/test_messages.yaml
vendored
Normal file
File diff suppressed because it is too large
Load diff
574
fedora-image-uploader/tests/test_gcp.py
Normal file
574
fedora-image-uploader/tests/test_gcp.py
Normal file
|
|
@ -0,0 +1,574 @@
|
|||
import json
|
||||
import os
|
||||
from unittest import mock
|
||||
|
||||
import freezegun
|
||||
import pytest
|
||||
from fedora_image_uploader_messages import GcpPublishedV1
|
||||
from fedora_messaging import config, message
|
||||
from fedora_messaging import testing as fm_testing
|
||||
from google.cloud import compute_v1, storage
|
||||
|
||||
from fedora_image_uploader import Uploader
|
||||
|
||||
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
"publish_amqp_messages": True,
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_image_filter(fixtures_dir):
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.upload_disk_image = mock.Mock()
|
||||
image = {
|
||||
"arch": "aarch64",
|
||||
"format": "tar.gz",
|
||||
"subvariant": "Cloud_Base",
|
||||
}
|
||||
ffrel = mock.Mock(relnum=42, release="Rawhide")
|
||||
|
||||
# Arch isn't supported
|
||||
image["arch"] = "ppc64le"
|
||||
handler(image, ffrel)
|
||||
handler.upload_disk_image.call_count == 0
|
||||
image["arch"] = "aarch64"
|
||||
|
||||
# Format isn't tar.gz
|
||||
image["format"] = "qcow2"
|
||||
handler(image, ffrel)
|
||||
handler.upload_disk_image.call_count == 0
|
||||
image["format"] = "tar.gz"
|
||||
|
||||
# Subvariant isn't supported
|
||||
image["subvariant"] = "CoreOS"
|
||||
handler(image, ffrel)
|
||||
handler.upload_disk_image.call_count == 0
|
||||
|
||||
# Don't bother with 39
|
||||
handler(image, mock.Mock(relnum=39, release="39"))
|
||||
handler.upload_disk_image.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.vcr
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
"publish_amqp_messages": True,
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_messages(fixtures_dir):
|
||||
with open(os.path.join(fixtures_dir, "messages/rc_compose.json")) as fd:
|
||||
msg = message.load_message(json.load(fd))
|
||||
fake_images = [
|
||||
mock.Mock(
|
||||
family="fedora-cloud-40",
|
||||
name="fedora-cloud-40-1-14-aarch64",
|
||||
self_link="http://example.com/link",
|
||||
storage_locations=["us"],
|
||||
),
|
||||
mock.Mock(
|
||||
family="fedora-cloud-40",
|
||||
name="fedora-cloud-40-1-14-x86-64",
|
||||
self_link="http://example.com/link",
|
||||
storage_locations=["us"],
|
||||
),
|
||||
]
|
||||
fake_images[0].name = "fedora-cloud-40-1-14-aarch64"
|
||||
fake_images[1].name = "fedora-cloud-40-1-14-x86-64"
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.upload_disk_image = mock.Mock()
|
||||
handler.import_image = mock.Mock(side_effect=fake_images)
|
||||
handler.promote_image = mock.Mock()
|
||||
handler.cleanup_old_images = mock.Mock()
|
||||
|
||||
expected_messages = [
|
||||
GcpPublishedV1(
|
||||
topic="fedora_image_uploader.published.v1.gcp.rc.Cloud_Base.aarch64",
|
||||
body={
|
||||
"architecture": "aarch64",
|
||||
"compose_id": "Fedora-40-20240414.0",
|
||||
"release": 40,
|
||||
"subvariant": "Cloud_Base",
|
||||
"family": "fedora-cloud-40",
|
||||
"image_name": "fedora-cloud-40-1-14-aarch64",
|
||||
"image_url": "http://example.com/link",
|
||||
"storage_locations": ["us"],
|
||||
},
|
||||
),
|
||||
GcpPublishedV1(
|
||||
topic="fedora_image_uploader.published.v1.gcp.rc.Cloud_Base.x86_64",
|
||||
body={
|
||||
"architecture": "x86_64",
|
||||
"compose_id": "Fedora-40-20240414.0",
|
||||
"release": 40,
|
||||
"subvariant": "Cloud_Base",
|
||||
"family": "fedora-cloud-40",
|
||||
"image_name": "fedora-cloud-40-1-14-x86-64",
|
||||
"image_url": "http://example.com/link",
|
||||
"storage_locations": ["us"],
|
||||
},
|
||||
),
|
||||
]
|
||||
with fm_testing.mock_sends(*expected_messages):
|
||||
consumer(msg)
|
||||
|
||||
|
||||
@pytest.mark.vcr
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_import_image(fixtures_dir):
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.images_client = mock.Mock()
|
||||
handler.upload_disk_image = mock.Mock()
|
||||
handler.promote_image = mock.Mock()
|
||||
handler.cleanup_old_image = mock.Mock()
|
||||
handler.images_client.list.return_value = []
|
||||
handler.images_client.insert.return_value.warnings = []
|
||||
handler.upload_disk_image.return_value = mock.Mock(
|
||||
spec=storage.Blob, self_link="http://example.com/link"
|
||||
)
|
||||
expected_calls = [
|
||||
mock.call(
|
||||
compute_v1.InsertImageRequest(
|
||||
request_id="7153713f-2a44-6073-76b4-5786f609026c",
|
||||
project="fedora-cloud-devel",
|
||||
image_resource=compute_v1.Image(
|
||||
architecture="ARM64",
|
||||
family="fedora-cloud-40",
|
||||
description="Fedora Cloud base image version 40.1.14",
|
||||
deprecated=compute_v1.DeprecationStatus(state="ACTIVE"),
|
||||
guest_os_features=[
|
||||
compute_v1.GuestOsFeature(
|
||||
type_=compute_v1.GuestOsFeature.Type.UEFI_COMPATIBLE.name
|
||||
),
|
||||
compute_v1.GuestOsFeature(
|
||||
type_=compute_v1.GuestOsFeature.Type.VIRTIO_SCSI_MULTIQUEUE.name
|
||||
),
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.IDPF.name),
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.GVNIC.name),
|
||||
],
|
||||
labels={
|
||||
"fedora-compose-id": "fedora-40-20240414-0",
|
||||
"fedora-subvariant": "cloud_base",
|
||||
"fedora-release": "40",
|
||||
"fedora-version": "40",
|
||||
"end-of-life": "2025-05-13",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="fedora-cloud-40-1-14-aarch64",
|
||||
raw_disk=compute_v1.RawDisk(
|
||||
source=handler.upload_disk_image.return_value.self_link
|
||||
),
|
||||
storage_locations=["us"],
|
||||
),
|
||||
),
|
||||
timeout=60,
|
||||
),
|
||||
mock.call(
|
||||
compute_v1.InsertImageRequest(
|
||||
request_id="dcc05425-e07b-87f6-f53f-142c693bff3d",
|
||||
project="fedora-cloud-devel",
|
||||
image_resource=compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-cloud-40",
|
||||
description="Fedora Cloud base image version 40.1.14",
|
||||
deprecated=compute_v1.DeprecationStatus(state="ACTIVE"),
|
||||
guest_os_features=[
|
||||
compute_v1.GuestOsFeature(
|
||||
type_=compute_v1.GuestOsFeature.Type.UEFI_COMPATIBLE.name
|
||||
),
|
||||
compute_v1.GuestOsFeature(
|
||||
type_=compute_v1.GuestOsFeature.Type.VIRTIO_SCSI_MULTIQUEUE.name
|
||||
),
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.IDPF.name),
|
||||
compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.GVNIC.name),
|
||||
compute_v1.GuestOsFeature(
|
||||
type_=compute_v1.GuestOsFeature.Type.SECURE_BOOT.name
|
||||
),
|
||||
],
|
||||
labels={
|
||||
"fedora-compose-id": "fedora-40-20240414-0",
|
||||
"fedora-subvariant": "cloud_base",
|
||||
"fedora-release": "40",
|
||||
"fedora-version": "40",
|
||||
"end-of-life": "2025-05-13",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="fedora-cloud-40-1-14-x86-64",
|
||||
raw_disk=compute_v1.RawDisk(
|
||||
source=handler.upload_disk_image.return_value.self_link
|
||||
),
|
||||
storage_locations=["us"],
|
||||
),
|
||||
),
|
||||
timeout=60,
|
||||
),
|
||||
]
|
||||
|
||||
with open(os.path.join(fixtures_dir, "messages/rc_compose.json")) as fd:
|
||||
consumer(message.load_message(json.load(fd)))
|
||||
|
||||
for call in handler.images_client.insert.call_args_list:
|
||||
assert call in expected_calls
|
||||
|
||||
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_promote_image_eln_rawhide():
|
||||
"""Assert this is a no-op for eln and rawhide"""
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.images_client = mock.Mock()
|
||||
|
||||
# rawhide
|
||||
handler.promote_image(
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "rawhide",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="fedora-40-1-14-x86-64",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-01T00:00:00Z",
|
||||
)
|
||||
)
|
||||
handler.images_client.list.call_count = 0
|
||||
|
||||
# eln
|
||||
handler.promote_image(
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "eln",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="fedora-40-1-14-x86-64",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-01T00:00:00Z",
|
||||
)
|
||||
)
|
||||
handler.images_client.list.call_count = 0
|
||||
|
||||
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_needs_promotion():
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.images_client = mock.Mock()
|
||||
handler.images_client.list.side_effect = (
|
||||
[
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "40",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="olde",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-01T00:00:00Z",
|
||||
)
|
||||
],
|
||||
)
|
||||
handler.images_client.deprecate.return_value.warnings = []
|
||||
|
||||
with freezegun.freeze_time("2024-10-16"):
|
||||
handler.promote_image(
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "40",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="promoted",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-15T00:00:00Z",
|
||||
)
|
||||
)
|
||||
|
||||
handler.images_client.deprecate.assert_called_once_with(
|
||||
request=compute_v1.DeprecateImageRequest(
|
||||
project="fedora-cloud-devel",
|
||||
image="promoted",
|
||||
deprecation_status_resource=compute_v1.DeprecationStatus(
|
||||
state=compute_v1.DeprecationStatus.State.ACTIVE.name,
|
||||
),
|
||||
),
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_no_promotion():
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.images_client = mock.Mock()
|
||||
handler.images_client.list.side_effect = (
|
||||
[
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "40",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="olde",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-05T00:00:00Z",
|
||||
)
|
||||
],
|
||||
)
|
||||
handler.images_client.deprecate.return_value.warnings = []
|
||||
|
||||
with freezegun.freeze_time("2024-10-16"):
|
||||
handler.promote_image(
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "40",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="promoted",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-15T00:00:00Z",
|
||||
)
|
||||
)
|
||||
|
||||
assert handler.images_client.deprecate.call_count == 0
|
||||
|
||||
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_cleanup_skips_unmanaged_images():
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.images_client = mock.Mock()
|
||||
handler.images_client.list.return_value = [
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
name="fedora-40-1-14-aarch64",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
)
|
||||
]
|
||||
|
||||
# Test that cleanup_old_images is not called when the image is not managed
|
||||
handler.cleanup_old_images()
|
||||
assert handler.images_client.delete.call_count == 0
|
||||
|
||||
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_cleanup_rawhide():
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.images_client = mock.Mock()
|
||||
handler.images_client.list.side_effect = (
|
||||
[
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "rawhide",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="gone-but-not-forgotten",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-01T00:00:00Z",
|
||||
),
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "rawhide",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
},
|
||||
name="soon-to-go-but-not-yet",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-02T00:00:00Z",
|
||||
),
|
||||
],
|
||||
[],
|
||||
)
|
||||
handler.images_client.delete.return_value.warnings = []
|
||||
|
||||
# Test that cleanup_old_images is not called when the image is not managed
|
||||
with freezegun.freeze_time("2024-10-16"):
|
||||
handler.cleanup_old_images()
|
||||
assert handler.images_client.delete.call_count == 1
|
||||
handler.images_client.delete.assert_called_once_with(
|
||||
compute_v1.DeleteImageRequest(
|
||||
project="fedora-cloud-devel",
|
||||
image="gone-but-not-forgotten",
|
||||
request_id="347bf095-cee1-dc0f-8b0b-287da4300d98",
|
||||
),
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
|
||||
@mock.patch.dict(
|
||||
config.conf,
|
||||
{
|
||||
"consumer_config": {
|
||||
"gcp": {
|
||||
"project": "fedora-cloud-devel",
|
||||
"bucket_name": "some-unique-bucket-name",
|
||||
"storage_locations": ["us"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
def test_cleanup_eol():
|
||||
consumer = Uploader()
|
||||
handler = consumer.handlers["gcp"]
|
||||
handler.images_client = mock.Mock()
|
||||
handler.images_client.list.side_effect = (
|
||||
[],
|
||||
[
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "40",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
"end-of-life": "2024-10-01",
|
||||
},
|
||||
name="gone-but-not-forgotten",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-01T00:00:00Z",
|
||||
),
|
||||
compute_v1.Image(
|
||||
architecture="X86_64",
|
||||
family="fedora-40",
|
||||
description="Fedora Cloud base image version",
|
||||
labels={
|
||||
"fedora-release": "40",
|
||||
"fedora-image-uploader-managed": "true",
|
||||
"end-of-life": "2024-10-17",
|
||||
},
|
||||
name="still-alive",
|
||||
raw_disk=compute_v1.RawDisk(source="http://example.com/link"),
|
||||
storage_locations=["us"],
|
||||
creation_timestamp="2024-10-01T00:00:00Z",
|
||||
),
|
||||
],
|
||||
)
|
||||
handler.images_client.delete.return_value.warnings = []
|
||||
|
||||
# Test that cleanup_old_images is not called when the image is not managed
|
||||
with freezegun.freeze_time("2024-10-16"):
|
||||
handler.cleanup_old_images()
|
||||
|
||||
handler.images_client.delete.assert_called_once_with(
|
||||
compute_v1.DeleteImageRequest(
|
||||
project="fedora-cloud-devel",
|
||||
image="gone-but-not-forgotten",
|
||||
request_id="347bf095-cee1-dc0f-8b0b-287da4300d98",
|
||||
),
|
||||
timeout=600,
|
||||
)
|
||||
|
|
@ -5,7 +5,7 @@ isolated_build = True
|
|||
[testenv]
|
||||
deps =
|
||||
../fedora-image-uploader-messages/
|
||||
.[test,azure,aws]
|
||||
.[test,azure,aws,gcp]
|
||||
sitepackages = False
|
||||
skip_install = True
|
||||
commands_pre =
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ queue = "my_queue"
|
|||
exchange = "amq.topic"
|
||||
routing_keys = ["org.fedoraproject.*.pungi.compose.status.change"]
|
||||
|
||||
|
||||
[consumer_config.azure]
|
||||
location = "eastus"
|
||||
resource_group_name = "fedora-test"
|
||||
|
|
@ -108,6 +109,11 @@ ami_regions = [
|
|||
]
|
||||
|
||||
|
||||
[consumer_config.gcp]
|
||||
project = "fedora-cloud-devel"
|
||||
bucket_name = "fedora-cloud-image-upload-devel"
|
||||
storage_locations = ["us"]
|
||||
|
||||
[qos]
|
||||
prefetch_size = 0
|
||||
prefetch_count = 25
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue