arif/zaza-openstack-tests
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Convert images to raw if ceph image backend (#1078)

Browse Source 
Convert images to raw if ceph image backend

We are currently uploading qcow2 images, and
Nova is converting them to raw when running
the tests, sometimes timing out the tests.

With this change we are pre-converting the
images and uploading them as raw, so Nova
does not have to convert them.
This commit is contained in:
Rodrigo Barbieri
2023-08-21 10:15:51 -03:00
committed by GitHub
parent 3b17ad9d97
commit e0498d6a16
3 changed files with 198 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
unit_tests/utilities/test_zaza_utilities_openstack.py
+127
View File
@@ -543,10 +543,99 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
expected_status='active',
msg='Image status wait')
def test_is_ceph_image_backend_True(self):
self.patch_object(openstack_utils.juju_utils, "get_full_juju_status",
return_value={
"applications": {
"glance": {
"relations": {
"ceph": [
"ceph-mon"
]
}
}
}
})
self.assertTrue(openstack_utils.is_ceph_image_backend())
self.get_full_juju_status.assert_called_once_with(model_name=None)
def test_is_ceph_image_backend_False(self):
self.patch_object(openstack_utils.juju_utils, "get_full_juju_status",
return_value={
"applications": {
"glance": {
"relations": {
}
}
}
})
self.assertFalse(openstack_utils.is_ceph_image_backend('foo'))
self.get_full_juju_status.assert_called_once_with(model_name='foo')
def test_convert_image_format_to_raw_if_qcow2_qemu_cmd_error(self):
self.patch_object(openstack_utils.subprocess, "check_output")
self.check_output.side_effect = subprocess.CalledProcessError(
returncode=42, cmd='mycmd')
self.assertEqual("/tmp/original_path",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_called_once_with([
"qemu-img", "info", "--output=json", '/tmp/original_path'])
def test_convert_image_format_to_raw_if_qcow2_raw_error(self):
self.patch_object(openstack_utils.os.path, "exists",
return_value=False)
self.patch_object(openstack_utils.subprocess, "check_output",
side_effect=[
'{"format": "qcow2"}',
subprocess.CalledProcessError(
returncode=42, cmd='mycmd')])
self.assertEqual("/tmp/original_path",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_has_calls([
mock.call(["qemu-img", "info", "--output=json",
'/tmp/original_path']),
mock.call(["qemu-img", "convert", '/tmp/original_path',
'/tmp/original_path.raw'])
])
def test_convert_image_format_to_raw_if_qcow2_raw_success(self):
self.patch_object(openstack_utils.os.path, "exists",
return_value=False)
self.patch_object(openstack_utils.subprocess, "check_output",
side_effect=['{"format": "qcow2"}', 'success'])
self.assertEqual("/tmp/original_path.raw",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_has_calls([
mock.call(["qemu-img", "info", "--output=json",
'/tmp/original_path']),
mock.call(["qemu-img", "convert", '/tmp/original_path',
'/tmp/original_path.raw'])
])
def test_convert_image_format_to_raw_if_qcow2_raw_already_exists(self):
self.patch_object(openstack_utils.pathlib.Path, "exists",
return_value=True)
self.patch_object(openstack_utils.subprocess, "check_output",
return_value='{"format": "qcow2"}')
self.assertEqual("/tmp/original_path.raw",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_called_once_with([
"qemu-img", "info", "--output=json", '/tmp/original_path'])
def test_create_image_use_tempdir(self):
glance_mock = mock.MagicMock()
self.patch_object(openstack_utils.os.path, "exists")
self.patch_object(openstack_utils, "download_image")
self.patch_object(openstack_utils, "is_ceph_image_backend",
return_value=True)
self.patch_object(openstack_utils,
"convert_image_format_to_raw_if_qcow2",
return_value='wibbly/c.img.raw')
self.patch_object(openstack_utils, "upload_image_to_glance")
self.patch_object(openstack_utils.tempfile, "gettempdir")
self.gettempdir.return_value = "wibbly"
@@ -555,6 +644,37 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
'http://cirros/c.img',
'bob')
self.exists.return_value = False
self.download_image.assert_called_once_with(
'http://cirros/c.img',
'wibbly/c.img')
self.upload_image_to_glance.assert_called_once_with(
glance_mock,
'wibbly/c.img.raw',
'bob',
backend=None,
disk_format='raw',
visibility='public',
container_format='bare',
force_import=False)
self.convert_image_format_to_raw_if_qcow2.assert_called_once_with(
'wibbly/c.img')
def test_create_image_not_convert(self):
glance_mock = mock.MagicMock()
self.patch_object(openstack_utils.os.path, "exists")
self.patch_object(openstack_utils, "download_image")
self.patch_object(openstack_utils, "is_ceph_image_backend")
self.patch_object(openstack_utils,
"convert_image_format_to_raw_if_qcow2")
self.patch_object(openstack_utils, "upload_image_to_glance")
self.patch_object(openstack_utils.tempfile, "gettempdir")
self.gettempdir.return_value = "wibbly"
openstack_utils.create_image(
glance_mock,
'http://cirros/c.img',
'bob',
convert_image_to_raw_if_ceph_used=False)
self.exists.return_value = False
self.download_image.assert_called_once_with(
'http://cirros/c.img',
'wibbly/c.img')
@@ -567,11 +687,17 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
visibility='public',
container_format='bare',
force_import=False)
self.is_ceph_image_backend.assert_not_called()
self.convert_image_format_to_raw_if_qcow2.assert_not_called()
def test_create_image_pass_directory(self):
glance_mock = mock.MagicMock()
self.patch_object(openstack_utils.os.path, "exists")
self.patch_object(openstack_utils, "download_image")
self.patch_object(openstack_utils,
"convert_image_format_to_raw_if_qcow2")
self.patch_object(openstack_utils, "is_ceph_image_backend",
return_value=False)
self.patch_object(openstack_utils, "upload_image_to_glance")
self.patch_object(openstack_utils.tempfile, "gettempdir")
openstack_utils.create_image(
@@ -593,6 +719,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase):
container_format='bare',
force_import=False)
self.gettempdir.assert_not_called()
self.convert_image_format_to_raw_if_qcow2.assert_not_called()
def test_create_ssh_key(self):
nova_mock = mock.MagicMock()
zaza/openstack/charm_tests/glance/tests.py
+2 -1
View File
@@ -92,7 +92,8 @@ class GlanceTest(test_utils.OpenStackBaseTest):
self.glance_client,
image_url,
'cirros-test-import',
force_import=True)
force_import=True,
convert_image_to_raw_if_ceph_used=False)
disk_format = self.glance_client.images.get(image.id).disk_format
self.assertEqual('raw', disk_format)
zaza/openstack/utilities/openstack.py
+69 -2
View File
@@ -22,10 +22,12 @@ import datetime
import enum
import io
import itertools
import json
import juju_wait
import logging
import os
import paramiko
import pathlib
import re
import shutil
import six
@@ -2612,7 +2614,7 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
:type local_path: str
:param image_name: The label to give the image in glance
:type image_name: str
:param disk_format: The of the underlying disk image.
:param disk_format: The format of the underlying disk image.
:type disk_format: str
:param visibility: Who can access image
:type visibility: str (public, private, shared or community)
@@ -2651,10 +2653,65 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
return image
def is_ceph_image_backend(model_name=None):
"""Check if glance is related to ceph, therefore using ceph as backend.
:returns: True if glance is related to Ceph, otherwise False
:rtype: bool
"""
status = juju_utils.get_full_juju_status(model_name=model_name)
result = False
try:
result = 'ceph-mon' in (
status['applications']['glance']['relations']['ceph'])
except KeyError:
pass
logging.debug("Detected Ceph related to Glance?: {}".format(result))
return result
def convert_image_format_to_raw_if_qcow2(local_path):
"""Convert the image format to raw if the detected format is qcow2.
:param local_path: The path to the original image file.
:type local_path: str
:returns: The path to the final image file
:rtype: str
"""
try:
output = subprocess.check_output([
"qemu-img", "info", "--output=json", local_path])
except subprocess.CalledProcessError:
logging.error("Image conversion: Failed to detect image format "
"of file {}".format(local_path))
return local_path
result = json.loads(output)
if result['format'] == 'qcow2':
logging.info("Image conversion: Detected qcow2 vs desired raw format"
" of file {}".format(local_path))
converted_path = pathlib.Path(local_path).resolve().with_suffix('.raw')
converted_path_str = str(converted_path)
if converted_path.exists():
logging.info("Image conversion: raw converted file already"
" exists: {}".format(converted_path_str))
return converted_path_str
logging.info("Image conversion: Converting image {} to raw".format(
local_path))
try:
output = subprocess.check_output([
"qemu-img", "convert", local_path, converted_path_str])
except subprocess.CalledProcessError:
logging.error("Image conversion: Failed to convert image"
" {} to raw".format(local_path))
return local_path
return converted_path_str
return local_path
def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
properties=None, backend=None, disk_format='qcow2',
visibility='public', container_format='bare',
force_import=False):
force_import=False, convert_image_to_raw_if_ceph_used=True):
"""Download the image and upload it to glance.
Download an image from image_url and upload it to glance labelling
@@ -2676,6 +2733,10 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
:param force_import: Force the use of glance image import
instead of direct upload
:type force_import: boolean
:param convert_image_to_raw_if_ceph_used: force conversion of requested
image to raw upon download if Ceph is present in the model and
has a relation to Glance
:type convert_image_to_raw_if_ceph_used: boolean
:returns: glance image pointer
:rtype: glanceclient.common.utils.RequestIdProxy
"""
@@ -2695,6 +2756,12 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
logging.info('Cached image found at {} - Skipping download'.format(
local_path))
if convert_image_to_raw_if_ceph_used and is_ceph_image_backend():
logging.info("Image conversion: Detected ceph backend, forcing"
" use of raw image format")
disk_format = 'raw'
local_path = convert_image_format_to_raw_if_qcow2(local_path)
image = upload_image_to_glance(
glance, local_path, image_name, backend=backend,
disk_format=disk_format, visibility=visibility,