[virt-tools-list] [virt-bootstrap] [PATCH v6 14/26] Create qcow2 images with guestfs-python
Radostin Stoyanov
rstoyanov1 at gmail.com
Thu Aug 17 09:39:52 UTC 2017
Use the python bindings of libguestfs to create qcow2 image with
backing chains to mimic the layers of container image.
This commit also changes the behaviour of FileSource when 'qcow2'
output format is used. Now the string layer-0.qcow2 will be used
as name of the output file.
This change is appied in the test suite as an update to the function
get_image_path().
Note: The following commits which introduce UID/GID mapping
and setting root password for 'qcow2' with FileSource will apply
these changes to layer-1.qcow2 image with backing file layer-0.qcow2.
---
src/virtBootstrap/sources/docker_source.py | 10 +-
src/virtBootstrap/sources/file_source.py | 14 +--
src/virtBootstrap/utils.py | 146 +++++++++++++++++------------
tests/file_source.py | 8 +-
4 files changed, 106 insertions(+), 72 deletions(-)
diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py
index 58e30ce..99247ab 100644
--- a/src/virtBootstrap/sources/docker_source.py
+++ b/src/virtBootstrap/sources/docker_source.py
@@ -275,7 +275,15 @@ class DockerSource(object):
elif self.output_format == 'qcow2':
self.progress("Extracting container layers into qcow2 images",
value=50, logger=logger)
- utils.extract_layers_in_qcow2(self.layers, dest, self.progress)
+
+ img = utils.BuildImage(
+ layers=self.layers,
+ dest=dest,
+ progress=self.progress
+ )
+ img.create_base_layer()
+ img.create_backing_chains()
+
else:
raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/sources/file_source.py b/src/virtBootstrap/sources/file_source.py
index 412db8a..69f024c 100644
--- a/src/virtBootstrap/sources/file_source.py
+++ b/src/virtBootstrap/sources/file_source.py
@@ -64,14 +64,16 @@ class FileSource(object):
utils.untar_layers(layer, dest, self.progress)
elif self.output_format == 'qcow2':
- # Remove the old path
- file_name = os.path.basename(self.path)
- qcow2_file = os.path.realpath('{}/{}.qcow2'.format(dest,
- file_name))
-
self.progress("Extracting files into qcow2 image", value=0,
logger=logger)
- utils.create_qcow2(self.path, qcow2_file)
+
+ img = utils.BuildImage(
+ layers=layer,
+ dest=dest,
+ progress=self.progress
+ )
+ img.create_base_layer()
+
else:
raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py
index 77177a3..482e4aa 100644
--- a/src/virtBootstrap/utils.py
+++ b/src/virtBootstrap/utils.py
@@ -33,6 +33,7 @@ import tempfile
import logging
import re
+import guestfs
import passlib.hosts
# pylint: disable=invalid-name
@@ -42,6 +43,8 @@ logger = logging.getLogger(__name__)
DEFAULT_OUTPUT_FORMAT = 'dir'
# Default virtual size of qcow2 image
DEF_QCOW2_SIZE = '5G'
+DEF_BASE_IMAGE_SIZE = 5 * 1024 * 1024 * 1024
+
if os.geteuid() == 0:
LIBVIRT_CONN = "lxc:///"
DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images"
@@ -51,6 +54,88 @@ else:
DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images"
+class BuildImage(object):
+ """
+ Use guestfs-python to create qcow2 disk images.
+ """
+
+ def __init__(self, layers, dest, progress):
+ """
+ @param tar_files: Tarballs to be converted to qcow2 images
+ @param dest: Directory where the qcow2 images will be created
+ @param progress: Instance of the progress module
+ """
+ self.g = guestfs.GuestFS(python_return_dict=True)
+ self.layers = layers
+ self.nlayers = len(layers)
+ self.dest = dest
+ self.progress = progress
+ self.qcow2_files = []
+
+ def create_base_layer(self, fmt='qcow2', size=DEF_BASE_IMAGE_SIZE):
+ """
+ Create and format qcow2 disk image which represnts the base layer.
+ """
+ self.qcow2_files = [os.path.join(self.dest, 'layer-0.qcow2')]
+ self.progress("Creating base layer", logger=logger)
+ self.g.disk_create(self.qcow2_files[0], fmt, size)
+ self.g.add_drive(self.qcow2_files[0], format=fmt)
+ self.g.launch()
+ self.progress("Formating disk image", logger=logger)
+ self.g.mkfs("ext3", '/dev/sda')
+ self.extract_layer(0, '/dev/sda')
+ # Shutdown qemu instance to avoid hot-plugging of devices.
+ self.g.shutdown()
+
+ def create_backing_chains(self):
+ """
+ Convert other layers to qcow2 images linked as backing chains.
+ """
+ for i in range(1, self.nlayers):
+ self.qcow2_files.append(
+ os.path.join(self.dest, 'layer-%d.qcow2' % i)
+ )
+ self.progress(
+ "Creating image (%d/%d)" % (i + 1, self.nlayers),
+ logger=logger
+ )
+ self.g.disk_create(
+ filename=self.qcow2_files[i],
+ format='qcow2',
+ size=-1,
+ backingfile=self.qcow2_files[i - 1],
+ backingformat='qcow2'
+ )
+ self.g.add_drive(self.qcow2_files[i], format='qcow2')
+ self.g.launch()
+ devices = self.g.list_devices()
+ # Tar-in layers (skip the base layer)
+ for index in range(1, self.nlayers):
+ self.extract_layer(index, devices[index - 1])
+ self.g.shutdown()
+
+ def extract_layer(self, index, dev):
+ """
+ Extract tarball of layer to device
+ """
+ tar_file, tar_size = self.layers[index]
+ log_layer_extract(
+ tar_file, tar_size, index + 1, self.nlayers, self.progress
+ )
+ self.tar_in(dev, tar_file)
+
+ def tar_in(self, dev, tar_file):
+ """
+ Common pattern used to tar-in archive into image
+ """
+ self.g.mount(dev, '/')
+ # Restore extended attributes, SELinux contexts and POSIX ACLs
+ # from tar file.
+ self.g.tar_in(tar_file, '/', get_compression_type(tar_file),
+ xattrs=True, selinux=True, acls=True)
+ self.g.umount('/')
+
+
def get_compression_type(tar_file):
"""
Get compression type of tar file.
@@ -209,67 +294,6 @@ def get_mime_type(path):
return output.read().decode('utf-8').split()[1]
-def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
- """
- Create qcow2 image from tarball.
- """
- qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]
-
- if not backing_file:
- logger.info("Creating base qcow2 image")
- execute(qemu_img_cmd)
-
- logger.info("Formatting qcow2 image")
- execute(['virt-format',
- '--format=qcow2',
- '--partition=none',
- '--filesystem=ext3',
- '-a', layer_file])
- else:
- # Add backing chain
- qemu_img_cmd.insert(2, "-b")
- qemu_img_cmd.insert(3, backing_file)
-
- logger.info("Creating qcow2 image with backing chain")
- execute(qemu_img_cmd)
-
- # Extract tarball using "tar-in" command from libguestfs
- tar_in_cmd = ["guestfish",
- "-a", layer_file,
- '-m', '/dev/sda',
- 'tar-in', tar_file, "/"]
-
- # Check if tarball is compressed
- compression = get_compression_type(tar_file)
- if compression is not None:
- tar_in_cmd.append('compress:' + compression)
-
- # Execute virt-tar-in command
- execute(tar_in_cmd)
-
-
-def extract_layers_in_qcow2(layers_list, dest_dir, progress):
- """
- Extract docker layers in qcow2 images with backing chains.
- """
- qcow2_backing_file = None
-
- nlayers = len(layers_list)
- for index, layer in enumerate(layers_list):
- tar_file, tar_size = layer
- log_layer_extract(tar_file, tar_size, index + 1, nlayers, progress)
-
- # Name format for the qcow2 image
- qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
- # Create the image layer
- create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
- # Keep the file path for the next layer
- qcow2_backing_file = qcow2_layer_file
-
- # Update progress value
- progress(value=(float(index + 1) / nlayers * 50) + 50)
-
-
def get_image_dir(no_cache=False):
"""
Get the directory where image layers are stored.
diff --git a/tests/file_source.py b/tests/file_source.py
index dd7fd00..2a0f964 100644
--- a/tests/file_source.py
+++ b/tests/file_source.py
@@ -50,13 +50,13 @@ class Qcow2BuildImage(BuildTarFiles):
g.umount('/')
g.shutdown()
- def get_image_path(self):
+ def get_image_path(self, n=0):
"""
Returns the path where the qcow2 image will be stored.
"""
return os.path.join(
self.dest_dir,
- "%s.qcow2" % os.path.basename(self.tar_file)
+ "layer-%d.qcow2" % n
)
def runTest(self):
@@ -92,7 +92,7 @@ class Qcow2OwnershipMapping(Qcow2BuildImage):
gid_map=self.gid_map
)
self.apply_mapping()
- self.check_qcow2_images(self.get_image_path())
+ self.check_qcow2_images(self.get_image_path(1))
@unittest.skip("Not implemented")
@@ -114,7 +114,7 @@ class Qcow2SettingRootPassword(Qcow2BuildImage):
root_password=self.root_password
)
self.check_image = self.validate_shadow_file_in_image
- self.check_qcow2_images(self.get_image_path())
+ self.check_qcow2_images(self.get_image_path(1))
@unittest.skipIf(os.geteuid() != 0, "Root privileges required")
--
2.13.5
More information about the virt-tools-list
mailing list