[virt-tools-list] [virt-bootstrap] [PATCH v7 14/26] Create qcow2 images with guestfs-python
Radostin Stoyanov
rstoyanov1 at gmail.com
Sat Aug 26 20:42:03 UTC 2017
Use the python bindings of libguestfs to create qcow2 image with
backing chains to mimic the layers of container image.
This commit also changes the behavior of FileSource when 'qcow2'
output format is used. Now the string layer-0.qcow2 will be used
as name of the output file.
This change is applied in the test suite as an update to the function
get_image_path().
---
src/virtBootstrap/sources/docker_source.py | 10 +-
src/virtBootstrap/sources/file_source.py | 14 +--
src/virtBootstrap/utils.py | 146 +++++++++++++++++------------
tests/file_source.py | 29 ++++++
4 files changed, 131 insertions(+), 68 deletions(-)
diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py
index 2dadb42..a6ea3e6 100644
--- a/src/virtBootstrap/sources/docker_source.py
+++ b/src/virtBootstrap/sources/docker_source.py
@@ -272,7 +272,15 @@ class DockerSource(object):
elif self.output_format == 'qcow2':
self.progress("Extracting container layers into qcow2 images",
value=50, logger=logger)
- utils.extract_layers_in_qcow2(self.layers, dest, self.progress)
+
+ img = utils.BuildImage(
+ layers=self.layers,
+ dest=dest,
+ progress=self.progress
+ )
+ img.create_base_layer()
+ img.create_backing_chains()
+
else:
raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/sources/file_source.py b/src/virtBootstrap/sources/file_source.py
index 412db8a..69f024c 100644
--- a/src/virtBootstrap/sources/file_source.py
+++ b/src/virtBootstrap/sources/file_source.py
@@ -64,14 +64,16 @@ class FileSource(object):
utils.untar_layers(layer, dest, self.progress)
elif self.output_format == 'qcow2':
- # Remove the old path
- file_name = os.path.basename(self.path)
- qcow2_file = os.path.realpath('{}/{}.qcow2'.format(dest,
- file_name))
-
self.progress("Extracting files into qcow2 image", value=0,
logger=logger)
- utils.create_qcow2(self.path, qcow2_file)
+
+ img = utils.BuildImage(
+ layers=layer,
+ dest=dest,
+ progress=self.progress
+ )
+ img.create_base_layer()
+
else:
raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py
index be9133c..5fb5d8c 100644
--- a/src/virtBootstrap/utils.py
+++ b/src/virtBootstrap/utils.py
@@ -33,6 +33,7 @@ import tempfile
import logging
import re
+import guestfs
import passlib.hosts
# pylint: disable=invalid-name
@@ -42,6 +43,8 @@ logger = logging.getLogger(__name__)
DEFAULT_OUTPUT_FORMAT = 'dir'
# Default virtual size of qcow2 image
DEF_QCOW2_SIZE = '5G'
+DEF_BASE_IMAGE_SIZE = 5 * 1024 * 1024 * 1024
+
if os.geteuid() == 0:
LIBVIRT_CONN = "lxc:///"
DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images"
@@ -51,6 +54,88 @@ else:
DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images"
+class BuildImage(object):
+ """
+ Use guestfs-python to create qcow2 disk images.
+ """
+
+ def __init__(self, layers, dest, progress):
+ """
+ @param tar_files: Tarballs to be converted to qcow2 images
+ @param dest: Directory where the qcow2 images will be created
+ @param progress: Instance of the progress module
+ """
+ self.g = guestfs.GuestFS(python_return_dict=True)
+ self.layers = layers
+ self.nlayers = len(layers)
+ self.dest = dest
+ self.progress = progress
+ self.qcow2_files = []
+
+ def create_base_layer(self, fmt='qcow2', size=DEF_BASE_IMAGE_SIZE):
+ """
+ Create and format qcow2 disk image which represnts the base layer.
+ """
+ self.qcow2_files = [os.path.join(self.dest, 'layer-0.qcow2')]
+ self.progress("Creating base layer", logger=logger)
+ self.g.disk_create(self.qcow2_files[0], fmt, size)
+ self.g.add_drive(self.qcow2_files[0], format=fmt)
+ self.g.launch()
+ self.progress("Formating disk image", logger=logger)
+ self.g.mkfs("ext3", '/dev/sda')
+ self.extract_layer(0, '/dev/sda')
+ # Shutdown qemu instance to avoid hot-plugging of devices.
+ self.g.shutdown()
+
+ def create_backing_chains(self):
+ """
+ Convert other layers to qcow2 images linked as backing chains.
+ """
+ for i in range(1, self.nlayers):
+ self.qcow2_files.append(
+ os.path.join(self.dest, 'layer-%d.qcow2' % i)
+ )
+ self.progress(
+ "Creating image (%d/%d)" % (i + 1, self.nlayers),
+ logger=logger
+ )
+ self.g.disk_create(
+ filename=self.qcow2_files[i],
+ format='qcow2',
+ size=-1,
+ backingfile=self.qcow2_files[i - 1],
+ backingformat='qcow2'
+ )
+ self.g.add_drive(self.qcow2_files[i], format='qcow2')
+ self.g.launch()
+ devices = self.g.list_devices()
+ # Tar-in layers (skip the base layer)
+ for index in range(1, self.nlayers):
+ self.extract_layer(index, devices[index - 1])
+ self.g.shutdown()
+
+ def extract_layer(self, index, dev):
+ """
+ Extract tarball of layer to device
+ """
+ tar_file, tar_size = self.layers[index]
+ log_layer_extract(
+ tar_file, tar_size, index + 1, self.nlayers, self.progress
+ )
+ self.tar_in(dev, tar_file)
+
+ def tar_in(self, dev, tar_file):
+ """
+ Common pattern used to tar-in archive into image
+ """
+ self.g.mount(dev, '/')
+ # Restore extended attributes, SELinux contexts and POSIX ACLs
+ # from tar file.
+ self.g.tar_in(tar_file, '/', get_compression_type(tar_file),
+ xattrs=True, selinux=True, acls=True)
+ self.g.umount('/')
+
+
def get_compression_type(tar_file):
"""
Get compression type of tar file.
@@ -212,67 +297,6 @@ def get_mime_type(path):
return output.read().decode('utf-8').split()[1]
-def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
- """
- Create qcow2 image from tarball.
- """
- qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]
-
- if not backing_file:
- logger.info("Creating base qcow2 image")
- execute(qemu_img_cmd)
-
- logger.info("Formatting qcow2 image")
- execute(['virt-format',
- '--format=qcow2',
- '--partition=none',
- '--filesystem=ext3',
- '-a', layer_file])
- else:
- # Add backing chain
- qemu_img_cmd.insert(2, "-b")
- qemu_img_cmd.insert(3, backing_file)
-
- logger.info("Creating qcow2 image with backing chain")
- execute(qemu_img_cmd)
-
- # Extract tarball using "tar-in" command from libguestfs
- tar_in_cmd = ["guestfish",
- "-a", layer_file,
- '-m', '/dev/sda',
- 'tar-in', tar_file, "/"]
-
- # Check if tarball is compressed
- compression = get_compression_type(tar_file)
- if compression is not None:
- tar_in_cmd.append('compress:' + compression)
-
- # Execute virt-tar-in command
- execute(tar_in_cmd)
-
-
-def extract_layers_in_qcow2(layers_list, dest_dir, progress):
- """
- Extract docker layers in qcow2 images with backing chains.
- """
- qcow2_backing_file = None
-
- nlayers = len(layers_list)
- for index, layer in enumerate(layers_list):
- tar_file, tar_size = layer
- log_layer_extract(tar_file, tar_size, index + 1, nlayers, progress)
-
- # Name format for the qcow2 image
- qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
- # Create the image layer
- create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
- # Keep the file path for the next layer
- qcow2_backing_file = qcow2_layer_file
-
- # Update progress value
- progress(value=(float(index + 1) / nlayers * 50) + 50)
-
-
def get_image_dir(no_cache=False):
"""
Get the directory where image layers are stored.
diff --git a/tests/file_source.py b/tests/file_source.py
index 79bb234..391ca48 100644
--- a/tests/file_source.py
+++ b/tests/file_source.py
@@ -27,6 +27,7 @@ import unittest
from . import mock
from . import virt_bootstrap
from . import ImageAccessor
+from . import Qcow2ImageAccessor
from . import NOT_ROOT
@@ -76,3 +77,31 @@ class TestDirFileSource(ImageAccessor):
self.root_password = 'my secret root password'
self.call_bootstrap()
self.validate_shadow_file()
+
+
+class TestQcow2FileSource(Qcow2ImageAccessor):
+ """
+ Test cases for the class FileSource used with qcow2 output format.
+ """
+
+ def call_bootstrap(self):
+ """
+ Execute the bootstrap method from virtBootstrap.
+ """
+ virt_bootstrap.bootstrap(
+ uri=self.tar_file,
+ dest=self.dest_dir,
+ fmt='qcow2',
+ progress_cb=mock.Mock(),
+ uid_map=self.uid_map,
+ gid_map=self.gid_map,
+ root_password=self.root_password
+ )
+
+ def test_qcow2_extract_rootfs(self):
+ """
+ Ensures root file system of tar archive is converted to single
+ partition qcow2 image.
+ """
+ self.call_bootstrap()
+ self.check_qcow2_images(self.get_image_path())
--
2.13.5
More information about the virt-tools-list
mailing list