[virt-tools-list] [virt-bootstrap] [PATCH v6 17/26] Add virt-builder source
Cedric Bosdonnat
cbosdonnat at suse.com
Sat Aug 19 13:19:51 UTC 2017
On Thu, 2017-08-17 at 10:39 +0100, Radostin Stoyanov wrote:
> Add implementation for virt-builder source which aims to create
> container root file system from VM image build with virt-builder.
'build' -> 'built'
>
> Usage examples:
> virt-bootstrap virt-builder://fedora-25 /tmp/foo
> virt-bootstrap virt-builder://ubuntu-16.04 /tmp/bar --root-password secret
> virt-bootstrap virt-builder://fedora-25 /tmp/foo -f qcow2 --idmap 0:1000:10
> sudo virt-bootstrap virt-builder://fedora-25 /tmp/foo --idmap 0:1000:10
>
> Tests are also introduced along with the implementation. They cover
> creation of root file system and UID/GID mapping for 'dir' and 'qcow2'
> output format by mocking the build_image() method to avoid the time
> consuming call to virt-builder which might also require network
> connection with function which creates dummy disk image.
> Setting root password is handled by virt-builder and hence the
> introduced test only ensures that the password string is passed
> correctly.
Testing a virt-builder source without any call to virt-builder
sounds fishy. I'ld rather not see virt-builder mocked up for real tests.
> ---
> src/virtBootstrap/sources/__init__.py | 1 +
> src/virtBootstrap/sources/virt_builder_source.py | 148 +++++++++++++++
> src/virtBootstrap/virt_bootstrap.py | 4 +-
> tests/__init__.py | 23 +++
> tests/file_source.py | 28 +--
> tests/virt_builder_source.py | 228 +++++++++++++++++++++++
> 6 files changed, 406 insertions(+), 26 deletions(-)
> create mode 100644 src/virtBootstrap/sources/virt_builder_source.py
> create mode 100644 tests/virt_builder_source.py
>
> diff --git a/src/virtBootstrap/sources/__init__.py b/src/virtBootstrap/sources/__init__.py
> index e891e9b..be6b25c 100644
> --- a/src/virtBootstrap/sources/__init__.py
> +++ b/src/virtBootstrap/sources/__init__.py
> @@ -24,3 +24,4 @@ sources - Class definitions which process container image or
>
> from virtBootstrap.sources.file_source import FileSource
> from virtBootstrap.sources.docker_source import DockerSource
> +from virtBootstrap.sources.virt_builder_source import VirtBuilderSource
> diff --git a/src/virtBootstrap/sources/virt_builder_source.py b/src/virtBootstrap/sources/virt_builder_source.py
> new file mode 100644
> index 0000000..780ffb1
> --- /dev/null
> +++ b/src/virtBootstrap/sources/virt_builder_source.py
> @@ -0,0 +1,148 @@
> +# -*- coding: utf-8 -*-
> +# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
> +#
> +# Copyright (C) 2017 Radostin Stoyanov
> +#
> +# This program is free software: you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation, either version 3 of the License, or
> +# (at your option) any later version.
> +
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +
> +# You should have received a copy of the GNU General Public License
> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> +
> +"""
> +VirtBuilderSource aim is to extract the root file system from VM image
> +build with virt-builder from template.
> +"""
> +
> +import os
> +import logging
> +import subprocess
> +import tempfile
> +
> +import guestfs
> +from virtBootstrap import utils
> +
> +
> +# pylint: disable=invalid-name
> +# Create logger
> +logger = logging.getLogger(__name__)
> +
> +
> +class VirtBuilderSource(object):
> + """
> + Extract root file system from image build with virt-builder.
> + """
> + def __init__(self, **kwargs):
> + """
> + Create container rootfs by building VM from virt-builder template
> + and extract the rootfs.
> +
> + @param uri: Template name
> + @param fmt: Format used to store the output [dir, qcow2]
> + @param uid_map: Mappings for UID of files in rootfs
> + @param gid_map: Mappings for GID of files in rootfs
> + @param root_password: Root password to set in rootfs
> + @param progress: Instance of the progress module
> + """
> + # Parsed URIs:
> + # - "virt-builder:///<template>"
> + # - "virt-builder://<template>"
> + # - "virt-builder:/<template>"
> + self.template = kwargs['uri'].netloc or kwargs['uri'].path[1:]
> + self.output_format = kwargs.get('fmt', utils.DEFAULT_OUTPUT_FORMAT)
> + self.uid_map = kwargs.get('uid_map', [])
> + self.gid_map = kwargs.get('gid_map', [])
> + self.root_password = kwargs.get('root_password', None)
> + self.progress = kwargs['progress'].update_progress
> +
> + def build_image(self, output_file):
> + """
> + Build VM from virt-builder template
> + """
> + cmd = ['virt-builder', self.template,
We could make this more configurable for testability. For example,
we could add a self._builder_cmd member that contains ['virt-builder']
for normal case, but the test case could add some '--source', '/path/to/test/source'.
That would have us test against virt-builder but get rid of the download
length and network requirement.
> + '-o', output_file,
> + '--no-network',
> + '--delete', '/dev/*',
> + '--delete', '/boot/*',
> + # Comment out all lines in fstab
> + '--edit', '/etc/fstab:s/^/#/']
> + if self.root_password is not None:
> + cmd += ['--root-password', "password:%s" % self.root_password]
> + subprocess.check_call(cmd)
> +
> + def unpack(self, dest):
> + """
> + Build image and extract root file system
> +
> + @param dest: Directory path where output files will be stored.
> + """
> +
> + with tempfile.NamedTemporaryFile(prefix='bootstrap_') as tmp_file:
> + if self.output_format == 'dir':
> + self.progress("Building image", value=0, logger=logger)
> + self.build_image(tmp_file.name)
> + self.progress("Extracting rootfs", value=50, logger=logger)
> + g = guestfs.GuestFS(python_return_dict=True)
> + g.add_drive_opts(tmp_file.name, readonly=False, format='raw')
> + g.launch()
> +
> + # Get the device with file system
> + root_dev = g.inspect_os()
> + if not root_dev:
> + raise Exception("No file system was found")
> + g.mount(root_dev[0], '/')
> +
> + # Extract file system to destination directory
> + g.copy_out('/', dest)
> +
> + g.umount('/')
> + g.shutdown()
> +
> + self.progress("Extraction completed successfully!",
> + value=100, logger=logger)
> + logger.info("Files are stored in: %s", dest)
> +
> + elif self.output_format == 'qcow2':
> + output_file = os.path.join(dest, 'layer-0.qcow2')
> +
> + self.progress("Building image", value=0, logger=logger)
> + self.build_image(tmp_file.name)
> +
> + self.progress("Extracting rootfs", value=50, logger=logger)
> + g = guestfs.GuestFS(python_return_dict=True)
> + g.add_drive_opts(tmp_file.name, readonly=True, format='raw')
> + # Create qcow2 disk image
> + g.disk_create(
> + filename=output_file,
> + format='qcow2',
> + size=os.path.getsize(tmp_file.name)
> + )
> + g.add_drive_opts(output_file, readonly=False, format='qcow2')
> + g.launch()
> + # Get the device with file system
> + root_dev = g.inspect_os()
> + if not root_dev:
> + raise Exception("No file system was found")
> + output_dev = g.list_devices()[1]
> + # Copy the file system to the new qcow2 disk
> + g.copy_device_to_device(root_dev[0], output_dev, sparse=True)
> + g.shutdown()
> +
> + # UID/GID mapping
> + if self.uid_map or self.gid_map:
> + logger.info("Mapping UID/GID")
> + utils.map_id_in_image(1, dest, self.uid_map, self.gid_map)
> +
> + self.progress("Extraction completed successfully!", value=100,
> + logger=logger)
> + logger.info("Image is stored in: %s", output_file)
> +
> + else:
> + raise Exception("Unknown format:" + self.output_format)
> diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py
> index e387842..68e6516 100755
> --- a/src/virtBootstrap/virt_bootstrap.py
> +++ b/src/virtBootstrap/virt_bootstrap.py
> @@ -62,7 +62,7 @@ def get_source(source_type):
> Get object which match the source type
> """
> try:
> - class_name = "%sSource" % source_type.capitalize()
> + class_name = "%sSource" % source_type.title().replace('-', '')
> clazz = getattr(sources, class_name)
> return clazz
> except Exception:
> @@ -179,6 +179,8 @@ def main():
> docker://docker.io/fedora
> docker://privateregistry:5000/image
> file:///path/to/local/rootfs.tar.xz
> + virt-builder://fedora-25
> + virt-builder://ubuntu-16.04
> ----------------------------------------
>
> ''')))
> diff --git a/tests/__init__.py b/tests/__init__.py
> index 7a53c38..8888a0d 100644
> --- a/tests/__init__.py
> +++ b/tests/__init__.py
> @@ -29,6 +29,8 @@ import tarfile
> import unittest
> import passlib.hosts
>
> +import guestfs
> +
> try:
> import mock
> except ImportError:
> @@ -434,3 +436,24 @@ class BuildTarFiles(unittest.TestCase):
> self.check_image_content(g, user, 'dirs', g.is_dir)
> # Check files
> self.check_image_content(g, user, 'files', g.is_file)
> +
> + def check_qcow2_image(self, image_path):
> + """
> + Ensures that qcow2 images contain all files.
> + """
> + g = guestfs.GuestFS(python_return_dict=True)
> + g.add_drive_opts(image_path, readonly=True)
> + g.launch()
> + g.mount('/dev/sda', '/')
> + self.check_image(g)
> + g.umount('/')
> + g.shutdown()
> +
> + def get_image_path(self, n=0):
> + """
> + Returns the path where the qcow2 image will be stored.
> + """
> + return os.path.join(
> + self.dest_dir,
> + "layer-%d.qcow2" % n
> + )
> diff --git a/tests/file_source.py b/tests/file_source.py
> index 8851c3f..895ba9e 100644
> --- a/tests/file_source.py
> +++ b/tests/file_source.py
> @@ -24,7 +24,6 @@ with FileSource.
>
> import os
> import unittest
> -import guestfs
>
> from . import virt_bootstrap
> from . import BuildTarFiles
> @@ -38,27 +37,6 @@ class Qcow2BuildImage(BuildTarFiles):
> works as expected.
> """
>
> - def check_qcow2_images(self, image_path):
> - """
> - Ensures that qcow2 images contain all files.
> - """
> - g = guestfs.GuestFS(python_return_dict=True)
> - g.add_drive_opts(image_path, readonly=True)
> - g.launch()
> - g.mount('/dev/sda', '/')
> - self.check_image(g)
> - g.umount('/')
> - g.shutdown()
> -
> - def get_image_path(self, n=0):
> - """
> - Returns the path where the qcow2 image will be stored.
> - """
> - return os.path.join(
> - self.dest_dir,
> - "layer-%d.qcow2" % n
> - )
> -
> def runTest(self):
> """
> Create qcow2 image from each dummy tarfile.
> @@ -69,7 +47,7 @@ class Qcow2BuildImage(BuildTarFiles):
> fmt='qcow2',
> progress_cb=mock.Mock()
> )
> - self.check_qcow2_images(self.get_image_path())
> + self.check_qcow2_image(self.get_image_path())
>
>
> class Qcow2OwnershipMapping(Qcow2BuildImage):
> @@ -91,7 +69,7 @@ class Qcow2OwnershipMapping(Qcow2BuildImage):
> gid_map=self.gid_map
> )
> self.apply_mapping()
> - self.check_qcow2_images(self.get_image_path(1))
> + self.check_qcow2_image(self.get_image_path(1))
>
>
> class Qcow2SettingRootPassword(Qcow2BuildImage):
> @@ -112,7 +90,7 @@ class Qcow2SettingRootPassword(Qcow2BuildImage):
> root_password=self.root_password
> )
> self.check_image = self.validate_shadow_file_in_image
> - self.check_qcow2_images(self.get_image_path(1))
> + self.check_qcow2_image(self.get_image_path(1))
>
>
> @unittest.skipIf(os.geteuid() != 0, "Root privileges required")
> diff --git a/tests/virt_builder_source.py b/tests/virt_builder_source.py
> new file mode 100644
> index 0000000..4fe7713
> --- /dev/null
> +++ b/tests/virt_builder_source.py
> @@ -0,0 +1,228 @@
> +# -*- coding: utf-8 -*-
> +# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
> +#
> +# Copyright (C) 2017 Radostin Stoyanov
> +#
> +# This program is free software: you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation, either version 3 of the License, or
> +# (at your option) any later version.
> +
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +
> +# You should have received a copy of the GNU General Public License
> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> +
> +
> +"""
> +Regression tests which aim to excercise the creation of root file system
typo: 'exercise'
> +with VirtBuilderSource.
> +"""
> +
> +import os
> +import unittest
> +
> +import guestfs
> +
> +from . import BuildTarFiles
> +from . import virt_bootstrap
> +from . import mock
> +
> +
> +# pylint: disable=invalid-name
> +class DirExtractRootFS(BuildTarFiles):
> + """
> + This test is replacing the method build_image() from VirtBuilderSource
> + with a function which generates gummy disk image.
> +
> + It ensures that all files and directories from the created root file
> + systems is extracted correctly.
> + """
> + def create_tar_files(self):
> + """
> + Don't need to build tar files for these tests.
> + """
> + pass
> +
> + def create_dummy_disk(self, output_file):
> + """
> + Create raw disk image with dummy root file system
> + """
> + g = guestfs.GuestFS(python_return_dict=True)
> + g.disk_create(output_file, format='raw', size=(1 * 1024 * 1024))
> + g.add_drive(output_file, readonly=False, format='raw')
> + g.launch()
> + g.mkfs('ext3', '/dev/sda')
> + g.mount('/dev/sda', '/')
> + for user in self.rootfs_tree:
> +
> + usr_uid = self.rootfs_tree[user]['uid']
> + usr_gid = self.rootfs_tree[user]['gid']
> + usr_dirs = self.rootfs_tree[user]['dirs']
> + usr_files = self.rootfs_tree[user]['files']
> +
> + for member in usr_dirs:
> + dir_name = '/' + member
> + g.mkdir_p(dir_name)
> + g.chown(usr_uid, usr_gid, dir_name)
> +
> + for member in usr_files:
> + if isinstance(member, tuple):
> + file_name = '/' + member[0]
> + g.write(file_name, member[2])
> + g.chmod(member[1] & 0o777, file_name)
> + else:
> + file_name = '/' + member
> + g.touch(file_name)
> + g.chmod(0o755 & 0o777, file_name)
> + g.chown(usr_uid, usr_gid, file_name)
> +
> + def runTest(self):
> + """
> + Mock the function build_image() and call bootstrap().
> + Then check the extracted root file system.
> + """
> + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
> + with mock.patch(build_image) as m_build_image:
> + m_build_image.side_effect = self.create_dummy_disk
> + virt_bootstrap.bootstrap(
> + uri='virt-builder://foobar',
> + dest=self.dest_dir,
> + fmt='dir',
> + progress_cb=mock.Mock()
> + )
> + self.check_rootfs(skip_ownership=(os.geteuid() != 0))
> +
The test would thus be:
* Create dummy disk and minimal source index file (no need to gpg sign it)
* something along this: virtBootstrap.sources.VirtBuilderSource._builder_args.append += ['--source', index_path]
* virt_bootstrap.bootstrap(...)
Same applies for the other tests.
--
Cedric
> +
> + at unittest.skipIf(os.geteuid() != 0, "Root privileges required")
> +class DirOwnershipMapping(DirExtractRootFS):
> + """
> + Ensures that UID/GID mapping for extracted root file system are applied
> + correctly.
> + """
> + def runTest(self):
> + """
> + Mock the function build_image() and call bootstrap() with uid/gid
> + mapping values.
> + Then check the ownership of the extracted root file system.
> + """
> + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
> + with mock.patch(build_image) as m_build_image:
> + m_build_image.side_effect = self.create_dummy_disk
> + virt_bootstrap.bootstrap(
> + progress_cb=mock.Mock(),
> + uri='virt-builder://foobar',
> + dest=self.dest_dir,
> + fmt='dir',
> + uid_map=self.uid_map,
> + gid_map=self.gid_map
> + )
> + self.apply_mapping()
> + self.check_rootfs(skip_ownership=(os.geteuid() != 0))
> +
> +
> +class DirSettingRootPassword(DirExtractRootFS):
> + """
> + The root password is set by virt-builder in this test we do not call
> + virt-builder as this is time consuming job and might require network
> + connection to download the VM template.
> +
> + Instead we only check if the root password is passed to virt-builder
> + and if virt-bootstrap extracts the shadow file correctly.
> + """
> + def verify_virt_builder_cmd(self, cmd):
> + """
> + Ensures that virt-builder is called with valid command and the root
> + password is passed.
> + """
> + self.assertEqual(
> + 'virt-builder',
> + cmd[0],
> + "virt-builder command does not start with 'virt-builder'"
> + )
> + self.assertIn(
> + '--root-password',
> + cmd,
> + "The flag '--root-password' is missing in virt-builder command"
> + )
> + self.assertEqual(
> + 'password:%s' % self.root_password,
> + cmd[cmd.index('--root-password') + 1],
> + "Root password doesn't match"
> + )
> + self.create_dummy_disk(cmd[cmd.index('-o') + 1])
> +
> + def runTest(self):
> + """
> + Mock the function subprocess.check_call() and call bootstrap().
> + Then check the extracted root file system.
> + """
> + self.root_password = "Root password"
> + with mock.patch('subprocess.check_call') as m_check_call:
> + m_check_call.side_effect = self.verify_virt_builder_cmd
> + virt_bootstrap.bootstrap(
> + uri='virt-builder://foobar',
> + dest=self.dest_dir,
> + fmt='dir',
> + root_password=self.root_password,
> + progress_cb=mock.Mock()
> + )
> + m_check_call.assert_called_once()
> + self.validate_shadow_file(
> + os.path.join(self.dest_dir, 'etc/shadow'),
> + skip_ownership=(os.geteuid() != 0),
> + skip_hash=True
> + )
> +
> +
> +class Qcow2BuildImage(DirExtractRootFS):
> + """
> + Ensures that the file system is copied correctly to the output qcow2 image.
> + """
> + def runTest(self):
> + """
> + Mock the function build_image() and call bootstrap().
> + Then check the content of the new image.
> + """
> + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
> + with mock.patch(build_image) as m_build_image:
> + m_build_image.side_effect = self.create_dummy_disk
> + virt_bootstrap.bootstrap(
> + progress_cb=mock.Mock(),
> + uri='virt-builder://foobar',
> + dest=self.dest_dir,
> + fmt='qcow2'
> + )
> + self.check_qcow2_image(self.get_image_path())
> +
> +
> +class Qcow2OwnershipMapping(DirExtractRootFS):
> + """
> + Ensures that UID/GID mapping is applied correctly with qcow2 conversion.
> + """
> + def runTest(self):
> + """
> + Mock the function build_image() and call bootstrap() with uid/gid
> + mapping values.
> + Then check the ownership of the extracted root file system.
> + """
> + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
> + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image'
> + with mock.patch(build_image) as m_build_image:
> + m_build_image.side_effect = self.create_dummy_disk
> + virt_bootstrap.bootstrap(
> + uri='virt-builder://foobar',
> + dest=self.dest_dir,
> + fmt='qcow2',
> + uid_map=self.uid_map,
> + gid_map=self.gid_map,
> + progress_cb=mock.Mock()
> + )
> + self.apply_mapping()
> + self.check_qcow2_image(self.get_image_path(1))
More information about the virt-tools-list
mailing list