Changeset
configure.ac             |   1 +
m4/virt-criu.m4          |  27 +++++
po/POTFILES.in           |   1 +
src/lxc/Makefile.inc.am  |   4 +
src/lxc/lxc_container.c  | 162 ++++++++++++++++++++++++++++--
src/lxc/lxc_container.h  |   3 +-
src/lxc/lxc_controller.c | 104 ++++++++++++++++++-
src/lxc/lxc_criu.c       | 253 +++++++++++++++++++++++++++++++++++++++++++++++
src/lxc/lxc_criu.h       |  36 +++++++
src/lxc/lxc_driver.c     | 238 +++++++++++++++++++++++++++++++++++++++++++-
src/lxc/lxc_process.c    |  23 ++++-
src/lxc/lxc_process.h    |   1 +
12 files changed, 836 insertions(+), 17 deletions(-)
create mode 100644 m4/virt-criu.m4
create mode 100644 src/lxc/lxc_criu.c
create mode 100644 src/lxc/lxc_criu.h
Git apply log
Switched to a new branch '20180411112915.5890-1-rstoyanov1@gmail.com'
Applying: configure: Include support for CRIU
Applying: lxc: Add save/restore helper functions
Applying: lxc: Add restore mode for libvirt-lxc
Applying: lxc: Add save/restore support
To https://github.com/patchew-project/libvirt
 * [new tag]         patchew/20180411112915.5890-1-rstoyanov1@gmail.com -> patchew/20180411112915.5890-1-rstoyanov1@gmail.com
Test passed: syntax-check

loading

[libvirt] [RFC PATCH 0/4] LXC - Implement save/restore domain state
Posted by Radostin Stoyanov, 5 weeks ago
This patch set contains rebased version of Katerina's work from GSoC 2016 [1].
It allows integrates CRIU [2] with the libvirt-lxc to enable save/resore of containers.

[1] https://wiki.libvirt.org/page/Google_Summer_of_Code_2016/lxc_migration
[2] https://criu.org

Radostin Stoyanov (4):
  configure: Include support for CRIU
  lxc: Add save/restore helper functions
  lxc: Add restore mode for libvirt-lxc
  lxc: Add save/restore support

 configure.ac             |   1 +
 m4/virt-criu.m4          |  27 +++++
 po/POTFILES.in           |   1 +
 src/lxc/Makefile.inc.am  |   4 +
 src/lxc/lxc_container.c  | 162 ++++++++++++++++++++++++++++--
 src/lxc/lxc_container.h  |   3 +-
 src/lxc/lxc_controller.c | 104 ++++++++++++++++++-
 src/lxc/lxc_criu.c       | 253 +++++++++++++++++++++++++++++++++++++++++++++++
 src/lxc/lxc_criu.h       |  36 +++++++
 src/lxc/lxc_driver.c     | 238 +++++++++++++++++++++++++++++++++++++++++++-
 src/lxc/lxc_process.c    |  23 ++++-
 src/lxc/lxc_process.h    |   1 +
 12 files changed, 836 insertions(+), 17 deletions(-)
 create mode 100644 m4/virt-criu.m4
 create mode 100644 src/lxc/lxc_criu.c
 create mode 100644 src/lxc/lxc_criu.h

-- 
2.14.3

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
Re: [libvirt] [RFC PATCH 0/4] LXC - Implement save/restore domain state
Posted by Daniel P. Berrangé, 5 weeks ago
On Wed, Apr 11, 2018 at 12:29:11PM +0100, Radostin Stoyanov wrote:
> This patch set contains rebased version of Katerina's work from GSoC 2016 [1].
> It allows integrates CRIU [2] with the libvirt-lxc to enable save/resore of containers.

I vaguely recall that when Katerina first did that work, we hit some
limitations of CRIU at the time, that blocked us merging. Does anyone
recall what that was, and if & when it was addressed in CRIU ?

> 
> [1] https://wiki.libvirt.org/page/Google_Summer_of_Code_2016/lxc_migration
> [2] https://criu.org
> 
> Radostin Stoyanov (4):
>   configure: Include support for CRIU
>   lxc: Add save/restore helper functions
>   lxc: Add restore mode for libvirt-lxc
>   lxc: Add save/restore support
> 
>  configure.ac             |   1 +
>  m4/virt-criu.m4          |  27 +++++
>  po/POTFILES.in           |   1 +
>  src/lxc/Makefile.inc.am  |   4 +
>  src/lxc/lxc_container.c  | 162 ++++++++++++++++++++++++++++--
>  src/lxc/lxc_container.h  |   3 +-
>  src/lxc/lxc_controller.c | 104 ++++++++++++++++++-
>  src/lxc/lxc_criu.c       | 253 +++++++++++++++++++++++++++++++++++++++++++++++
>  src/lxc/lxc_criu.h       |  36 +++++++
>  src/lxc/lxc_driver.c     | 238 +++++++++++++++++++++++++++++++++++++++++++-
>  src/lxc/lxc_process.c    |  23 ++++-
>  src/lxc/lxc_process.h    |   1 +
>  12 files changed, 836 insertions(+), 17 deletions(-)
>  create mode 100644 m4/virt-criu.m4
>  create mode 100644 src/lxc/lxc_criu.c
>  create mode 100644 src/lxc/lxc_criu.h
> 
> -- 
> 2.14.3
> 
> --
> libvir-list mailing list
> libvir-list@redhat.com
> https://www.redhat.com/mailman/listinfo/libvir-list

Regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
Re: [libvirt] [RFC PATCH 0/4] LXC - Implement save/restore domain state
Posted by Radostin Stoyanov, 5 weeks ago
On 11/04/18 12:34, Daniel P. Berrangé wrote:
> On Wed, Apr 11, 2018 at 12:29:11PM +0100, Radostin Stoyanov wrote:
>> This patch set contains rebased version of Katerina's work from GSoC 2016 [1].
>> It allows integrates CRIU [2] with the libvirt-lxc to enable save/resore of containers.
> I vaguely recall that when Katerina first did that work, we hit some
> limitations of CRIU at the time, that blocked us merging. Does anyone
> recall what that was, and if & when it was addressed in CRIU ?
The previous patch series (from 2016) is
https://www.redhat.com/archives/libvir-list/2016-July/msg00855.html
One current limitation of CRIU is that it fails to restore a containers
with enabled user namespace (See
https://github.com/checkpoint-restore/criu/issues/466)

Radostin

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
Re: [libvirt] [RFC PATCH 0/4] LXC - Implement save/restore domain state
Posted by Cedric Bosdonnat, 5 weeks ago
On Wed, 2018-04-11 at 12:34 +0100, Daniel P. Berrangé wrote:
> On Wed, Apr 11, 2018 at 12:29:11PM +0100, Radostin Stoyanov wrote:
> > This patch set contains rebased version of Katerina's work from GSoC 2016 [1].
> > It allows integrates CRIU [2] with the libvirt-lxc to enable save/resore of containers.
> 
> I vaguely recall that when Katerina first did that work, we hit some
> limitations of CRIU at the time, that blocked us merging. Does anyone
> recall what that was, and if & when it was addressed in CRIU ?

What was missing in CRIU was the possibility to write the data into a stream
rather than in files. From what I recall this work has been merged upstream
in the mean time.

--
Cedric

> > 
> > [1] https://wiki.libvirt.org/page/Google_Summer_of_Code_2016/lxc_migration
> > [2] https://criu.org
> > 
> > Radostin Stoyanov (4):
> >   configure: Include support for CRIU
> >   lxc: Add save/restore helper functions
> >   lxc: Add restore mode for libvirt-lxc
> >   lxc: Add save/restore support
> > 
> >  configure.ac             |   1 +
> >  m4/virt-criu.m4          |  27 +++++
> >  po/POTFILES.in           |   1 +
> >  src/lxc/Makefile.inc.am  |   4 +
> >  src/lxc/lxc_container.c  | 162 ++++++++++++++++++++++++++++--
> >  src/lxc/lxc_container.h  |   3 +-
> >  src/lxc/lxc_controller.c | 104 ++++++++++++++++++-
> >  src/lxc/lxc_criu.c       | 253 +++++++++++++++++++++++++++++++++++++++++++++++
> >  src/lxc/lxc_criu.h       |  36 +++++++
> >  src/lxc/lxc_driver.c     | 238 +++++++++++++++++++++++++++++++++++++++++++-
> >  src/lxc/lxc_process.c    |  23 ++++-
> >  src/lxc/lxc_process.h    |   1 +
> >  12 files changed, 836 insertions(+), 17 deletions(-)
> >  create mode 100644 m4/virt-criu.m4
> >  create mode 100644 src/lxc/lxc_criu.c
> >  create mode 100644 src/lxc/lxc_criu.h
> > 
> > -- 
> > 2.14.3
> > 
> > --
> > libvir-list mailing list
> > libvir-list@redhat.com
> > https://www.redhat.com/mailman/listinfo/libvir-list
> 
> Regards,
> Daniel

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
[libvirt] [RFC PATCH 1/4] configure: Include support for CRIU
Posted by Radostin Stoyanov, 5 weeks ago
Checkpoint/restore in User-space [1] enables the current running state
of lxc containers to be stored as a set of image files. Then, one could
restore the container to its previous state (before checkpointing) on
the same or another system.

[1] https://criu.org/

Signed-off-by: Radostin Stoyanov <rstoyanov1@gmail.com>
---
 configure.ac    |  1 +
 m4/virt-criu.m4 | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 m4/virt-criu.m4

diff --git a/configure.ac b/configure.ac
index ae798faa5..8defba3bc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -282,6 +282,7 @@ LIBVIRT_CHECK_AVAHI
 LIBVIRT_CHECK_BASH_COMPLETION
 LIBVIRT_CHECK_BLKID
 LIBVIRT_CHECK_CAPNG
+LIBVIRT_CHECK_CRIU
 LIBVIRT_CHECK_CURL
 LIBVIRT_CHECK_DBUS
 LIBVIRT_CHECK_DEVMAPPER
diff --git a/m4/virt-criu.m4 b/m4/virt-criu.m4
new file mode 100644
index 000000000..05847d96b
--- /dev/null
+++ b/m4/virt-criu.m4
@@ -0,0 +1,27 @@
+dnl CRIU is used to checkpoint/restore LXC containers.
+dnl
+dnl Copyright (C) 2016 Katerina Koukiou
+dnl
+dnl This library is free software; you can redistribute it and/or
+dnl modify it under the terms of the GNU Lesser General Public
+dnl License as published by the Free Software Foundation; either
+dnl version 2.1 of the License, or (at your option) any later version.
+dnl
+dnl This library is distributed in the hope that it will be useful,
+dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
+dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+dnl Lesser General Public License for more details.
+dnl
+dnl You should have received a copy of the GNU Lesser General Public
+dnl License along with this library.  If not, see
+dnl <http://www.gnu.org/licenses/>.
+dnl
+
+AC_DEFUN([LIBVIRT_CHECK_CRIU],[
+    AC_PATH_PROG([CRIU], [criu], [no],
+        [$PATH:/sbin:/usr/sbin:/usr/local/sbin])
+    if test "x$ac_cv_path_CRIU" != "xno"; then
+        AC_DEFINE_UNQUOTED([CRIU], ["$CRIU"],
+        [Location of criu program])
+    fi
+])
-- 
2.14.3

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
[libvirt] [RFC PATCH 2/4] lxc: Add save/restore helper functions
Posted by Radostin Stoyanov, 5 weeks ago
Add helper functions for checkpoint/restore of linux containers using
CRIU.

Signed-off-by: Radostin Stoyanov <rstoyanov1@gmail.com>
---
 po/POTFILES.in          |   1 +
 src/lxc/Makefile.inc.am |   4 +
 src/lxc/lxc_criu.c      | 253 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/lxc/lxc_criu.h      |  36 +++++++
 4 files changed, 294 insertions(+)
 create mode 100644 src/lxc/lxc_criu.c
 create mode 100644 src/lxc/lxc_criu.h

diff --git a/po/POTFILES.in b/po/POTFILES.in
index 0dcd1cab2..a6d3a5743 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -106,6 +106,7 @@ src/lxc/lxc_fuse.c
 src/lxc/lxc_hostdev.c
 src/lxc/lxc_native.c
 src/lxc/lxc_process.c
+src/lxc/lxc_criu.c
 src/network/bridge_driver.c
 src/network/bridge_driver_linux.c
 src/network/leaseshelper.c
diff --git a/src/lxc/Makefile.inc.am b/src/lxc/Makefile.inc.am
index 8dd2e9ea9..fbdc87b24 100644
--- a/src/lxc/Makefile.inc.am
+++ b/src/lxc/Makefile.inc.am
@@ -42,6 +42,8 @@ LXC_DRIVER_SOURCES = \
 	lxc/lxc_native.h \
 	lxc/lxc_driver.c \
 	lxc/lxc_driver.h \
+	lxc/lxc_criu.c \
+	lxc/lxc_criu.h \
 	$(NULL)
 
 LXC_CONTROLLER_SOURCES = \
@@ -58,6 +60,8 @@ LXC_CONTROLLER_SOURCES = \
 	lxc/lxc_fuse.c \
 	lxc/lxc_fuse.h \
 	lxc/lxc_controller.c \
+	lxc/lxc_criu.c \
+	lxc/lxc_criu.h \
 	$(NULL)
 
 
diff --git a/src/lxc/lxc_criu.c b/src/lxc/lxc_criu.c
new file mode 100644
index 000000000..b45f9b9f3
--- /dev/null
+++ b/src/lxc/lxc_criu.c
@@ -0,0 +1,253 @@
+/*
+ * Copyright (C) 2016 Katerina Koukiou
+ *
+ * lxc_criu.c: Helper functions for checkpoint/restore of linux containers
+ *
+ * Authors:
+ *  Katerina Koukiou <k.koukiou@gmail.com>
+ *  Radostin Stoyanov <r.stoyanov.14@aberdeen.ac.uk>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/mount.h>
+
+#include "virobject.h"
+#include "virerror.h"
+#include "virlog.h"
+#include "virfile.h"
+#include "vircommand.h"
+#include "virstring.h"
+#include "viralloc.h"
+
+#include "lxc_domain.h"
+#include "lxc_driver.h"
+#include "lxc_criu.h"
+
+#define VIR_FROM_THIS VIR_FROM_LXC
+
+VIR_LOG_INIT("lxc.lxc_criu");
+
+#ifdef CRIU
+
+int lxcCriuDump(virLXCDriverPtr driver ATTRIBUTE_UNUSED,
+                virDomainObjPtr vm,
+                const char *checkpointdir)
+{
+    int fd;
+    int ret = -1;
+    pid_t initpid;
+    virCommandPtr cmd;
+    struct stat sb;
+    char *path = NULL;
+    char *tty_info_path = NULL;
+    char *ttyinfo = NULL;
+    int status;
+
+    initpid = ((virLXCDomainObjPrivatePtr) vm->privateData)->initpid;
+
+    if (virFileMakePath(checkpointdir) < 0) {
+        virReportSystemError(errno, _("Failed to mkdir %s"), checkpointdir);
+        return -1;
+    }
+
+    fd = open(checkpointdir, O_DIRECTORY);
+    if (fd < 0) {
+        virReportSystemError(errno,
+                             _("Failed to open directory %s"), checkpointdir);
+        return -1;
+    }
+
+    cmd = virCommandNew(CRIU);
+    virCommandAddArg(cmd, "dump");
+    virCommandAddArg(cmd, "--tree");
+    virCommandAddArgFormat(cmd, "%d", initpid);
+    virCommandAddArgList(cmd,
+        "--images-dir", checkpointdir,
+        "--tcp-established",
+        "--log-file", "dump.log",
+        "-v4",
+        "--file-locks",
+        "--link-remap",
+        "--force-irmap",
+        "--manage-cgroups=full",
+        "--enable-fs", "hugetlbfs",
+        "--enable-fs", "tracefs",
+        "--external", "mnt[]{:ms}",
+        "--external", "mnt[/proc/meminfo]:fuse",
+        "--external", "mnt[/dev/console]:console",
+        "--external", "mnt[/dev/tty1]:tty1",
+        NULL
+    );
+
+    /* The master pair of the /dev/pts device lives outside from what is dumped
+     * inside the libvirt-lxc process. Add the slave pair as an external tty
+     * otherwise criu will fail.
+     */
+    if (virAsprintf(&path, "/proc/%d/root/dev/pts/0", initpid) < 0)
+        goto cleanup;
+
+    if (stat(path, &sb) < 0) {
+        virReportSystemError(errno, _("Unable to stat %s"), path);
+        goto cleanup;
+    }
+
+    if (virAsprintf(&tty_info_path, "%s/tty.info", checkpointdir) < 0)
+        goto cleanup;
+
+    if (virAsprintf(&ttyinfo, "tty[%llx:%llx]",
+                    (long long unsigned) sb.st_rdev,
+                    (long long unsigned) sb.st_dev) < 0)
+        goto cleanup;
+
+    if (virFileWriteStr(tty_info_path, ttyinfo, 0600) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("Failed to write tty info to %s"), tty_info_path);
+        goto cleanup;
+    }
+
+    virCommandAddArg(cmd, "--external");
+    virCommandAddArgFormat(cmd, "tty[%llx:%llx]",
+                          (long long unsigned) sb.st_rdev,
+                          (long long unsigned) sb.st_dev);
+
+    virCommandAddEnvString(cmd, "PATH=/bin:/sbin");
+
+    VIR_DEBUG("About to checkpoint domain %s (pid = %d)",
+              vm->def->name, initpid);
+    virCommandRawStatus(cmd);
+    if (virCommandRun(cmd, &status) < 0)
+        goto cleanup;
+
+    ret = 0;
+
+ cleanup:
+    VIR_FORCE_CLOSE(fd);
+    VIR_FREE(path);
+    VIR_FREE(tty_info_path);
+    VIR_FREE(ttyinfo);
+    virCommandFree(cmd);
+
+    return (ret < 0) ? ret : status;
+}
+
+
+int lxcCriuRestore(virDomainDefPtr def, int restorefd,
+                   int ttyfd)
+{
+    int ret = -1;
+    virCommandPtr cmd;
+    char *ttyinfo = NULL;
+    char *inheritfd = NULL;
+    char *tty_info_path = NULL;
+    char *checkpointfd = NULL;
+    char *checkpointdir = NULL;
+    virDomainFSDefPtr root;
+    gid_t *groups = NULL;
+    int ngroups;
+
+    cmd = virCommandNew(CRIU);
+    virCommandAddArg(cmd, "restore");
+
+    if (virAsprintf(&checkpointfd, "/proc/self/fd/%d", restorefd) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to write checkpoint dir path"));
+        goto cleanup;
+    }
+
+    if (virFileResolveLink(checkpointfd, &checkpointdir) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to readlink checkpoint dir path"));
+        goto cleanup;
+    }
+
+    virCommandAddArgList(cmd,
+        "--pidfile", "pidfile",
+        "--restore-detached",
+        "--restore-sibling",
+        "--tcp-established",
+        "--file-locks",
+        "--link-remap",
+        "--manage-cgroups=full",
+        "--enable-fs", "hugetlbfs",
+        "--enable-fs", "tracefs",
+        "--images-dir", checkpointdir,
+        "--log-file", "restore.log",
+        "-v4",
+        "--external", "mnt[]{:ms}",
+        "--external", "mnt[fuse]:/proc/meminfo",
+        "--external", "mnt[console]:/dev/console",
+        "--external", "mnt[tty1]:/dev/tty1",
+        NULL
+    );
+
+    /* Restore external tty from tty.info file */
+    if (virAsprintf(&tty_info_path, "%s/tty.info", checkpointdir) < 0)
+        goto cleanup;
+
+    if (virFileReadAll(tty_info_path, 1024, &ttyinfo) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("Failed to read tty info from %s"), tty_info_path);
+        goto cleanup;
+    }
+    if (virAsprintf(&inheritfd, "fd[%d]:%s", ttyfd, ttyinfo) < 0)
+        goto cleanup;
+
+    virCommandAddArgList(cmd, "--inherit-fd", inheritfd, NULL);
+
+    root = virDomainGetFilesystemForTarget(def, "/");
+    virCommandAddArgList(cmd, "--root", root->src->path, NULL);
+
+    virCommandAddEnvString(cmd, "PATH=/bin:/sbin");
+
+    if ((ngroups = virGetGroupList(virCommandGetUID(cmd), virCommandGetGID(cmd), &groups)) < 0)
+        goto cleanup;
+
+    /* If virCommandExec returns here we have an error */
+    ignore_value(virCommandExec(cmd, groups, ngroups));
+
+    ret = -1;
+
+ cleanup:
+    VIR_FREE(tty_info_path);
+    VIR_FREE(ttyinfo);
+    VIR_FREE(inheritfd);
+    VIR_FREE(checkpointdir);
+    VIR_FREE(checkpointfd);
+    virCommandFree(cmd);
+
+    return ret;
+}
+#else
+int lxcCriuDump(virLXCDriverPtr driver ATTRIBUTE_UNUSED,
+                virDomainObjPtr vm ATTRIBUTE_UNUSED,
+                const char *checkpointdir ATTRIBUTE_UNUSED)
+{
+    virReportUnsupportedError();
+    return -1;
+}
+
+int lxcCriuRestore(virDomainDefPtr def ATTRIBUTE_UNUSED,
+                   int fd ATTRIBUTE_UNUSED,
+                   int ttyfd ATTRIBUTE_UNUSED)
+{
+    virReportUnsupportedError();
+    return -1;
+}
+#endif
diff --git a/src/lxc/lxc_criu.h b/src/lxc/lxc_criu.h
new file mode 100644
index 000000000..aadc3ac34
--- /dev/null
+++ b/src/lxc/lxc_criu.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2016 Katerina Koukiou
+ *
+ * lxc_criu.h: Helper functions for checkpoint/restore of linux containers
+ *
+ * Authors:
+ *  Katerina Koukiou <k.koukiou@gmail.com>
+ *  Radostin Stoyanov <r.stoyanov.14@aberdeen.ac.uk>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef LXC_CRIU_H
+# define LXC_CRIU_H
+
+# include "virobject.h"
+
+int lxcCriuDump(virLXCDriverPtr driver,
+                virDomainObjPtr vm,
+                const char *checkpointdir);
+
+int lxcCriuRestore(virDomainDefPtr def, int fd,
+                   int ttyfd);
+#endif /* LXC_CRIU_H */
-- 
2.14.3

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
[libvirt] [RFC PATCH 3/4] lxc: Add restore mode for libvirt-lxc
Posted by Radostin Stoyanov, 5 weeks ago
Extend `lxcContainerStart` with support for restore from fd
of directory that contains saved state of lxc container.

Signed-off-by: Radostin Stoyanov <rstoyanov1@gmail.com>
---
 src/lxc/lxc_container.c  | 162 +++++++++++++++++++++++++++++++++++++++++++++--
 src/lxc/lxc_container.h  |   3 +-
 src/lxc/lxc_controller.c | 104 ++++++++++++++++++++++++++++--
 src/lxc/lxc_driver.c     |   4 +-
 src/lxc/lxc_process.c    |  23 +++++--
 src/lxc/lxc_process.h    |   1 +
 6 files changed, 280 insertions(+), 17 deletions(-)

diff --git a/src/lxc/lxc_container.c b/src/lxc/lxc_container.c
index 532fd0be0..6cd203d7f 100644
--- a/src/lxc/lxc_container.c
+++ b/src/lxc/lxc_container.c
@@ -69,6 +69,8 @@
 #include "virprocess.h"
 #include "virstring.h"
 
+#include "lxc_criu.h"
+
 #define VIR_FROM_THIS VIR_FROM_LXC
 
 VIR_LOG_INIT("lxc.lxc_container");
@@ -111,6 +113,7 @@ struct __lxc_child_argv {
     char **ttyPaths;
     int handshakefd;
     int *nsInheritFDs;
+    int restorefd;
 };
 
 static int lxcContainerMountFSBlock(virDomainFSDefPtr fs,
@@ -263,6 +266,7 @@ static virCommandPtr lxcContainerBuildInitCmd(virDomainDefPtr vmDef,
  * @ttyfd: FD of tty to set as the container console
  * @npassFDs: number of extra FDs
  * @passFDs: list of extra FDs
+ * @restorefd: FD of folder where container was dumped
  *
  * Setup file descriptors in the container. @ttyfd is set to be
  * the container's stdin, stdout & stderr. Any FDs included in
@@ -272,7 +276,7 @@ static virCommandPtr lxcContainerBuildInitCmd(virDomainDefPtr vmDef,
  * Returns 0 on success or -1 in case of error
  */
 static int lxcContainerSetupFDs(int *ttyfd,
-                                size_t npassFDs, int *passFDs)
+                                size_t npassFDs, int *passFDs, int restorefd)
 {
     int rc = -1;
     int open_max;
@@ -368,6 +372,8 @@ static int lxcContainerSetupFDs(int *ttyfd,
     }
 
     for (fd = last_fd + 1; fd < open_max; fd++) {
+        if (fd == restorefd)
+            continue;
         int tmpfd = fd;
         VIR_MASS_CLOSE(tmpfd);
     }
@@ -1083,6 +1089,31 @@ static int lxcContainerMountFSDev(virDomainDefPtr def,
     return ret;
 }
 
+static int lxcContainerMountFSDevPTSRestore(virDomainDefPtr def,
+                                            const char *stateDir)
+{
+    int ret = -1;
+    char *path = NULL;
+    int flags = MS_MOVE;
+
+    VIR_DEBUG("Mount /dev/pts stateDir=%s", stateDir);
+
+    if (virAsprintf(&path, "%s/%s.devpts", stateDir, def->name) < 0)
+        return ret;
+
+    VIR_DEBUG("Trying to move %s to /dev/pts", path);
+
+    if (mount(path, "/dev/pts", NULL, flags, NULL) < 0) {
+        virReportSystemError(errno, _("Failed to mount %s on /dev/pts"), path);
+        goto cleanup;
+    }
+
+    ret = 0;
+ cleanup:
+    VIR_FREE(path);
+    return ret;
+}
+
 static int lxcContainerMountFSDevPTS(virDomainDefPtr def,
                                      const char *stateDir)
 {
@@ -2191,6 +2222,116 @@ static int lxcContainerSetHostname(virDomainDefPtr def)
     return ret;
 }
 
+/*
+ * lxcContainerChildRestore:
+ * @data: pointer to container arguments
+ */
+static int lxcContainerChildRestore(void *data)
+{
+    lxc_child_argv_t *argv = data;
+    virDomainDefPtr vmDef = argv->config;
+    int ttyfd = -1;
+    int ret = -1;
+    char *ttyPath = NULL;
+    virDomainFSDefPtr root;
+    char *sec_mount_options = NULL;
+    char *stateDir = NULL;
+
+    if (vmDef == NULL) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       "%s", _("lxcChild() passed invalid vm definition"));
+        goto cleanup;
+    }
+
+    if (lxcContainerWaitForContinue(argv->monitor) < 0) {
+        virReportSystemError(errno, "%s",
+                             _("Failed to read the container continue message"));
+        goto cleanup;
+    }
+    VIR_DEBUG("Received container continue message");
+
+    if (lxcContainerSetID(vmDef) < 0)
+        goto cleanup;
+
+    root = virDomainGetFilesystemForTarget(vmDef, "/");
+
+    if (argv->nttyPaths) {
+        const char *tty = argv->ttyPaths[0];
+        if (STRPREFIX(tty, "/dev/pts/"))
+            tty += strlen("/dev/pts/");
+        if (virAsprintf(&ttyPath, "%s/%s.devpts/%s",
+                        LXC_STATE_DIR, vmDef->name, tty) < 0)
+            goto cleanup;
+    } else {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("At least one tty is required"));
+        goto cleanup;
+    }
+
+    VIR_DEBUG("Container TTY path: %s", ttyPath);
+
+    ttyfd = open(ttyPath, O_RDWR);
+    if (ttyfd < 0) {
+        virReportSystemError(errno, _("Failed to open tty %s"), ttyPath);
+        goto cleanup;
+    }
+    VIR_DEBUG("Container TTY fd: %d", ttyfd);
+
+    if (!(sec_mount_options = virSecurityManagerGetMountOptions(
+                                        argv->securityDriver,
+                                        vmDef)))
+        goto cleanup;
+
+    if (lxcContainerPrepareRoot(vmDef, root, sec_mount_options) < 0)
+        goto cleanup;
+
+    if (lxcContainerSendContinue(argv->handshakefd) < 0) {
+        virReportSystemError(errno, "%s",
+                            _("Failed to send continue signal to controller"));
+        goto cleanup;
+    }
+
+    VIR_DEBUG("Setting up container's std streams");
+
+    if (lxcContainerSetupFDs(&ttyfd, argv->npassFDs,
+                             argv->passFDs, argv->restorefd) < 0)
+        goto cleanup;
+
+    if (virFileResolveAllLinks(LXC_STATE_DIR, &stateDir) < 0)
+        goto cleanup;
+
+    /* Mounts /dev/pts */
+    if (lxcContainerMountFSDevPTSRestore(vmDef, stateDir) < 0) {
+        virReportSystemError(errno, "%s", _("Failed to mount dev/pts"));
+        goto cleanup;
+    }
+
+    if (setsid() < 0)
+        virReportSystemError(errno, "%s", _("Unable to become session leader"));
+
+    VIR_DEBUG("Executing container restore criu function");
+    ret = lxcCriuRestore(vmDef, argv->restorefd, 0);
+
+ cleanup:
+    VIR_FORCE_CLOSE(argv->monitor);
+    VIR_FORCE_CLOSE(argv->handshakefd);
+    VIR_FORCE_CLOSE(ttyfd);
+    VIR_FREE(ttyPath);
+    VIR_FREE(stateDir);
+    VIR_FREE(sec_mount_options);
+
+    if (ret != 0) {
+        VIR_DEBUG("Tearing down container");
+        fprintf(stderr,
+                _("Failure in libvirt_lxc startup: %s\n"),
+                virGetLastErrorMessage());
+    }
+
+    return ret;
+}
+
+
+
 /**
  * lxcContainerChild:
  * @data: pointer to container arguments
@@ -2322,7 +2463,7 @@ static int lxcContainerChild(void *data)
     VIR_FORCE_CLOSE(argv->handshakefd);
     VIR_FORCE_CLOSE(argv->monitor);
     if (lxcContainerSetupFDs(&ttyfd,
-                             argv->npassFDs, argv->passFDs) < 0)
+                             argv->npassFDs, argv->passFDs, -1) < 0)
         goto cleanup;
 
     /* Make init process of the container the leader of the new session.
@@ -2403,6 +2544,7 @@ virArch lxcContainerGetAlt32bitArch(virArch arch)
  * @veths: interface names
  * @control: control FD to the container
  * @ttyPath: path of tty to set as the container console
+ * @restorefd: FD to folder where container was dumped
  *
  * Starts a container process by calling clone() with the namespace flags
  *
@@ -2418,7 +2560,8 @@ int lxcContainerStart(virDomainDefPtr def,
                       int handshakefd,
                       int *nsInheritFDs,
                       size_t nttyPaths,
-                      char **ttyPaths)
+                      char **ttyPaths,
+                      int restorefd)
 {
     pid_t pid;
     int cflags;
@@ -2436,6 +2579,7 @@ int lxcContainerStart(virDomainDefPtr def,
         .ttyPaths = ttyPaths,
         .handshakefd = handshakefd,
         .nsInheritFDs = nsInheritFDs,
+        .restorefd = restorefd,
     };
 
     /* allocate a stack for the container */
@@ -2484,10 +2628,16 @@ int lxcContainerStart(virDomainDefPtr def,
         VIR_DEBUG("Inheriting a UTS namespace");
     }
 
-    VIR_DEBUG("Cloning container init process");
-    pid = clone(lxcContainerChild, stacktop, cflags, &args);
+    if (restorefd != -1) {
+        VIR_DEBUG("Cloning container process that will spawn criu restore");
+        pid = clone(lxcContainerChildRestore, stacktop, SIGCHLD, &args);
+    } else {
+        VIR_DEBUG("Cloning container init process");
+        pid = clone(lxcContainerChild, stacktop, cflags, &args);
+        VIR_DEBUG("clone() completed, new container PID is %d", pid);
+    }
+
     VIR_FREE(stack);
-    VIR_DEBUG("clone() completed, new container PID is %d", pid);
 
     if (pid < 0) {
         virReportSystemError(errno, "%s",
diff --git a/src/lxc/lxc_container.h b/src/lxc/lxc_container.h
index 641e2d460..9a6ac2073 100644
--- a/src/lxc/lxc_container.h
+++ b/src/lxc/lxc_container.h
@@ -58,7 +58,8 @@ int lxcContainerStart(virDomainDefPtr def,
                       int handshakefd,
                       int *nsInheritFDs,
                       size_t nttyPaths,
-                      char **ttyPaths);
+                      char **ttyPaths,
+                      int restorefd);
 
 int lxcContainerSetupHostdevCapsMakePath(const char *dev);
 
diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c
index 507bffda0..a5eb5e336 100644
--- a/src/lxc/lxc_controller.c
+++ b/src/lxc/lxc_controller.c
@@ -146,6 +146,8 @@ struct _virLXCController {
     virCgroupPtr cgroup;
 
     virLXCFusePtr fuse;
+
+    int restore;
 };
 
 #include "lxc_controller_dispatch.h"
@@ -1015,6 +1017,65 @@ static int lxcControllerClearCapabilities(void)
     return 0;
 }
 
+static int
+lxcControllerFindRestoredPid(int fd)
+{
+    int initpid = 0;
+    int ret = -1;
+    char *checkpointdir = NULL;
+    char *pidfile = NULL;
+    char *checkpointfd = NULL;
+    int pidfilefd;
+    char c;
+
+    if (fd < 0)
+        goto cleanup;
+
+    if (virAsprintf(&checkpointfd, "/proc/self/fd/%d", fd) < 0) {
+            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                           _("Failed to write checkpoint dir path"));
+            goto cleanup;
+    }
+
+    if (virFileResolveLink(checkpointfd, &checkpointdir) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to readlink checkpoint dir path"));
+        goto cleanup;
+    }
+
+    if (virAsprintf(&pidfile, "%s/pidfile", checkpointdir) < 0) {
+            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                                           _("Failed to write pidfile path"));
+        goto cleanup;
+    }
+
+    if ((pidfilefd = virFileOpenAs(pidfile, O_RDONLY, 0, -1, -1, 0)) < 0) {
+        virReportSystemError(pidfilefd,
+                             _("Failed to open domain's pidfile '%s'"),
+                             pidfile);
+        goto cleanup;
+    }
+
+    while ((saferead(pidfilefd,  &c, 1) == 1) &&  c != EOF)
+        initpid = initpid*10 + c - '0';
+
+    ret = initpid;
+
+    if (virFileRemove(pidfile, -1, -1) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to delete pidfile path"));
+    }
+
+ cleanup:
+    VIR_FORCE_CLOSE(fd);
+    VIR_FORCE_CLOSE(pidfilefd);
+    VIR_FREE(pidfile);
+    VIR_FREE(checkpointdir);
+    VIR_FREE(checkpointfd);
+    return ret;
+}
+
+
 static bool wantReboot;
 static virMutex lock = VIR_MUTEX_INITIALIZER;
 
@@ -2327,6 +2388,7 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
     int containerhandshake[2] = { -1, -1 };
     char **containerTTYPaths = NULL;
     size_t i;
+    bool restore_mode = (ctrl->restore != -1);
 
     if (VIR_ALLOC_N(containerTTYPaths, ctrl->nconsoles) < 0)
         goto cleanup;
@@ -2383,7 +2445,8 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
                                            containerhandshake[1],
                                            ctrl->nsFDs,
                                            ctrl->nconsoles,
-                                           containerTTYPaths)) < 0)
+                                           containerTTYPaths,
+                                           ctrl->restore)) < 0)
         goto cleanup;
     VIR_FORCE_CLOSE(control[1]);
     VIR_FORCE_CLOSE(containerhandshake[1]);
@@ -2395,10 +2458,10 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
         for (i = 0; i < VIR_LXC_DOMAIN_NAMESPACE_LAST; i++)
             VIR_FORCE_CLOSE(ctrl->nsFDs[i]);
 
-    if (virLXCControllerSetupCgroupLimits(ctrl) < 0)
+    if (!restore_mode && virLXCControllerSetupCgroupLimits(ctrl) < 0)
         goto cleanup;
 
-    if (virLXCControllerSetupUserns(ctrl) < 0)
+    if (!restore_mode && virLXCControllerSetupUserns(ctrl) < 0)
         goto cleanup;
 
     if (virLXCControllerMoveInterfaces(ctrl) < 0)
@@ -2423,6 +2486,26 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
     if (lxcControllerClearCapabilities() < 0)
         goto cleanup;
 
+    if (restore_mode) {
+        int status;
+        int ret = waitpid(-1, &status, 0);
+        VIR_DEBUG("Got sig child %d", ret);
+
+        /* There could be two cases here:
+         * 1. CRIU died bacause of restore error and the container is not running
+         * 2. CRIU detached itself from the running container
+         */
+        int initpid;
+        if ((initpid = lxcControllerFindRestoredPid(ctrl->restore)) < 0) {
+            virReportSystemError(errno, "%s",
+                                 _("Unable to get restored task pid"));
+            virNetDaemonQuit(ctrl->daemon);
+            goto cleanup;
+        }
+
+        ctrl->initpid = initpid;
+    }
+
     for (i = 0; i < ctrl->nconsoles; i++)
         if (virLXCControllerConsoleSetNonblocking(&(ctrl->consoles[i])) < 0)
             goto cleanup;
@@ -2466,6 +2549,7 @@ int main(int argc, char *argv[])
     int ns_fd[VIR_LXC_DOMAIN_NAMESPACE_LAST];
     int handshakeFd = -1;
     bool bg = false;
+    int restore = -1;
     const struct option options[] = {
         { "background", 0, NULL, 'b' },
         { "name",   1, NULL, 'n' },
@@ -2477,6 +2561,7 @@ int main(int argc, char *argv[])
         { "share-net", 1, NULL, 'N' },
         { "share-ipc", 1, NULL, 'I' },
         { "share-uts", 1, NULL, 'U' },
+        { "restore", 1, NULL, 'r' },
         { "help", 0, NULL, 'h' },
         { 0, 0, 0, 0 },
     };
@@ -2504,7 +2589,7 @@ int main(int argc, char *argv[])
     while (1) {
         int c;
 
-        c = getopt_long(argc, argv, "dn:v:p:m:c:s:h:S:N:I:U:",
+        c = getopt_long(argc, argv, "dn:v:p:m:c:s:h:S:N:I:U:r:",
                         options, NULL);
 
         if (c == -1)
@@ -2580,6 +2665,14 @@ int main(int argc, char *argv[])
             securityDriver = optarg;
             break;
 
+        case 'r':
+             if (virStrToLong_i(optarg, NULL, 10, &restore) < 0) {
+                fprintf(stderr, "malformed --restore argument '%s'",
+                        optarg);
+                goto cleanup;
+            }
+            break;
+
         case 'h':
         case '?':
             fprintf(stderr, "\n");
@@ -2596,6 +2689,7 @@ int main(int argc, char *argv[])
             fprintf(stderr, "  -N FD, --share-net FD\n");
             fprintf(stderr, "  -I FD, --share-ipc FD\n");
             fprintf(stderr, "  -U FD, --share-uts FD\n");
+            fprintf(stderr, "  -r FD, --restore FD\n");
             fprintf(stderr, "  -h, --help\n");
             fprintf(stderr, "\n");
             rc = 0;
@@ -2648,6 +2742,8 @@ int main(int argc, char *argv[])
     ctrl->passFDs = passFDs;
     ctrl->npassFDs = npassFDs;
 
+    ctrl->restore = restore;
+
     for (i = 0; i < VIR_LXC_DOMAIN_NAMESPACE_LAST; i++) {
         if (ns_fd[i] != -1) {
             if (!ctrl->nsFDs) {/*allocate only once */
diff --git a/src/lxc/lxc_driver.c b/src/lxc/lxc_driver.c
index 4f600f3df..f52085ebf 100644
--- a/src/lxc/lxc_driver.c
+++ b/src/lxc/lxc_driver.c
@@ -1125,7 +1125,7 @@ static int lxcDomainCreateWithFiles(virDomainPtr dom,
 
     ret = virLXCProcessStart(dom->conn, driver, vm,
                              nfiles, files,
-                             (flags & VIR_DOMAIN_START_AUTODESTROY),
+                             (flags & VIR_DOMAIN_START_AUTODESTROY), -1,
                              VIR_DOMAIN_RUNNING_BOOTED);
 
     if (ret == 0) {
@@ -1252,7 +1252,7 @@ lxcDomainCreateXMLWithFiles(virConnectPtr conn,
 
     if (virLXCProcessStart(conn, driver, vm,
                            nfiles, files,
-                           (flags & VIR_DOMAIN_START_AUTODESTROY),
+                           (flags & VIR_DOMAIN_START_AUTODESTROY), -1,
                            VIR_DOMAIN_RUNNING_BOOTED) < 0) {
         virDomainAuditStart(vm, "booted", false);
         virLXCDomainObjEndJob(driver, vm);
diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c
index 96041f2ec..1cd7f5bfe 100644
--- a/src/lxc/lxc_process.c
+++ b/src/lxc/lxc_process.c
@@ -118,7 +118,7 @@ virLXCProcessReboot(virLXCDriverPtr driver,
     virLXCProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN);
     vm->newDef = savedDef;
     if (virLXCProcessStart(conn, driver, vm,
-                           0, NULL, autodestroy, reason) < 0) {
+                           0, NULL, autodestroy, -1, reason) < 0) {
         VIR_WARN("Unable to handle reboot of vm %s",
                  vm->def->name);
         goto cleanup;
@@ -914,7 +914,8 @@ virLXCProcessBuildControllerCmd(virLXCDriverPtr driver,
                                 size_t nfiles,
                                 int handshakefd,
                                 int * const logfd,
-                                const char *pidfile)
+                                const char *pidfile,
+                                int restorefd)
 {
     size_t i;
     char *filterstr;
@@ -993,6 +994,12 @@ virLXCProcessBuildControllerCmd(virLXCDriverPtr driver,
     for (i = 0; i < nveths; i++)
         virCommandAddArgList(cmd, "--veth", veths[i], NULL);
 
+    if (restorefd != -1) {
+        virCommandAddArg(cmd, "--restore");
+        virCommandAddArgFormat(cmd, "%d", restorefd);
+        virCommandPassFD(cmd, restorefd, 0);
+    }
+
     virCommandPassFD(cmd, handshakefd, 0);
     virCommandDaemonize(cmd);
     virCommandSetPidFile(cmd, pidfile);
@@ -1166,6 +1173,8 @@ virLXCProcessEnsureRootFS(virDomainObjPtr vm)
  * @driver: pointer to driver structure
  * @vm: pointer to virtual machine structure
  * @autoDestroy: mark the domain for auto destruction
+ * @restorefd: file descriptor pointing to the restore directory (-1 if not
+ *             restoring)
  * @reason: reason for switching vm to running state
  *
  * Starts a vm
@@ -1177,6 +1186,7 @@ int virLXCProcessStart(virConnectPtr conn,
                        virDomainObjPtr vm,
                        unsigned int nfiles, int *files,
                        bool autoDestroy,
+                       int restorefd,
                        virDomainRunningReason reason)
 {
     int rc = -1, r;
@@ -1386,7 +1396,7 @@ int virLXCProcessStart(virConnectPtr conn,
                                                 files, nfiles,
                                                 handshakefds[1],
                                                 &logfd,
-                                                pidfile)))
+                                                pidfile, restorefd)))
         goto cleanup;
 
     /* now that we know it is about to start call the hook if present */
@@ -1494,6 +1504,9 @@ int virLXCProcessStart(virConnectPtr conn,
     if (!priv->machineName)
         goto cleanup;
 
+    if (restorefd != -1)
+        goto skip_cgroup_checks;
+
     /* We know the cgroup must exist by this synchronization
      * point so lets detect that first, since it gives us a
      * more reliable way to kill everything off if something
@@ -1510,6 +1523,8 @@ int virLXCProcessStart(virConnectPtr conn,
         goto cleanup;
     }
 
+ skip_cgroup_checks:
+
     /* And we can get the first monitor connection now too */
     if (!(priv->monitor = virLXCProcessConnectMonitor(driver, vm))) {
         /* Intentionally overwrite the real monitor error message,
@@ -1596,7 +1611,7 @@ virLXCProcessAutostartDomain(virDomainObjPtr vm,
     if (vm->autostart &&
         !virDomainObjIsActive(vm)) {
         ret = virLXCProcessStart(data->conn, data->driver, vm,
-                                 0, NULL, false,
+                                 0, NULL, false, -1,
                                  VIR_DOMAIN_RUNNING_BOOTED);
         virDomainAuditStart(vm, "booted", ret >= 0);
         if (ret < 0) {
diff --git a/src/lxc/lxc_process.h b/src/lxc/lxc_process.h
index d78cddef4..c724f31a7 100644
--- a/src/lxc/lxc_process.h
+++ b/src/lxc/lxc_process.h
@@ -29,6 +29,7 @@ int virLXCProcessStart(virConnectPtr conn,
                        virDomainObjPtr vm,
                        unsigned int nfiles, int *files,
                        bool autoDestroy,
+                       int restorefd,
                        virDomainRunningReason reason);
 int virLXCProcessStop(virLXCDriverPtr driver,
                       virDomainObjPtr vm,
-- 
2.14.3

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list
[libvirt] [RFC PATCH 4/4] lxc: Add save/restore support
Posted by Radostin Stoyanov, 5 weeks ago
Add support for saving the state of lxc domain into a set of files.

Usage:
	virsh save [domain-name or domain-uuid] [path]
	virsh restore [path]

Example:
	virsh -c lxc:// save container1 /tmp/container1-dump
	virsh -c lxc:// restore /tmp/container1-dump

Signed-off-by: Radostin Stoyanov <rstoyanov1@gmail.com>
---
 src/lxc/lxc_driver.c | 234 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 234 insertions(+)

diff --git a/src/lxc/lxc_driver.c b/src/lxc/lxc_driver.c
index f52085ebf..871e853a0 100644
--- a/src/lxc/lxc_driver.c
+++ b/src/lxc/lxc_driver.c
@@ -81,6 +81,7 @@
 #include "virhostdev.h"
 #include "netdev_bandwidth_conf.h"
 
+#include "lxc_criu.h"
 #define VIR_FROM_THIS VIR_FROM_LXC
 
 VIR_LOG_INIT("lxc.lxc_driver");
@@ -3198,6 +3199,235 @@ static int lxcDomainResume(virDomainPtr dom)
     return ret;
 }
 
+static int
+lxcDoDomainSave(virLXCDriverPtr driver, virDomainObjPtr vm,
+                const char *to)
+{
+    int ret = -1;
+    virCapsPtr caps = NULL;
+    uint32_t xml_len = -1;
+    char *xml = NULL;
+    char xmlLen[33];
+    char *xml_image_path = NULL;
+    char *str = NULL;
+
+    if (!(caps = virLXCDriverGetCapabilities(driver, false)))
+        goto cleanup;
+
+    if ((xml = virDomainDefFormat(vm->def, caps, 0)) == NULL)
+        goto cleanup;
+
+    if ((ret = lxcCriuDump(driver, vm, to)) != 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to checkpoint domain with CRIU"));
+        goto cleanup;
+    }
+
+    virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF,
+                         VIR_DOMAIN_SHUTOFF_SAVED);
+
+    if (virAsprintf(&xml_image_path, "%s/xml-image", to) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to write image path"));
+        goto cleanup;
+    }
+
+    xml_len = strlen(xml) + 1;
+    snprintf(xmlLen, sizeof(xmlLen), "%d", xml_len);
+    VIR_DEBUG("xmlLen = %d %s", xml_len, xmlLen);
+
+    if (virAsprintf(&str, "%s\n%s", xmlLen, xml) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to write xml info to string"));
+        goto cleanup;
+    }
+
+    if (virFileWriteStr(xml_image_path, str, 0666) < 0) {
+        virReportError(VIR_ERR_OPERATION_FAILED,
+                       _("Failed to write xml description to %s"),
+                       xml_image_path);
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    VIR_FREE(xml);
+    VIR_FREE(xml_image_path);
+    VIR_FREE(str);
+    return ret != 0 ? -1 : ret;
+}
+
+static int
+lxcDomainSaveFlags(virDomainPtr dom, const char *to, const char *dxml,
+                   unsigned int flags)
+{
+    int ret = -1;
+    virLXCDriverPtr driver = dom->conn->privateData;
+    virDomainObjPtr vm;
+    bool remove_dom = false;
+
+    virCheckFlags(0, -1);
+    if (dxml) {
+        virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
+                       _("xml modification unsupported"));
+        return -1;
+    }
+
+    if (!(vm = lxcDomObjFromDomain(dom)))
+        goto cleanup;
+
+    if (virDomainSaveFlagsEnsureACL(dom->conn, vm->def) < 0)
+        goto cleanup;
+
+    if (virLXCDomainObjBeginJob(driver, vm, LXC_JOB_MODIFY) < 0)
+        goto cleanup;
+
+    if (!virDomainObjIsActive(vm)) {
+        virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                       _("Domain is not running"));
+        goto endjob;
+    }
+
+    if (lxcDoDomainSave(driver, vm, to) < 0)
+        goto endjob;
+
+    if (!vm->persistent)
+        remove_dom = true;
+
+    ret = 0;
+
+ endjob:
+    virLXCDomainObjEndJob(driver, vm);
+
+ cleanup:
+    if (remove_dom && vm)
+        virDomainObjListRemove(driver->domains, vm);
+    virDomainObjEndAPI(&vm);
+    return ret;
+}
+
+static int
+lxcDomainSave(virDomainPtr dom, const char *to)
+{
+    return lxcDomainSaveFlags(dom, to, NULL, 0);
+}
+
+static int
+lxcDomainRestoreFlags(virConnectPtr conn, const char *from,
+                      const char* dxml, unsigned int flags)
+{
+    virLXCDriverPtr driver = conn->privateData;
+    virDomainObjPtr vm = NULL;
+    virDomainDefPtr def = NULL;
+    virCapsPtr caps = NULL;
+    int ret = -1;
+    int restorefd;
+    char *xml = NULL;
+    u_int32_t xmlLen = 0;
+    int fd;
+    char *xml_image_path = NULL;
+    char c;
+
+    virCheckFlags(0, -1);
+    if (dxml) {
+        virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
+                       _("xml modification unsupported"));
+        goto out;
+    }
+
+    if (virAsprintf(&xml_image_path, "%s/xml-image", from) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                             _("Failed to write xml image path"));
+        goto out;
+    }
+
+    if ((fd = virFileOpenAs(xml_image_path, O_RDONLY, 0, -1, -1, 0)) < 0) {
+        virReportSystemError(-fd,
+                             _("Failed to open domain image file '%s'"),
+                             xml_image_path);
+        goto out;
+    }
+
+    while ((saferead(fd,  &c, 1) == 1) &&  c != '\n')
+        xmlLen = xmlLen*10 + c - '0';
+    xmlLen--;
+
+    if (VIR_ALLOC_N(xml, xmlLen) < 0)
+        goto cleanup;
+
+    if (saferead(fd, xml, xmlLen) != xmlLen) {
+        virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("failed to read XML"));
+        goto cleanup;
+    }
+
+    if (!xml) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("no domain XML parsed"));
+        goto cleanup;
+    }
+
+    if (!(caps = virLXCDriverGetCapabilities(driver, false)))
+        goto cleanup;
+
+    if (!(def = virDomainDefParseString(xml, caps, driver->xmlopt,
+                                        NULL, VIR_DOMAIN_DEF_PARSE_INACTIVE)))
+        goto cleanup;
+
+    if (virDomainRestoreFlagsEnsureACL(conn, def) < 0)
+        goto cleanup;
+
+    if (!(vm = virDomainObjListAdd(driver->domains, def,
+                                   driver->xmlopt,
+                                   VIR_DOMAIN_OBJ_LIST_ADD_LIVE |
+                                   VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE,
+                                   NULL)))
+        goto cleanup;
+    def = NULL;
+
+    if (virLXCDomainObjBeginJob(driver, vm, LXC_JOB_MODIFY) < 0) {
+        if (!vm->persistent)
+            virDomainObjListRemove(driver->domains, vm);
+        goto cleanup;
+    }
+
+    restorefd = open(from, O_DIRECTORY);
+    if (restorefd < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       "%s", _("Can't open images dir"));
+        if (!vm->persistent)
+            virDomainObjListRemove(driver->domains, vm);
+        virLXCDomainObjEndJob(driver, vm);
+        goto cleanup;
+    }
+
+    ret = virLXCProcessStart(conn, driver, vm,
+                             0, NULL,
+                             0, restorefd,
+                             VIR_DOMAIN_RUNNING_RESTORED);
+
+    VIR_FORCE_CLOSE(restorefd);
+
+    if (ret < 0 && !vm->persistent)
+        virDomainObjListRemove(driver->domains, vm);
+
+ virLXCDomainObjEndJob(driver, vm);
+ cleanup:
+    VIR_FORCE_CLOSE(fd);
+ out:
+    virDomainDefFree(def);
+    VIR_FREE(xml_image_path);
+    VIR_FREE(xml);
+    virDomainObjEndAPI(&vm);
+    return ret;
+}
+
+static int
+lxcDomainRestore(virConnectPtr conn, const char *from)
+{
+    return lxcDomainRestoreFlags(conn, from, NULL, 0);
+}
+
 static int
 lxcDomainOpenConsole(virDomainPtr dom,
                       const char *dev_name,
@@ -5550,6 +5780,10 @@ static virHypervisorDriver lxcHypervisorDriver = {
     .domainLookupByName = lxcDomainLookupByName, /* 0.4.2 */
     .domainSuspend = lxcDomainSuspend, /* 0.7.2 */
     .domainResume = lxcDomainResume, /* 0.7.2 */
+    .domainSave = lxcDomainSave, /* x.x.x */
+    .domainSaveFlags = lxcDomainSaveFlags, /* x.x.x */
+    .domainRestore = lxcDomainRestore, /* x.x.x */
+    .domainRestoreFlags = lxcDomainRestoreFlags, /* x.x.x */
     .domainDestroy = lxcDomainDestroy, /* 0.4.4 */
     .domainDestroyFlags = lxcDomainDestroyFlags, /* 0.9.4 */
     .domainGetOSType = lxcDomainGetOSType, /* 0.4.2 */
-- 
2.14.3

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list