Ian Jackson
2012-May-17 18:53 UTC
[RFC PATCH 0/5] libxl: domain save/restore: run in a separate process
This is a first cut of my proposal to fix the libxl save/restore machinery so that it is suitable for use in a single-process daemon. See 5/5 for the full rationale. Note that i have not executed this series, although I have compiled it. Nor have I reread the patches particularly carefully. This is an initial posting to get comments on the approach. 1/5 libxc: xc_domain_restore, make toolstack_restore const-correct 2/5 libxl: domain save: rename variables etc. 3/5 libxl: domain restore: reshuffle, preparing for ao 4/5 libxl: domain save: API changes for asynchrony 5/5 libxl: domain save/restore: run in a separate process These are also available as the final few commits on the following git branch: http://xenbits.xen.org/gitweb/?p=people/iwj/xen-unstable.git;a=shortlog;h=refs/heads/rebasing.ao-save-restore Note that this branch is REBASING and has a DISJOINT HISTORY to most of the other xen-unstable.git trees on xenbits. "master" in that repo is the xen-unstable tip (non-rebasing). It also contains my topgit forest; the tip of this series is the patch t/xen/xc.save-restore-protocol.
Ian Jackson
2012-May-17 18:53 UTC
[PATCH 1/5] libxc: xc_domain_restore, make toolstack_restore const-correct
Also provide typedefs for the nontrivial function callback types.
Update the one provider of this callback, in libxl.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
tools/libxc/xenguest.h | 14 ++++++++++----
tools/libxl/libxl_dom.c | 4 ++--
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/tools/libxc/xenguest.h b/tools/libxc/xenguest.h
index 6435f65..ddf9a0b 100644
--- a/tools/libxc/xenguest.h
+++ b/tools/libxc/xenguest.h
@@ -31,6 +31,10 @@
#define X86_64_B_SIZE 64
#define X86_32_B_SIZE 32
+typedef int xc_switch_qemu_logdirty_cb(int domid, unsigned enable, void *data);
+typedef int xc_toolstack_save_cb(uint32_t domid, uint8_t **buf,
+ uint32_t *len, void *data);
+
/* callbacks provided by xc_domain_save */
struct save_callbacks {
int (*suspend)(void* data);
@@ -42,7 +46,7 @@ struct save_callbacks {
int (*checkpoint)(void* data);
/* Enable qemu-dm logging dirty pages to xen */
- int (*switch_qemu_logdirty)(int domid, unsigned enable, void *data); /* HVM
only */
+ xc_switch_qemu_logdirty_cb *switch_qemu_logdirty; /* HVM only */
/* Save toolstack specific data
* @param buf the buffer with the data to be saved
@@ -50,7 +54,7 @@ struct save_callbacks {
* The callee allocates the buffer, the caller frees it (buffer must
* be free''able).
*/
- int (*toolstack_save)(uint32_t domid, uint8_t **buf, uint32_t *len, void
*data);
+ xc_toolstack_save_cb *toolstack_save;
/* to be provided as the last argument to each callback function */
void* data;
@@ -70,11 +74,13 @@ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t
dom, uint32_t max_iter
unsigned long vm_generationid_addr);
+typedef int xc_toolstack_restore_cb(uint32_t domid, const uint8_t *buf,
+ uint32_t size, void* data);
+
/* callbacks provided by xc_domain_restore */
struct restore_callbacks {
/* callback to restore toolstack specific data */
- int (*toolstack_restore)(uint32_t domid, uint8_t *buf,
- uint32_t size, void* data);
+ xc_toolstack_restore_cb *toolstack_restore;
/* to be provided as the last argument to each callback function */
void* data;
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 6064d44..01d5595 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -461,13 +461,13 @@ static inline char *restore_helper(libxl__gc *gc, uint32_t
domid,
domid, phys_offset, node);
}
-static int libxl__toolstack_restore(uint32_t domid, uint8_t *buf,
+static int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
uint32_t size, void *data)
{
libxl__gc *gc = (libxl__gc *) data;
libxl_ctx *ctx = gc->owner;
int i, ret;
- uint8_t *ptr = buf;
+ const uint8_t *ptr = buf;
uint32_t count = 0, version = 0;
struct libxl__physmap_info* pi;
char *xs_path;
--
1.7.2.5
Preparatory work for making domain suspend asynchronous:
* Rename `struct suspendinfo'' to `libxl__domain_suspend_state''
and move it to libxl_internal.h.
* Rename variables `si'' to `dss''.
* Change the stack-allocated state from
struct suspendinfo si;
to
libxl__domain_suspend_state dss[1];
so that it may be referred to as a pointer variable everywhere.
* Rename the variable `flags'' (in libxl__domain_suspend_common
and in libxl__domain_suspend_state) to `xcflags'', to help
distinguish it from the other `flags'' which is passed in
from the calling application in libxl_domain_suspend_info.
* Move the prototypes of suspend-related functions in libxl_internal.h
to after the definition of the state struct.
* Replace several ctx variables with gc variables and
consequently references to ctx with CTX. Change references
to `dss->gc'' in the functional code to simply `gc''.
* Use LOG* rather than LIBXL__LOG* in a number of places.
* In libxl__domain_save_device_model use `ret'' instead of
`rc''.
* Introduce and use `gc'' and `domid'' in
libxl__domain_suspend_common_callback.
* Wrap some long lines.
* Add an extra pair of parens for clarity in a flag test.
* Remove two pointless casts from void* to a struct*.
No functional change whatsoever.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
tools/libxl/libxl_dom.c | 202 ++++++++++++++++++++---------------------
tools/libxl/libxl_internal.h | 25 +++++-
2 files changed, 121 insertions(+), 106 deletions(-)
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 01d5595..0dc1427 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -464,7 +464,7 @@ static inline char *restore_helper(libxl__gc *gc, uint32_t
domid,
static int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
uint32_t size, void *data)
{
- libxl__gc *gc = (libxl__gc *) data;
+ libxl__gc *gc = data;
libxl_ctx *ctx = gc->owner;
int i, ret;
const uint8_t *ptr = buf;
@@ -558,84 +558,79 @@ int libxl__domain_restore_common(libxl__gc *gc, uint32_t
domid,
return 0;
}
-struct suspendinfo {
- libxl__gc *gc;
- xc_evtchn *xce; /* event channel handle */
- int suspend_eventchn;
- int domid;
- int hvm;
- unsigned int flags;
- int guest_responded;
-};
-
-static int libxl__domain_suspend_common_switch_qemu_logdirty(int domid,
unsigned int enable, void *data)
+static int libxl__domain_suspend_common_switch_qemu_logdirty
+ (int domid, unsigned int enable, void *data)
{
- struct suspendinfo *si = data;
- libxl_ctx *ctx = libxl__gc_owner(si->gc);
+ libxl__domain_suspend_state *dss = data;
+ libxl__gc *gc = dss->gc;
char *path;
bool rc;
- path = libxl__sprintf(si->gc,
"/local/domain/0/device-model/%u/logdirty/cmd", domid);
+ path = libxl__sprintf(gc,
+ "/local/domain/0/device-model/%u/logdirty/cmd",
domid);
if (!path)
return 1;
if (enable)
- rc = xs_write(ctx->xsh, XBT_NULL, path, "enable",
strlen("enable"));
+ rc = xs_write(CTX->xsh, XBT_NULL, path, "enable",
strlen("enable"));
else
- rc = xs_write(ctx->xsh, XBT_NULL, path, "disable",
strlen("disable"));
+ rc = xs_write(CTX->xsh, XBT_NULL, path, "disable",
strlen("disable"));
return rc ? 0 : 1;
}
static int libxl__domain_suspend_common_callback(void *data)
{
- struct suspendinfo *si = data;
+ libxl__domain_suspend_state *dss = data;
+ libxl__gc *gc = dss->gc;
unsigned long hvm_s_state = 0, hvm_pvdrv = 0;
int ret;
char *state = "suspend";
int watchdog;
- libxl_ctx *ctx = libxl__gc_owner(si->gc);
xs_transaction_t t;
- if (si->hvm) {
- xc_get_hvm_param(ctx->xch, si->domid, HVM_PARAM_CALLBACK_IRQ,
&hvm_pvdrv);
- xc_get_hvm_param(ctx->xch, si->domid, HVM_PARAM_ACPI_S_STATE,
&hvm_s_state);
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+
+ if (dss->hvm) {
+ xc_get_hvm_param(CTX->xch, domid, HVM_PARAM_CALLBACK_IRQ,
&hvm_pvdrv);
+ xc_get_hvm_param(CTX->xch, domid, HVM_PARAM_ACPI_S_STATE,
&hvm_s_state);
}
- if ((hvm_s_state == 0) && (si->suspend_eventchn >= 0)) {
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "issuing %s suspend request via
event channel",
- si->hvm ? "PVHVM" : "PV");
- ret = xc_evtchn_notify(si->xce, si->suspend_eventchn);
+ if ((hvm_s_state == 0) && (dss->suspend_eventchn >= 0)) {
+ LOG(DEBUG, "issuing %s suspend request via event channel",
+ dss->hvm ? "PVHVM" : "PV");
+ ret = xc_evtchn_notify(dss->xce, dss->suspend_eventchn);
if (ret < 0) {
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "xc_evtchn_notify failed
ret=%d", ret);
+ LOG(ERROR, "xc_evtchn_notify failed ret=%d", ret);
return 0;
}
- ret = xc_await_suspend(ctx->xch, si->xce,
si->suspend_eventchn);
+ ret = xc_await_suspend(CTX->xch, dss->xce,
dss->suspend_eventchn);
if (ret < 0) {
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "xc_await_suspend failed
ret=%d", ret);
+ LOG(ERROR, "xc_await_suspend failed ret=%d", ret);
return 0;
}
- si->guest_responded = 1;
+ dss->guest_responded = 1;
return 1;
}
- if (si->hvm && (!hvm_pvdrv || hvm_s_state)) {
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "Calling xc_domain_shutdown on
HVM domain");
- xc_domain_shutdown(ctx->xch, si->domid, SHUTDOWN_suspend);
+ if (dss->hvm && (!hvm_pvdrv || hvm_s_state)) {
+ LOG(DEBUG, "Calling xc_domain_shutdown on HVM domain");
+ xc_domain_shutdown(CTX->xch, domid, SHUTDOWN_suspend);
/* The guest does not (need to) respond to this sort of request. */
- si->guest_responded = 1;
+ dss->guest_responded = 1;
} else {
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "issuing %s suspend request via
XenBus control node",
- si->hvm ? "PVHVM" : "PV");
+ LOG(DEBUG, "issuing %s suspend request via XenBus control
node",
+ dss->hvm ? "PVHVM" : "PV");
- libxl__domain_pvcontrol_write(si->gc, XBT_NULL, si->domid,
"suspend");
+ libxl__domain_pvcontrol_write(gc, XBT_NULL, domid,
"suspend");
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "wait for the guest to
acknowledge suspend request");
+ LOG(DEBUG, "wait for the guest to acknowledge suspend
request");
watchdog = 60;
while (!strcmp(state, "suspend") && watchdog > 0)
{
usleep(100000);
- state = libxl__domain_pvcontrol_read(si->gc, XBT_NULL,
si->domid);
+ state = libxl__domain_pvcontrol_read(gc, XBT_NULL, domid);
if (!state) state = "";
watchdog--;
@@ -651,17 +646,17 @@ static int libxl__domain_suspend_common_callback(void
*data)
* at the last minute.
*/
if (!strcmp(state, "suspend")) {
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "guest didn''t
acknowledge suspend, cancelling request");
+ LOG(ERROR, "guest didn''t acknowledge suspend,
cancelling request");
retry_transaction:
- t = xs_transaction_start(ctx->xsh);
+ t = xs_transaction_start(CTX->xsh);
- state = libxl__domain_pvcontrol_read(si->gc, t, si->domid);
+ state = libxl__domain_pvcontrol_read(gc, t, domid);
if (!state) state = "";
if (!strcmp(state, "suspend"))
- libxl__domain_pvcontrol_write(si->gc, t, si->domid,
"");
+ libxl__domain_pvcontrol_write(gc, t, domid, "");
- if (!xs_transaction_end(ctx->xsh, t, 0))
+ if (!xs_transaction_end(CTX->xsh, t, 0))
if (errno == EAGAIN)
goto retry_transaction;
@@ -673,27 +668,29 @@ static int libxl__domain_suspend_common_callback(void
*data)
* case we lost the race while cancelling and should continue.
*/
if (!strcmp(state, "suspend")) {
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "guest didn''t
acknowledge suspend, request cancelled");
+ LOG(ERROR, "guest didn''t acknowledge suspend, request
cancelled");
return 0;
}
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "guest acknowledged suspend
request");
- si->guest_responded = 1;
+ LOG(DEBUG, "guest acknowledged suspend request");
+ dss->guest_responded = 1;
}
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "wait for the guest to
suspend");
+ LOG(DEBUG, "wait for the guest to suspend");
watchdog = 60;
while (watchdog > 0) {
xc_domaininfo_t info;
usleep(100000);
- ret = xc_domain_getinfolist(ctx->xch, si->domid, 1, &info);
- if (ret == 1 && info.domain == si->domid &&
info.flags & XEN_DOMINF_shutdown) {
+ ret = xc_domain_getinfolist(CTX->xch, domid, 1, &info);
+ if (ret == 1 && info.domain == domid &&
+ (info.flags & XEN_DOMINF_shutdown)) {
int shutdown_reason;
- shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
& XEN_DOMINF_shutdownmask;
+ shutdown_reason = (info.flags >> XEN_DOMINF_shutdownshift)
+ & XEN_DOMINF_shutdownmask;
if (shutdown_reason == SHUTDOWN_suspend) {
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "guest has
suspended");
+ LOG(DEBUG, "guest has suspended");
return 1;
}
}
@@ -701,7 +698,7 @@ static int libxl__domain_suspend_common_callback(void *data)
watchdog--;
}
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "guest did not suspend");
+ LOG(ERROR, "guest did not suspend");
return 0;
}
@@ -716,9 +713,8 @@ static inline char *save_helper(libxl__gc *gc, uint32_t
domid,
static int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
uint32_t *len, void *data)
{
- struct suspendinfo *si = (struct suspendinfo *) data;
- libxl__gc *gc = (libxl__gc *) si->gc;
- libxl_ctx *ctx = gc->owner;
+ libxl__domain_suspend_state *dss = data;
+ libxl__gc *gc = dss->gc;
int i = 0;
char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
unsigned int num = 0;
@@ -747,21 +743,21 @@ static int libxl__toolstack_save(uint32_t domid, uint8_t
**buf,
char *xs_path;
phys_offset = entries[i];
if (phys_offset == NULL) {
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "phys_offset %d is
NULL", i);
+ LOG(ERROR, "phys_offset %d is NULL", i);
return -1;
}
xs_path = save_helper(gc, domid, phys_offset, "start_addr");
start_addr = libxl__xs_read(gc, 0, xs_path);
if (start_addr == NULL) {
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "%s is NULL", xs_path);
+ LOG(ERROR, "%s is NULL", xs_path);
return -1;
}
xs_path = save_helper(gc, domid, phys_offset, "size");
size = libxl__xs_read(gc, 0, xs_path);
if (size == NULL) {
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "%s is NULL", xs_path);
+ LOG(ERROR, "%s is NULL", xs_path);
return -1;
}
@@ -793,11 +789,10 @@ int libxl__domain_suspend_common(libxl__gc *gc, uint32_t
domid, int fd,
libxl_domain_type type,
int live, int debug)
{
- libxl_ctx *ctx = libxl__gc_owner(gc);
- int flags;
+ int xcflags;
int port;
struct save_callbacks callbacks;
- struct suspendinfo si;
+ libxl__domain_suspend_state dss[1];
int hvm, rc = ERROR_FAIL;
unsigned long vm_generationid_addr;
@@ -822,29 +817,30 @@ int libxl__domain_suspend_common(libxl__gc *gc, uint32_t
domid, int fd,
return ERROR_INVAL;
}
- flags = (live) ? XCFLAGS_LIVE : 0
+ xcflags = (live) ? XCFLAGS_LIVE : 0
| (debug) ? XCFLAGS_DEBUG : 0
| (hvm) ? XCFLAGS_HVM : 0;
- si.domid = domid;
- si.flags = flags;
- si.hvm = hvm;
- si.gc = gc;
- si.suspend_eventchn = -1;
- si.guest_responded = 0;
+ dss->domid = domid;
+ dss->xcflags = xcflags;
+ dss->hvm = hvm;
+ dss->gc = gc;
+ dss->suspend_eventchn = -1;
+ dss->guest_responded = 0;
- si.xce = xc_evtchn_open(NULL, 0);
- if (si.xce == NULL)
+ dss->xce = xc_evtchn_open(NULL, 0);
+ if (dss->xce == NULL)
goto out;
else
{
- port = xs_suspend_evtchn_port(si.domid);
+ port = xs_suspend_evtchn_port(dss->domid);
if (port >= 0) {
- si.suspend_eventchn = xc_suspend_evtchn_init(ctx->xch, si.xce,
si.domid, port);
+ dss->suspend_eventchn +
xc_suspend_evtchn_init(CTX->xch, dss->xce, dss->domid, port);
- if (si.suspend_eventchn < 0)
- LIBXL__LOG(ctx, LIBXL__LOG_WARNING, "Suspend event channel
initialization failed");
+ if (dss->suspend_eventchn < 0)
+ LOG(WARN, "Suspend event channel initialization
failed");
}
}
@@ -852,25 +848,26 @@ int libxl__domain_suspend_common(libxl__gc *gc, uint32_t
domid, int fd,
callbacks.suspend = libxl__domain_suspend_common_callback;
callbacks.switch_qemu_logdirty =
libxl__domain_suspend_common_switch_qemu_logdirty;
callbacks.toolstack_save = libxl__toolstack_save;
- callbacks.data = &si;
+ callbacks.data = dss;
- rc = xc_domain_save(ctx->xch, fd, domid, 0, 0, flags, &callbacks,
+ rc = xc_domain_save(CTX->xch, fd, domid, 0, 0, xcflags, &callbacks,
hvm, vm_generationid_addr);
if ( rc ) {
- LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "saving domain: %s",
- si.guest_responded ?
+ LOGE(ERROR, "saving domain: %s",
+ dss->guest_responded ?
"domain responded to suspend request" :
"domain did not respond to suspend
request");
- if ( !si.guest_responded )
+ if ( !dss->guest_responded )
rc = ERROR_GUEST_TIMEDOUT;
else
rc = ERROR_FAIL;
}
- if (si.suspend_eventchn > 0)
- xc_suspend_evtchn_release(ctx->xch, si.xce, domid,
si.suspend_eventchn);
- if (si.xce != NULL)
- xc_evtchn_close(si.xce);
+ if (dss->suspend_eventchn > 0)
+ xc_suspend_evtchn_release(CTX->xch, dss->xce, domid,
+ dss->suspend_eventchn);
+ if (dss->xce != NULL)
+ xc_evtchn_close(dss->xce);
out:
return rc;
@@ -878,8 +875,7 @@ out:
int libxl__domain_save_device_model(libxl__gc *gc, uint32_t domid, int fd)
{
- libxl_ctx *ctx = libxl__gc_owner(gc);
- int ret, fd2 = -1, c;
+ int rc, fd2 = -1, c;
char buf[1024];
const char *filename = libxl__device_model_savefile(gc, domid);
struct stat st;
@@ -887,15 +883,15 @@ int libxl__domain_save_device_model(libxl__gc *gc,
uint32_t domid, int fd)
switch (libxl__device_model_version_running(gc, domid)) {
case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: {
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG,
+ LOG(DEBUG,
"Saving device model state to %s", filename);
libxl__qemu_traditional_cmd(gc, domid, "save");
libxl__wait_for_device_model(gc, domid, "paused", NULL, NULL,
NULL);
break;
}
case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
- ret = libxl__qmp_save(gc, domid, (char *)filename);
- if (ret)
+ rc = libxl__qmp_save(gc, domid, (char *)filename);
+ if (rc)
goto out;
break;
default:
@@ -904,46 +900,46 @@ int libxl__domain_save_device_model(libxl__gc *gc,
uint32_t domid, int fd)
if (stat(filename, &st) < 0)
{
- LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "Unable to stat qemu save
file\n");
- ret = ERROR_FAIL;
+ LOG(ERROR, "Unable to stat qemu save file\n");
+ rc = ERROR_FAIL;
goto out;
}
qemu_state_len = st.st_size;
- LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "Qemu state is %d bytes\n",
qemu_state_len);
+ LOG(DEBUG, "Qemu state is %d bytes\n", qemu_state_len);
- ret = libxl_write_exactly(ctx, fd, QEMU_SIGNATURE, strlen(QEMU_SIGNATURE),
+ rc = libxl_write_exactly(CTX, fd, QEMU_SIGNATURE, strlen(QEMU_SIGNATURE),
"saved-state file", "qemu
signature");
- if (ret)
+ if (rc)
goto out;
- ret = libxl_write_exactly(ctx, fd, &qemu_state_len,
sizeof(qemu_state_len),
+ rc = libxl_write_exactly(CTX, fd, &qemu_state_len,
sizeof(qemu_state_len),
"saved-state file", "saved-state
length");
- if (ret)
+ if (rc)
goto out;
fd2 = open(filename, O_RDONLY);
if (fd2 < 0) {
- LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "Unable to open qemu save
file\n");
+ LOGE(ERROR, "Unable to open qemu save file\n");
goto out;
}
while ((c = read(fd2, buf, sizeof(buf))) != 0) {
if (c < 0) {
if (errno == EINTR)
continue;
- ret = errno;
+ rc = errno;
goto out;
}
- ret = libxl_write_exactly(
- ctx, fd, buf, c, "saved-state file", "qemu
state");
- if (ret)
+ rc = libxl_write_exactly(
+ CTX, fd, buf, c, "saved-state file", "qemu
state");
+ if (rc)
goto out;
}
- ret = 0;
+ rc = 0;
out:
if (fd2 >= 0) close(fd2);
unlink(filename);
- return ret;
+ return rc;
}
char *libxl__uuid2string(libxl__gc *gc, const libxl_uuid uuid)
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 73b9915..b9afeb8 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -755,9 +755,6 @@ _hidden int libxl__domain_restore_common(libxl__gc *gc,
uint32_t domid,
libxl_domain_build_info *info,
libxl__domain_build_state *state,
int fd);
-_hidden int libxl__domain_suspend_common(libxl__gc *gc, uint32_t domid, int fd,
- libxl_domain_type type,
- int live, int debug);
_hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t
domid);
_hidden int libxl__domain_save_device_model(libxl__gc *gc, uint32_t domid, int
fd);
_hidden void libxl__userdata_destroyall(libxl__gc *gc, uint32_t domid);
@@ -1708,6 +1705,21 @@ _hidden void
libxl__datacopier_kill(libxl__datacopier_state *dc);
_hidden int libxl__datacopier_start(libxl__datacopier_state *dc);
+/*----- Domain suspend (save) state structure -----*/
+
+typedef struct libxl__domain_suspend_state libxl__domain_suspend_state;
+
+struct libxl__domain_suspend_state {
+ libxl__gc *gc;
+ xc_evtchn *xce; /* event channel handle */
+ int suspend_eventchn;
+ int domid;
+ int hvm;
+ unsigned int xcflags;
+ int guest_responded;
+};
+
+
/*----- openpty -----*/
/*
@@ -1803,6 +1815,13 @@ struct libxl__domain_create_state {
* for the non-stubdom device model. */
};
+/*----- Domain suspend (save) functions -----*/
+
+_hidden int libxl__domain_suspend_common(libxl__gc *gc, uint32_t domid, int fd,
+ libxl_domain_type type,
+ int live, int debug);
+
+
/*
* Convenience macros.
--
1.7.2.5
Ian Jackson
2012-May-17 18:53 UTC
[PATCH 3/5] libxl: domain restore: reshuffle, preparing for ao
We are going to arrange that libxl, instead of calling
xc_domain_restore, calls a stub function which forks and execs a
helper program, so that restore can be asynchronous rather than
blocking the whole toolstack.
This stub function will be called libxl__xc_domain_restore.
However, its prospective call site is unsuitable for a function which
needs to make a callback, and is buried in two nested single-call-site
functions which are logically part of the domain creation procedure.
So we first abolish those single-call-site functions, integrate their
contents into domain creation in their proper temporal order, and
break out libxl__xc_domain_restore ready for its reimplementation.
No functional change - just the following reorganisation:
* Abolish libxl__domain_restore_common, as it had only one caller.
Move its contents into (what was) domain_restore.
* There is a new stage function domcreate_rebuild_done containing what
used to be the bulk of domcreate_bootloader_done, since
domcreate_bootloader_done now simply starts the restore (or does the
rebuild) and arranges to call the next stage.
* Move the contents of domain_restore into its correct place in the
domain creation sequence. We put it inside
domcreate_bootloader_done, which now either calls
libxl__xc_domain_restore which will call the new function
domcreate_rebuild_done.
* Various general-purpose local variables (`i'' etc.) and convenience
alias variables need to be shuffled about accordingly.
* Consequently libxl__toolstack_restore needs to gain external linkage
as it is now in a different file to its user.
In general the moved code remains almost identical. Two returns in
what used to be libxl__domain_restore_common have been changed to set
the return value and "goto out", and the call sites for the abolished
and new functions have been adjusted.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
tools/libxl/Makefile | 1 +
tools/libxl/libxl_create.c | 245 ++++++++++++++++++++++++--------------
tools/libxl/libxl_dom.c | 45 +-------
tools/libxl/libxl_internal.h | 19 +++-
tools/libxl/libxl_save_callout.c | 43 +++++++
5 files changed, 213 insertions(+), 140 deletions(-)
create mode 100644 tools/libxl/libxl_save_callout.c
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 5d9227e..9b84421 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -66,6 +66,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o
libxl_pci.o \
libxl_dom.o libxl_exec.o libxl_xshelp.o libxl_device.o \
libxl_internal.o libxl_utils.o libxl_uuid.o \
libxl_json.o libxl_aoutils.o \
+ libxl_save_callout.o \
libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 14721eb..7f42737 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -20,7 +20,6 @@
#include "libxl_internal.h"
#include <xc_dom.h>
-#include <xenguest.h>
void libxl_domain_config_init(libxl_domain_config *d_config)
{
@@ -311,89 +310,6 @@ out:
return ret;
}
-static int domain_restore(libxl__gc *gc, libxl_domain_build_info *info,
- uint32_t domid, int fd,
- libxl__domain_build_state *state)
-{
- libxl_ctx *ctx = libxl__gc_owner(gc);
- char **vments = NULL, **localents = NULL;
- struct timeval start_time;
- int i, ret, esave, flags;
-
- ret = libxl__build_pre(gc, domid, info, state);
- if (ret)
- goto out;
-
- ret = libxl__domain_restore_common(gc, domid, info, state, fd);
- if (ret)
- goto out;
-
- gettimeofday(&start_time, NULL);
-
- switch (info->type) {
- case LIBXL_DOMAIN_TYPE_HVM:
- vments = libxl__calloc(gc, 7, sizeof(char *));
- vments[0] = "rtc/timeoffset";
- vments[1] = (info->u.hvm.timeoffset) ? info->u.hvm.timeoffset :
"";
- vments[2] = "image/ostype";
- vments[3] = "hvm";
- vments[4] = "start_time";
- vments[5] = libxl__sprintf(gc, "%lu.%02d",
start_time.tv_sec,(int)start_time.tv_usec/10000);
- break;
- case LIBXL_DOMAIN_TYPE_PV:
- vments = libxl__calloc(gc, 11, sizeof(char *));
- i = 0;
- vments[i++] = "image/ostype";
- vments[i++] = "linux";
- vments[i++] = "image/kernel";
- vments[i++] = (char*) info->u.pv.kernel.path;
- vments[i++] = "start_time";
- vments[i++] = libxl__sprintf(gc, "%lu.%02d",
start_time.tv_sec,(int)start_time.tv_usec/10000);
- if (info->u.pv.ramdisk.path) {
- vments[i++] = "image/ramdisk";
- vments[i++] = (char*) info->u.pv.ramdisk.path;
- }
- if (info->u.pv.cmdline) {
- vments[i++] = "image/cmdline";
- vments[i++] = (char*) info->u.pv.cmdline;
- }
- break;
- default:
- ret = ERROR_INVAL;
- goto out;
- }
- ret = libxl__build_post(gc, domid, info, state, vments, localents);
- if (ret)
- goto out;
-
- if (info->type == LIBXL_DOMAIN_TYPE_HVM) {
- ret = asprintf(&state->saved_state,
- XC_DEVICE_MODEL_RESTORE_FILE".%d", domid);
- ret = (ret < 0) ? ERROR_FAIL : 0;
- }
-
-out:
- if (info->type == LIBXL_DOMAIN_TYPE_PV) {
- libxl__file_reference_unmap(&info->u.pv.kernel);
- libxl__file_reference_unmap(&info->u.pv.ramdisk);
- }
-
- esave = errno;
-
- flags = fcntl(fd, F_GETFL);
- if (flags == -1) {
- LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "unable to get flags on
restore fd");
- } else {
- flags &= ~O_NONBLOCK;
- if (fcntl(fd, F_SETFL, flags) == -1)
- LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "unable to put restore
fd"
- " back to blocking mode");
- }
-
- errno = esave;
- return ret;
-}
-
int libxl__domain_make(libxl__gc *gc, libxl_domain_create_info *info,
uint32_t *domid)
{
@@ -574,10 +490,13 @@ static void
domcreate_bootloader_console_available(libxl__egc *egc,
static void domcreate_bootloader_done(libxl__egc *egc,
libxl__bootloader_state *bl,
int rc);
-
static void domcreate_console_available(libxl__egc *egc,
libxl__domain_create_state *dcs);
+static void domcreate_rebuild_done(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int ret);
+
/* Our own function to clean up and call the user''s callback.
* The final call in the sequence. */
static void domcreate_complete(libxl__egc *egc,
@@ -660,20 +579,19 @@ static void domcreate_console_available(libxl__egc *egc,
static void domcreate_bootloader_done(libxl__egc *egc,
libxl__bootloader_state *bl,
- int ret)
+ int rc)
{
libxl__domain_create_state *dcs = CONTAINER_OF(bl, *dcs, bl);
STATE_AO_GC(bl->ao);
- int i;
/* convenience aliases */
const uint32_t domid = dcs->guest_domid;
libxl_domain_config *const d_config = dcs->guest_config;
+ libxl_domain_build_info *const info = &d_config->b_info;
const int restore_fd = dcs->restore_fd;
libxl__domain_build_state *const state = &dcs->build_state;
- libxl_ctx *const ctx = CTX;
- if (ret) goto error_out;
+ if (rc) domcreate_rebuild_done(egc, dcs, rc);
/* We might be going to call libxl__spawn_local_dm, or _spawn_stub_dm.
* Fill in any field required by either, including both relevant
@@ -684,12 +602,155 @@ static void domcreate_bootloader_done(libxl__egc *egc,
dcs->dmss.dm.callback = domcreate_devmodel_started;
dcs->dmss.callback = domcreate_devmodel_started;
- if ( restore_fd >= 0 ) {
- ret = domain_restore(gc, &d_config->b_info, domid, restore_fd,
state);
+ if ( restore_fd < 0 ) {
+ rc = libxl__domain_build(gc, &d_config->b_info, domid, state);
+ domcreate_rebuild_done(egc, dcs, rc);
+ return;
+ }
+
+ /* Restore */
+
+ rc = libxl__build_pre(gc, domid, info, state);
+ if (rc)
+ goto out;
+
+ /* read signature */
+ int hvm, pae, superpages;
+ int no_incr_generationid;
+ xc_toolstack_restore_cb *toolstack_restore = NULL;
+ switch (info->type) {
+ case LIBXL_DOMAIN_TYPE_HVM:
+ hvm = 1;
+ superpages = 1;
+ pae = libxl_defbool_val(info->u.hvm.pae);
+ no_incr_generationid =
!libxl_defbool_val(info->u.hvm.incr_generationid);
+ toolstack_restore = libxl__toolstack_restore;
+ break;
+ case LIBXL_DOMAIN_TYPE_PV:
+ hvm = 0;
+ superpages = 0;
+ pae = 1;
+ no_incr_generationid = 0;
+ break;
+ default:
+ rc = ERROR_INVAL;
+ goto out;
+ }
+ libxl__xc_domain_restore(egc, dcs,
+ hvm, pae, superpages, no_incr_generationid,
+ toolstack_restore);
+ return;
+
+ out:
+ libxl__xc_domain_restore_done(egc, dcs, rc, 0, 0);
+}
+
+void libxl__xc_domain_restore_done(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int ret, int retval, int errnoval)
+{
+ STATE_AO_GC(dcs->ao);
+ libxl_ctx *ctx = libxl__gc_owner(gc);
+ char **vments = NULL, **localents = NULL;
+ struct timeval start_time;
+ int i, esave, flags;
+
+ /* convenience aliases */
+ const uint32_t domid = dcs->guest_domid;
+ libxl_domain_config *const d_config = dcs->guest_config;
+ libxl_domain_build_info *const info = &d_config->b_info;
+ libxl__domain_build_state *const state = &dcs->build_state;
+ const int restore_fd = dcs->restore_fd;
+
+ if (ret)
+ goto out;
+
+ if (retval) {
+ LOGEV(ERROR, errnoval, "restoring domain");
+ ret = ERROR_FAIL;
+ goto out;
+ }
+
+ gettimeofday(&start_time, NULL);
+
+ switch (info->type) {
+ case LIBXL_DOMAIN_TYPE_HVM:
+ vments = libxl__calloc(gc, 7, sizeof(char *));
+ vments[0] = "rtc/timeoffset";
+ vments[1] = (info->u.hvm.timeoffset) ? info->u.hvm.timeoffset :
"";
+ vments[2] = "image/ostype";
+ vments[3] = "hvm";
+ vments[4] = "start_time";
+ vments[5] = libxl__sprintf(gc, "%lu.%02d",
start_time.tv_sec,(int)start_time.tv_usec/10000);
+ break;
+ case LIBXL_DOMAIN_TYPE_PV:
+ vments = libxl__calloc(gc, 11, sizeof(char *));
+ i = 0;
+ vments[i++] = "image/ostype";
+ vments[i++] = "linux";
+ vments[i++] = "image/kernel";
+ vments[i++] = (char*) info->u.pv.kernel.path;
+ vments[i++] = "start_time";
+ vments[i++] = libxl__sprintf(gc, "%lu.%02d",
start_time.tv_sec,(int)start_time.tv_usec/10000);
+ if (info->u.pv.ramdisk.path) {
+ vments[i++] = "image/ramdisk";
+ vments[i++] = (char*) info->u.pv.ramdisk.path;
+ }
+ if (info->u.pv.cmdline) {
+ vments[i++] = "image/cmdline";
+ vments[i++] = (char*) info->u.pv.cmdline;
+ }
+ break;
+ default:
+ ret = ERROR_INVAL;
+ goto out;
+ }
+ ret = libxl__build_post(gc, domid, info, state, vments, localents);
+ if (ret)
+ goto out;
+
+ if (info->type == LIBXL_DOMAIN_TYPE_HVM) {
+ ret = asprintf(&state->saved_state,
+ XC_DEVICE_MODEL_RESTORE_FILE".%d", domid);
+ ret = (ret < 0) ? ERROR_FAIL : 0;
+ }
+
+out:
+ if (info->type == LIBXL_DOMAIN_TYPE_PV) {
+ libxl__file_reference_unmap(&info->u.pv.kernel);
+ libxl__file_reference_unmap(&info->u.pv.ramdisk);
+ }
+
+ esave = errno;
+
+ flags = fcntl(restore_fd, F_GETFL);
+ if (flags == -1) {
+ LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "unable to get flags on
restore fd");
} else {
- ret = libxl__domain_build(gc, &d_config->b_info, domid, state);
+ flags &= ~O_NONBLOCK;
+ if (fcntl(restore_fd, F_SETFL, flags) == -1)
+ LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "unable to put restore
fd"
+ " back to blocking mode");
}
+ errno = esave;
+
+ domcreate_rebuild_done(egc, dcs, ret);
+}
+
+static void domcreate_rebuild_done(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int ret)
+{
+ STATE_AO_GC(dcs->ao);
+ int i;
+
+ /* convenience aliases */
+ const uint32_t domid = dcs->guest_domid;
+ libxl_domain_config *const d_config = dcs->guest_config;
+ libxl__domain_build_state *const state = &dcs->build_state;
+ libxl_ctx *const ctx = CTX;
+
if (ret) {
LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "cannot (re-)build domain:
%d", ret);
ret = ERROR_FAIL;
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 0dc1427..d29c705 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -19,7 +19,6 @@
#include <xenctrl.h>
#include <xc_dom.h>
-#include <xenguest.h>
#include <xen/hvm/hvm_info_table.h>
@@ -461,7 +460,7 @@ static inline char *restore_helper(libxl__gc *gc, uint32_t
domid,
domid, phys_offset, node);
}
-static int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
+int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
uint32_t size, void *data)
{
libxl__gc *gc = data;
@@ -516,48 +515,6 @@ static int libxl__toolstack_restore(uint32_t domid, const
uint8_t *buf,
return 0;
}
-int libxl__domain_restore_common(libxl__gc *gc, uint32_t domid,
- libxl_domain_build_info *info,
- libxl__domain_build_state *state,
- int fd)
-{
- libxl_ctx *ctx = libxl__gc_owner(gc);
- /* read signature */
- int rc;
- int hvm, pae, superpages;
- struct restore_callbacks callbacks;
- int no_incr_generationid;
- switch (info->type) {
- case LIBXL_DOMAIN_TYPE_HVM:
- hvm = 1;
- superpages = 1;
- pae = libxl_defbool_val(info->u.hvm.pae);
- no_incr_generationid =
!libxl_defbool_val(info->u.hvm.incr_generationid);
- callbacks.toolstack_restore = libxl__toolstack_restore;
- callbacks.data = gc;
- break;
- case LIBXL_DOMAIN_TYPE_PV:
- hvm = 0;
- superpages = 0;
- pae = 1;
- no_incr_generationid = 0;
- break;
- default:
- return ERROR_INVAL;
- }
- rc = xc_domain_restore(ctx->xch, fd, domid,
- state->store_port, &state->store_mfn,
- state->store_domid, state->console_port,
- &state->console_mfn, state->console_domid,
- hvm, pae, superpages, no_incr_generationid,
- &state->vm_generationid_addr,
&callbacks);
- if ( rc ) {
- LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "restoring domain");
- return ERROR_FAIL;
- }
- return 0;
-}
-
static int libxl__domain_suspend_common_switch_qemu_logdirty
(int domid, unsigned int enable, void *data)
{
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index b9afeb8..fa70a00 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -46,6 +46,7 @@
#include <xenstore.h>
#include <xenctrl.h>
+#include <xenguest.h>
#include "xentoollog.h"
@@ -751,10 +752,8 @@ _hidden int libxl__domain_rename(libxl__gc *gc, uint32_t
domid,
const char *old_name, const char *new_name,
xs_transaction_t trans);
-_hidden int libxl__domain_restore_common(libxl__gc *gc, uint32_t domid,
- libxl_domain_build_info *info,
- libxl__domain_build_state *state,
- int fd);
+_hidden int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
+ uint32_t size, void *data);
_hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t
domid);
_hidden int libxl__domain_save_device_model(libxl__gc *gc, uint32_t domid, int
fd);
_hidden void libxl__userdata_destroyall(libxl__gc *gc, uint32_t domid);
@@ -1821,6 +1820,18 @@ _hidden int libxl__domain_suspend_common(libxl__gc *gc,
uint32_t domid, int fd,
libxl_domain_type type,
int live, int debug);
+/* calls libxl__xc_domain_restore_done when done */
+_hidden void libxl__xc_domain_restore(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int hvm, int pae, int superpages,
+ int no_incr_generationid,
+ xc_toolstack_restore_cb*);
+/* If rc==0 then retval is the return value from xc_domain_save
+ * and errnoval is the errno value it provided.
+ * If rc!=0, retval and errnoval are undefined. */
+_hidden void libxl__xc_domain_restore_done(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int rc, int retval, int errnoval);
/*
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
new file mode 100644
index 0000000..bad4e30
--- /dev/null
+++ b/tools/libxl/libxl_save_callout.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2012 Citrix Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h"
+
+#include "libxl_internal.h"
+
+void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
+ int hvm, int pae, int superpages,
+ int no_incr_generationid,
+ xc_toolstack_restore_cb *toolstack_restore)
+/* calls libxl__xc_domain_restore_done when done */
+{
+ STATE_AO_GC(dcs->ao);
+
+ /* Convenience aliases */
+ const uint32_t domid = dcs->guest_domid;
+ const int restore_fd = dcs->restore_fd;
+ libxl__domain_build_state *const state = &dcs->build_state;
+
+ struct restore_callbacks callbacks;
+ callbacks.toolstack_restore = toolstack_restore;
+ callbacks.data = gc;
+
+ int r = xc_domain_restore(CTX->xch, restore_fd, domid,
+ state->store_port, &state->store_mfn,
+ state->store_domid, state->console_port,
+ &state->console_mfn,
state->console_domid,
+ hvm, pae, superpages, no_incr_generationid,
+ &state->vm_generationid_addr,
&callbacks);
+ libxl__xc_domain_restore_done(egc, dcs, 0, r, errno);
+}
--
1.7.2.5
Ian Jackson
2012-May-17 18:53 UTC
[PATCH 4/5] libxl: domain save: API changes for asynchrony
Change the internal and external APIs for domain save (suspend) to be
capable of asynchronous operation. The implementation remains
synchronous.
Public API changes:
* libxl_domain_save takes an ao_how.
* The `suspend_callback'' function passed to libxl_domain_save is
never called by the existing implementation in libxl. Abolish it.
* libxl_domain_save takes its flags parameter as an argument.
Thus libxl_domain_suspend_info is abolished.
* libxl_domain_save renamed from libxl_domain_suspend for coherency
with "restore" and general consistency with higher-level
terminology.
* XL_SUSPEND_* flags renamed to LIBXL_SAVE_*.
* Callers in xl updated.
Internal code restructuring:
* libxl__domain_suspend renamed from libxl__domain_suspend_common.
(_common here actually meant "internal function").
* libxl__domain_suspend takes a libxl__domain_suspend_state, which
where the parameters to the operation are filled in by the caller.
* xc_domain_save is now called via libxl__xc_domain_save which can
itself become asynchronous.
* Consequently, libxl__domain_suspend is split into two functions at
the callback boundary; the second half is
libxl__xc_domain_save_done.
* libxl__domain_save_device_model is now called by the actual
implementation rather than by the public wrapper. It is already in
its proper place in the domain save execution sequence. So
officially make it part of that execution sequence, renaming it to
domain_save_device_model.
* Effectively, rewrite the public wrapper function libxl_domain_save.
* Remove a needless #include <xenctrl.h>
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
tools/libxl/libxl.c | 45 ++++++++++----
tools/libxl/libxl.h | 16 ++---
tools/libxl/libxl_dom.c | 124 ++++++++++++++++++++++++++-----------
tools/libxl/libxl_internal.h | 41 +++++++++++--
tools/libxl/libxl_save_callout.c | 21 ++++++-
tools/libxl/xl_cmdimpl.c | 7 +--
6 files changed, 185 insertions(+), 69 deletions(-)
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 4d01cf8..a8dfd68 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -614,20 +614,43 @@ libxl_vminfo * libxl_list_vm(libxl_ctx *ctx, int *nb_vm)
return ptr;
}
-int libxl_domain_suspend(libxl_ctx *ctx, libxl_domain_suspend_info *info,
- uint32_t domid, int fd)
+static void domain_save_cb(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc)
{
- GC_INIT(ctx);
+ STATE_AO_GC(dss->ao);
+ libxl__ao_complete(egc,ao,rc);
+}
+
+int libxl_domain_save(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
+ const libxl_asyncop_how *ao_how)
+{
+ AO_CREATE(ctx, domid, ao_how);
+ int rc;
+
libxl_domain_type type = libxl__domain_type(gc, domid);
- int live = info != NULL && info->flags & XL_SUSPEND_LIVE;
- int debug = info != NULL && info->flags & XL_SUSPEND_DEBUG;
- int rc = 0;
+ if (type < 0) {
+ LOG(ERROR,"domain %"PRIu32": unable to determine domain
type", domid);
+ rc = ERROR_FAIL;
+ goto out_err;
+ }
- rc = libxl__domain_suspend_common(gc, domid, fd, type, live, debug);
- if (!rc && type == LIBXL_DOMAIN_TYPE_HVM)
- rc = libxl__domain_save_device_model(gc, domid, fd);
- GC_FREE;
- return rc;
+ libxl__domain_suspend_state *dss;
+ GCNEW(dss);
+
+ dss->ao = ao;
+ dss->callback = domain_save_cb;
+
+ dss->domid = domid;
+ dss->fd = fd;
+ dss->type = type;
+ dss->live = flags & LIBXL_SAVE_LIVE;
+ dss->debug = flags & LIBXL_SAVE_DEBUG;
+
+ libxl__domain_suspend(egc, dss);
+ return AO_INPROGRESS;
+
+ out_err:
+ return AO_ABORT(rc);
}
int libxl_domain_pause(libxl_ctx *ctx, uint32_t domid)
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index c86d8e7..467ef5d 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -359,13 +359,6 @@ typedef struct libxl__ctx libxl_ctx;
const libxl_version_info* libxl_get_version_info(libxl_ctx *ctx);
-typedef struct {
-#define XL_SUSPEND_DEBUG 1
-#define XL_SUSPEND_LIVE 2
- int flags;
- int (*suspend_callback)(void *, int);
-} libxl_domain_suspend_info;
-
enum {
ERROR_NONSPECIFIC = -1,
ERROR_VERSION = -2,
@@ -525,8 +518,13 @@ int libxl_domain_create_restore(libxl_ctx *ctx,
libxl_domain_config *d_config,
void libxl_domain_config_init(libxl_domain_config *d_config);
void libxl_domain_config_dispose(libxl_domain_config *d_config);
-int libxl_domain_suspend(libxl_ctx *ctx, libxl_domain_suspend_info *info,
- uint32_t domid, int fd);
+
+int libxl_domain_save(libxl_ctx *ctx, uint32_t domid, int fd,
+ int flags, /* LIBXL_SAVE_* */
+ const libxl_asyncop_how *ao_how);
+#define LIBXL_SAVE_DEBUG 1
+#define LIBXL_SAVE_LIVE 2
+
int libxl_domain_resume(libxl_ctx *ctx, uint32_t domid);
int libxl_domain_shutdown(libxl_ctx *ctx, uint32_t domid);
int libxl_domain_reboot(libxl_ctx *ctx, uint32_t domid);
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index d29c705..d28f740 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -17,13 +17,11 @@
#include <glob.h>
-#include <xenctrl.h>
-#include <xc_dom.h>
+#include "libxl_internal.h"
+#include <xc_dom.h>
#include <xen/hvm/hvm_info_table.h>
-#include "libxl_internal.h"
-
libxl_domain_type libxl__domain_type(libxl__gc *gc, uint32_t domid)
{
libxl_ctx *ctx = libxl__gc_owner(gc);
@@ -515,11 +513,20 @@ int libxl__toolstack_restore(uint32_t domid, const uint8_t
*buf,
return 0;
}
-static int libxl__domain_suspend_common_switch_qemu_logdirty
+/*==================== Domain suspend (save) ====================*/
+
+static void domain_save_device_model(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+static void domain_suspend_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc);
+
+/*----- callbacks, called by xc_domain_save -----*/
+
+int libxl__domain_suspend_common_switch_qemu_logdirty
(int domid, unsigned int enable, void *data)
{
libxl__domain_suspend_state *dss = data;
- libxl__gc *gc = dss->gc;
+ STATE_AO_GC(dss->ao);
char *path;
bool rc;
@@ -536,10 +543,10 @@ static int
libxl__domain_suspend_common_switch_qemu_logdirty
return rc ? 0 : 1;
}
-static int libxl__domain_suspend_common_callback(void *data)
+int libxl__domain_suspend_common_callback(void *data)
{
libxl__domain_suspend_state *dss = data;
- libxl__gc *gc = dss->gc;
+ STATE_AO_GC(dss->ao);
unsigned long hvm_s_state = 0, hvm_pvdrv = 0;
int ret;
char *state = "suspend";
@@ -667,11 +674,11 @@ static inline char *save_helper(libxl__gc *gc, uint32_t
domid,
domid, phys_offset, node);
}
-static int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
+int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
uint32_t *len, void *data)
{
libxl__domain_suspend_state *dss = data;
- libxl__gc *gc = dss->gc;
+ STATE_AO_GC(dss->ao);
int i = 0;
char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
unsigned int num = 0;
@@ -742,17 +749,22 @@ static int libxl__toolstack_save(uint32_t domid, uint8_t
**buf,
return 0;
}
-int libxl__domain_suspend_common(libxl__gc *gc, uint32_t domid, int fd,
- libxl_domain_type type,
- int live, int debug)
+/*----- main code for suspending, in order of execution -----*/
+
+void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss)
{
+ STATE_AO_GC(dss->ao);
int xcflags;
int port;
- struct save_callbacks callbacks;
- libxl__domain_suspend_state dss[1];
int hvm, rc = ERROR_FAIL;
unsigned long vm_generationid_addr;
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+ const libxl_domain_type type = dss->type;
+ const int live = dss->live;
+ const int debug = dss->debug;
+
switch (type) {
case LIBXL_DOMAIN_TYPE_HVM: {
char *path;
@@ -771,17 +783,14 @@ int libxl__domain_suspend_common(libxl__gc *gc, uint32_t
domid, int fd,
hvm = 0;
break;
default:
- return ERROR_INVAL;
+ abort();
}
xcflags = (live) ? XCFLAGS_LIVE : 0
| (debug) ? XCFLAGS_DEBUG : 0
| (hvm) ? XCFLAGS_HVM : 0;
- dss->domid = domid;
- dss->xcflags = xcflags;
dss->hvm = hvm;
- dss->gc = gc;
dss->suspend_eventchn = -1;
dss->guest_responded = 0;
@@ -801,16 +810,27 @@ int libxl__domain_suspend_common(libxl__gc *gc, uint32_t
domid, int fd,
}
}
- memset(&callbacks, 0, sizeof(callbacks));
- callbacks.suspend = libxl__domain_suspend_common_callback;
- callbacks.switch_qemu_logdirty =
libxl__domain_suspend_common_switch_qemu_logdirty;
- callbacks.toolstack_save = libxl__toolstack_save;
- callbacks.data = dss;
+ libxl__xc_domain_save(egc, dss, xcflags, hvm, vm_generationid_addr);
+ return;
- rc = xc_domain_save(CTX->xch, fd, domid, 0, 0, xcflags, &callbacks,
- hvm, vm_generationid_addr);
- if ( rc ) {
- LOGE(ERROR, "saving domain: %s",
+ out:
+ domain_suspend_done(egc, dss, rc);
+}
+
+void libxl__xc_domain_save_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss,
+ int rc, int retval, int errnoval)
+{
+ STATE_AO_GC(dss->ao);
+
+ /* Convenience aliases */
+ const libxl_domain_type type = dss->type;
+
+ if (rc)
+ goto out;
+
+ if (retval) {
+ LOGEV(ERROR, errnoval, "saving domain: %s",
dss->guest_responded ?
"domain responded to suspend request" :
"domain did not respond to suspend
request");
@@ -818,20 +838,29 @@ int libxl__domain_suspend_common(libxl__gc *gc, uint32_t
domid, int fd,
rc = ERROR_GUEST_TIMEDOUT;
else
rc = ERROR_FAIL;
+ goto out;
}
- if (dss->suspend_eventchn > 0)
- xc_suspend_evtchn_release(CTX->xch, dss->xce, domid,
- dss->suspend_eventchn);
- if (dss->xce != NULL)
- xc_evtchn_close(dss->xce);
+ if (type == LIBXL_DOMAIN_TYPE_HVM) {
+ domain_save_device_model(egc, dss);
+ } else {
+ domain_suspend_done(egc, dss, 0);
+ }
+ return;
out:
- return rc;
+ domain_suspend_done(egc, dss, rc);
}
-int libxl__domain_save_device_model(libxl__gc *gc, uint32_t domid, int fd)
+static void domain_save_device_model(libxl__egc *egc,
+ libxl__domain_suspend_state *dss)
{
+ STATE_AO_GC(dss->ao);
+
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+ const int fd = dss->fd;
+
int rc, fd2 = -1, c;
char buf[1024];
const char *filename = libxl__device_model_savefile(gc, domid);
@@ -852,7 +881,8 @@ int libxl__domain_save_device_model(libxl__gc *gc, uint32_t
domid, int fd)
goto out;
break;
default:
- return ERROR_INVAL;
+ rc = ERROR_INVAL;
+ goto out;
}
if (stat(filename, &st) < 0)
@@ -896,9 +926,29 @@ int libxl__domain_save_device_model(libxl__gc *gc, uint32_t
domid, int fd)
out:
if (fd2 >= 0) close(fd2);
unlink(filename);
- return rc;
+
+ domain_suspend_done(egc, dss, rc);
}
+static void domain_suspend_done(libxl__egc *egc,
+ libxl__domain_suspend_state *dss, int rc)
+{
+ STATE_AO_GC(dss->ao);
+
+ /* Convenience aliases */
+ const uint32_t domid = dss->domid;
+
+ if (dss->suspend_eventchn > 0)
+ xc_suspend_evtchn_release(CTX->xch, dss->xce, domid,
+ dss->suspend_eventchn);
+ if (dss->xce != NULL)
+ xc_evtchn_close(dss->xce);
+
+ dss->callback(egc, dss, rc);
+}
+
+/*==================== Miscellaneous ====================*/
+
char *libxl__uuid2string(libxl__gc *gc, const libxl_uuid uuid)
{
char *s = libxl__sprintf(gc, LIBXL_UUID_FMT, LIBXL_UUID_BYTES(uuid));
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index fa70a00..a06cc65 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -1708,13 +1708,23 @@ _hidden int
libxl__datacopier_start(libxl__datacopier_state *dc);
typedef struct libxl__domain_suspend_state libxl__domain_suspend_state;
+typedef void libxl__domain_suspend_cb(libxl__egc*,
+ libxl__domain_suspend_state*, int rc);
+
struct libxl__domain_suspend_state {
- libxl__gc *gc;
+ /* set by caller of libxl__domain_suspend */
+ libxl__ao *ao;
+ libxl__domain_suspend_cb *callback;
+
+ uint32_t domid;
+ int fd;
+ libxl_domain_type type;
+ int live;
+ int debug;
+ /* private */
xc_evtchn *xce; /* event channel handle */
int suspend_eventchn;
- int domid;
int hvm;
- unsigned int xcflags;
int guest_responded;
};
@@ -1816,9 +1826,28 @@ struct libxl__domain_create_state {
/*----- Domain suspend (save) functions -----*/
-_hidden int libxl__domain_suspend_common(libxl__gc *gc, uint32_t domid, int fd,
- libxl_domain_type type,
- int live, int debug);
+/* calls callback when done */
+_hidden void libxl__domain_suspend(libxl__egc *egc,
+ libxl__domain_suspend_state *dss);
+
+
+/* calls libxl__xc_domain_suspend_done when done */
+_hidden void libxl__xc_domain_save(libxl__egc*, libxl__domain_suspend_state*,
+ int xcflags, int hvm,
+ unsigned long vm_generationid_addr);
+/* If rc==0 then retval is the return value from xc_domain_save
+ * and errnoval is the errno value it provided.
+ * If rc!=0, retval and errnoval are undefined. */
+_hidden void libxl__xc_domain_save_done(libxl__egc*,
+ libxl__domain_suspend_state*,
+ int rc, int retval, int errnoval);
+
+_hidden int libxl__domain_suspend_common_callback(void *data);
+_hidden int libxl__domain_suspend_common_switch_qemu_logdirty
+ (int domid, unsigned int enable, void *data);
+_hidden int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
+ uint32_t *len, void *data);
+
/* calls libxl__xc_domain_restore_done when done */
_hidden void libxl__xc_domain_restore(libxl__egc *egc,
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index bad4e30..2355cc0 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -20,7 +20,6 @@ void libxl__xc_domain_restore(libxl__egc *egc,
libxl__domain_create_state *dcs,
int hvm, int pae, int superpages,
int no_incr_generationid,
xc_toolstack_restore_cb *toolstack_restore)
-/* calls libxl__xc_domain_restore_done when done */
{
STATE_AO_GC(dcs->ao);
@@ -41,3 +40,23 @@ void libxl__xc_domain_restore(libxl__egc *egc,
libxl__domain_create_state *dcs,
&state->vm_generationid_addr,
&callbacks);
libxl__xc_domain_restore_done(egc, dcs, 0, r, errno);
}
+
+void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss,
+ int xcflags, int hvm,
+ unsigned long vm_generationid_addr)
+{
+ STATE_AO_GC(dss->ao);
+ struct save_callbacks callbacks;
+ int r;
+
+ memset(&callbacks, 0, sizeof(callbacks));
+ callbacks.suspend = libxl__domain_suspend_common_callback;
+ callbacks.switch_qemu_logdirty +
libxl__domain_suspend_common_switch_qemu_logdirty;
+ callbacks.toolstack_save = libxl__toolstack_save;
+ callbacks.data = dss;
+
+ r = xc_domain_save(CTX->xch, dss->fd, dss->domid, 0, 0, xcflags,
&callbacks,
+ hvm, vm_generationid_addr);
+ libxl__xc_domain_save_done(egc, dss, 0, r, errno);
+}
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 38c5726..2dc8a07 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -2808,7 +2808,7 @@ static int save_domain(const char *p, const char
*filename, int checkpoint,
save_domain_core_writeconfig(fd, filename, config_data, config_len);
- CHK_ERRNO(libxl_domain_suspend(ctx, NULL, domid, fd));
+ CHK_ERRNO(libxl_domain_save(ctx, domid, fd, 0, NULL));
close(fd);
if (checkpoint)
@@ -2911,7 +2911,6 @@ static void migrate_domain(const char *domain_spec, const
char *rune,
int rc;
int sendpipe[2], recvpipe[2];
int send_fd, recv_fd;
- libxl_domain_suspend_info suspinfo;
char *away_domname;
char rc_buf;
uint8_t *config_data;
@@ -2964,9 +2963,7 @@ static void migrate_domain(const char *domain_spec, const
char *rune,
xtl_stdiostream_adjust_flags(logger, XTL_STDIOSTREAM_HIDE_PROGRESS, 0);
- memset(&suspinfo, 0, sizeof(suspinfo));
- suspinfo.flags |= XL_SUSPEND_LIVE;
- rc = libxl_domain_suspend(ctx, &suspinfo, domid, send_fd);
+ rc = libxl_domain_save(ctx, domid, send_fd, LIBXL_SAVE_LIVE, NULL);
if (rc) {
fprintf(stderr, "migration sender: libxl_domain_suspend
failed"
" (rc=%d)\n", rc);
--
1.7.2.5
Ian Jackson
2012-May-17 18:53 UTC
[PATCH 5/5] libxl: domain save/restore: run in a separate process
libxenctrl expects to be able to simply run the save or restore
operation synchronously. This won''t work well in a process which is
trying to handle multiple domains.
The options are:
- Block such a whole process (eg, the whole of libvirt) while
migration completes (or until it fails).
- Create a thread to run xc_domain_save and xc_domain_restore on.
This is quite unpalatable. Multithreaded programming is error
prone enough without generating threads in libraries, particularly
if the thread does some very complex operation.
- Fork and run the operation in the child without execing. This is
no good because we would need to negotiate with the caller about
fds we would inherit (and we might be a very large process).
- Fork and exec a helper.
Of these options the latter is the most palatable.
Consequently:
* A new helper program libxl-save-helper (which does both save and
restore). It will be installed in /usr/lib/xen/bin. It does not
link against libxl, only libxc, and its error handling does not
need to be very advanced. It does contain a plumbing through of
the logging interface into the callback stream.
* A small ad-hoc protocol between the helper and libxl which allows
log messages and the libxc callbacks to be passed up and down.
Protocol doc comment is in libxl_save_helper.c.
* To avoid a lot of tedium the marshalling boilerplate (stubs for the
helper and the callback decoder for libxl) is generated with a
small perl script.
* The callback functions in libxl are renamed to the systematic
naming scheme from the marshalling boilerplate. This hardwires the
libxl callbacks.
* Implement new functionality to spawn the helper, monitor its
output, provide responses, and check on its exit status.
* The functions libxl__xc_domain_restore_done and
libxl__xc_domain_save_done now turn out to want be called in the
same place. So make their state argument a void* so that the two
functions are type compatible.
The domain save path still writes the qemu savefile synchronously.
This will need to be fixed in a subsequent patch.
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
.gitignore | 1 +
.hgignore | 2 +
tools/libxl/Makefile | 19 ++-
tools/libxl/libxl_create.c | 24 ++-
tools/libxl/libxl_dom.c | 34 +++--
tools/libxl/libxl_internal.h | 47 ++++--
tools/libxl/libxl_save_callout.c | 318 +++++++++++++++++++++++++++++++++---
tools/libxl/libxl_save_helper.c | 273 +++++++++++++++++++++++++++++++
tools/libxl/libxl_save_msgs_gen.pl | 305 ++++++++++++++++++++++++++++++++++
9 files changed, 965 insertions(+), 58 deletions(-)
create mode 100644 tools/libxl/libxl_save_helper.c
create mode 100755 tools/libxl/libxl_save_msgs_gen.pl
diff --git a/.gitignore b/.gitignore
index 7770e54..3451e52 100644
--- a/.gitignore
+++ b/.gitignore
@@ -353,6 +353,7 @@ tools/libxl/_*.[ch]
tools/libxl/testidl
tools/libxl/testidl.c
tools/libxl/*.pyc
+tools/libxl/libxl-save-helper
tools/blktap2/control/tap-ctl
tools/firmware/etherboot/eb-roms.h
tools/firmware/etherboot/gpxe-git-snapshot.tar.gz
diff --git a/.hgignore b/.hgignore
index 27d8f79..05304ea 100644
--- a/.hgignore
+++ b/.hgignore
@@ -180,9 +180,11 @@
^tools/libxl/_.*\.c$
^tools/libxl/libxlu_cfg_y\.output$
^tools/libxl/xl$
+^tools/libxl/libxl-save-helper$
^tools/libxl/testidl$
^tools/libxl/testidl\.c$
^tools/libxl/tmp\..*$
+^tools/libxl/.*\.new$
^tools/libvchan/vchan-node[12]$
^tools/libaio/src/.*\.ol$
^tools/libaio/src/.*\.os$
diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 9b84421..09f5e9d 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -66,25 +66,29 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o
libxl_pci.o \
libxl_dom.o libxl_exec.o libxl_xshelp.o libxl_device.o \
libxl_internal.o libxl_utils.o libxl_uuid.o \
libxl_json.o libxl_aoutils.o \
- libxl_save_callout.o \
+ libxl_save_callout.o _libxl_save_msgs_callout.o \
libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
$(LIBXL_OBJS): CFLAGS += $(CFLAGS_libxenctrl) $(CFLAGS_libxenguest)
$(CFLAGS_libxenstore) $(CFLAGS_libblktapctl) -include $(XEN_ROOT)/tools/config.h
-AUTOINCS= libxlu_cfg_y.h libxlu_cfg_l.h _libxl_list.h
+AUTOINCS= libxlu_cfg_y.h libxlu_cfg_l.h _libxl_save_msgs.h _libxl_list.h
AUTOSRCS= libxlu_cfg_y.c libxlu_cfg_l.c
+AUTOSRCS += _libxl_save_msgs_callout.c _libxl_save_msgs_helper.c
LIBXLU_OBJS = libxlu_cfg_y.o libxlu_cfg_l.o libxlu_cfg.o \
libxlu_disk_l.o libxlu_disk.o libxlu_vif.o libxlu_pci.o
$(LIBXLU_OBJS): CFLAGS += $(CFLAGS_libxenctrl) # For xentoollog.h
-CLIENTS = xl testidl
+CLIENTS = xl testidl libxl-save-helper
XL_OBJS = xl.o xl_cmdimpl.o xl_cmdtable.o xl_sxp.o
$(XL_OBJS): CFLAGS += $(CFLAGS_libxenctrl) # For xentoollog.h
$(XL_OBJS): CFLAGS += $(CFLAGS_libxenlight)
$(XL_OBJS): CFLAGS += -include $(XEN_ROOT)/tools/config.h # libxl_json.h needs
it.
+SAVE_HELPER_OBJS = libxl_save_helper.o _libxl_save_msgs_helper.o
+$(SAVE_HELPER_OBJS): CFLAGS += $(CFLAGS_libxenctrl) $(CFLAGS_libxenlight)
+
testidl.o: CFLAGS += $(CFLAGS_libxenctrl) $(CFLAGS_libxenlight)
testidl.c: libxl_types.idl gentest.py libxl.h $(AUTOINCS)
$(PYTHON) gentest.py libxl_types.idl testidl.c.new
@@ -116,6 +120,11 @@ _libxl_list.h:
$(XEN_INCLUDE)/xen-external/bsd-sys-queue-h-seddery $(XEN_INCLUDE
perl $^ --prefix=libxl >$@.new
$(call move-if-changed,$@.new,$@)
+_libxl_save_msgs.h _libxl_save_msgs_helper.c _libxl_save_msgs_callout.c: \
+ libxl_save_msgs_gen.pl
+ $(PERL) -w $< $@ >$@.new
+ $(call move-if-changed,$@.new,$@)
+
libxl.h: _libxl_types.h
libxl_json.h: _libxl_types_json.h
libxl_internal.h: _libxl_types_internal.h _libxl_paths.h
@@ -157,6 +166,9 @@ libxlutil.a: $(LIBXLU_OBJS)
xl: $(XL_OBJS) libxlutil.so libxenlight.so
$(CC) $(LDFLAGS) -o $@ $(XL_OBJS) libxlutil.so $(LDLIBS_libxenlight)
$(LDLIBS_libxenctrl) -lyajl $(APPEND_LDFLAGS)
+libxl-save-helper: $(SAVE_HELPER_OBJS) libxenlight.so
+ $(CC) $(LDFLAGS) -o $@ $(SAVE_HELPER_OBJS) $(LDLIBS_libxenctrl)
$(LDLIBS_libxenguest) $(APPEND_LDFLAGS)
+
testidl: testidl.o libxlutil.so libxenlight.so
$(CC) $(LDFLAGS) -o $@ testidl.o libxlutil.so $(LDLIBS_libxenlight)
$(LDLIBS_libxenctrl) $(APPEND_LDFLAGS)
@@ -168,6 +180,7 @@ install: all
$(INSTALL_DIR) $(DESTDIR)$(BASH_COMPLETION_DIR)
$(INSTALL_DIR) $(DESTDIR)$(XEN_RUN_DIR)
$(INSTALL_PROG) xl $(DESTDIR)$(SBINDIR)
+ $(INSTALL_PROG) libxl-save-helper $(DESTDIR)$(PRIVATE_BINDIR)
$(INSTALL_PROG) libxenlight.so.$(MAJOR).$(MINOR) $(DESTDIR)$(LIBDIR)
ln -sf libxenlight.so.$(MAJOR).$(MINOR)
$(DESTDIR)$(LIBDIR)/libxenlight.so.$(MAJOR)
ln -sf libxenlight.so.$(MAJOR) $(DESTDIR)$(LIBDIR)/libxenlight.so
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 7f42737..952c0aa 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -617,14 +617,12 @@ static void domcreate_bootloader_done(libxl__egc *egc,
/* read signature */
int hvm, pae, superpages;
int no_incr_generationid;
- xc_toolstack_restore_cb *toolstack_restore = NULL;
switch (info->type) {
case LIBXL_DOMAIN_TYPE_HVM:
hvm = 1;
superpages = 1;
pae = libxl_defbool_val(info->u.hvm.pae);
no_incr_generationid =
!libxl_defbool_val(info->u.hvm.incr_generationid);
- toolstack_restore = libxl__toolstack_restore;
break;
case LIBXL_DOMAIN_TYPE_PV:
hvm = 0;
@@ -637,18 +635,32 @@ static void domcreate_bootloader_done(libxl__egc *egc,
goto out;
}
libxl__xc_domain_restore(egc, dcs,
- hvm, pae, superpages, no_incr_generationid,
- toolstack_restore);
+ hvm, pae, superpages, no_incr_generationid);
return;
out:
libxl__xc_domain_restore_done(egc, dcs, rc, 0, 0);
}
-void libxl__xc_domain_restore_done(libxl__egc *egc,
- libxl__domain_create_state *dcs,
+void libxl_srm_callout_callback_restore_results(unsigned long store_mfn,
+ unsigned long console_mfn, unsigned long genidad, void *user_void)
+{
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__domain_create_state *dcs = user->caller_state;
+ libxl__save_helper_state *shs = user->shs;
+ STATE_AO_GC(dcs->ao);
+ libxl__domain_build_state *const state = &dcs->build_state;
+
+ state->store_mfn = store_mfn;
+ state->console_mfn = console_mfn;
+ state->vm_generationid_addr = genidad;
+ shs->need_results = 0;
+}
+
+void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
int ret, int retval, int errnoval)
{
+ libxl__domain_create_state *dcs = dcs_void;
STATE_AO_GC(dcs->ao);
libxl_ctx *ctx = libxl__gc_owner(gc);
char **vments = NULL, **localents = NULL;
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index d28f740..464078d 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -458,11 +458,14 @@ static inline char *restore_helper(libxl__gc *gc, uint32_t
domid,
domid, phys_offset, node);
}
-int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
- uint32_t size, void *data)
+int libxl_srm_callout_callback_toolstack_restore(uint32_t domid,
+ const uint8_t *buf, uint32_t size,
+ void *user_void)
{
- libxl__gc *gc = data;
- libxl_ctx *ctx = gc->owner;
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__domain_create_state *dcs = user->caller_state;
+ STATE_AO_GC(dcs->ao);
+ libxl_ctx *ctx = CTX;
int i, ret;
const uint8_t *ptr = buf;
uint32_t count = 0, version = 0;
@@ -522,10 +525,14 @@ static void domain_suspend_done(libxl__egc *egc,
/*----- callbacks, called by xc_domain_save -----*/
-int libxl__domain_suspend_common_switch_qemu_logdirty
- (int domid, unsigned int enable, void *data)
+int libxl_srm_callout_callback_postcopy(void *user) { return 0; }
+int libxl_srm_callout_callback_checkpoint(void *user) { return 0; }
+
+int libxl_srm_callout_callback_switch_qemu_logdirty(int domid, unsigned enable,
+ void *user_void)
{
- libxl__domain_suspend_state *dss = data;
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__domain_suspend_state *dss = user->caller_state;
STATE_AO_GC(dss->ao);
char *path;
bool rc;
@@ -543,9 +550,10 @@ int libxl__domain_suspend_common_switch_qemu_logdirty
return rc ? 0 : 1;
}
-int libxl__domain_suspend_common_callback(void *data)
+int libxl_srm_callout_callback_suspend(void *user_void)
{
- libxl__domain_suspend_state *dss = data;
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__domain_suspend_state *dss = user->caller_state;
STATE_AO_GC(dss->ao);
unsigned long hvm_s_state = 0, hvm_pvdrv = 0;
int ret;
@@ -675,9 +683,9 @@ static inline char *save_helper(libxl__gc *gc, uint32_t
domid,
}
int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
- uint32_t *len, void *data)
+ uint32_t *len, void *dss_void)
{
- libxl__domain_suspend_state *dss = data;
+ libxl__domain_suspend_state *dss = dss_void;
STATE_AO_GC(dss->ao);
int i = 0;
char *start_addr = NULL, *size = NULL, *phys_offset = NULL, *name = NULL;
@@ -817,10 +825,10 @@ void libxl__domain_suspend(libxl__egc *egc,
libxl__domain_suspend_state *dss)
domain_suspend_done(egc, dss, rc);
}
-void libxl__xc_domain_save_done(libxl__egc *egc,
- libxl__domain_suspend_state *dss,
+void libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void,
int rc, int retval, int errnoval)
{
+ libxl__domain_suspend_state *dss = dss_void;
STATE_AO_GC(dss->ao);
/* Convenience aliases */
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index a06cc65..d86d204 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -54,6 +54,7 @@
#include "libxl.h"
#include "_libxl_paths.h"
+#include "_libxl_save_msgs.h"
#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1)
#define _hidden __attribute__((visibility("hidden")))
@@ -752,8 +753,6 @@ _hidden int libxl__domain_rename(libxl__gc *gc, uint32_t
domid,
const char *old_name, const char *new_name,
xs_transaction_t trans);
-_hidden int libxl__toolstack_restore(uint32_t domid, const uint8_t *buf,
- uint32_t size, void *data);
_hidden const char *libxl__device_model_savefile(libxl__gc *gc, uint32_t
domid);
_hidden int libxl__domain_save_device_model(libxl__gc *gc, uint32_t domid, int
fd);
_hidden void libxl__userdata_destroyall(libxl__gc *gc, uint32_t domid);
@@ -1704,6 +1703,36 @@ _hidden void
libxl__datacopier_kill(libxl__datacopier_state *dc);
_hidden int libxl__datacopier_start(libxl__datacopier_state *dc);
+/*----- Save/restore helper (used by creation and suspend) -----*/
+
+typedef struct libxl__save_helper_state {
+ /* public, caller of run_helper initialises */
+ libxl__ao *ao;
+ uint32_t domid;
+ int (*recv_callback)(const unsigned char *msg, uint32_t len, void *user);
+ void (*completion_callback)(libxl__egc *egc, void *caller_state,
+ int rc, int retval, int errnoval);
+ void *caller_state;
+ int need_results; /* set to 0 or 1 by caller of run_helper;
+ * if set to 1 then the ultimate caller''s
+ * results function must set it to 0 */
+ /* private */
+ int rc;
+ int completed; /* retval/errnoval valid iff completed */
+ int retval, errnoval; /* from xc_domain_save / xc_domain_restore */
+ libxl__carefd *pipes[2]; /* 0 = helper''s stdin, 1 =
helper''s stdout */
+ libxl__ev_fd readable;
+ libxl__ev_child child;
+ const char *stdin_what, *stdout_what;
+} libxl__save_helper_state;
+
+typedef struct libxl__save_helper_callback_user {
+ libxl__egc *egc;
+ libxl__save_helper_state *shs;
+ void *caller_state;
+} libxl__save_helper_callback_user;
+
+
/*----- Domain suspend (save) state structure -----*/
typedef struct libxl__domain_suspend_state libxl__domain_suspend_state;
@@ -1726,6 +1755,7 @@ struct libxl__domain_suspend_state {
int suspend_eventchn;
int hvm;
int guest_responded;
+ libxl__save_helper_state shs;
};
@@ -1822,6 +1852,7 @@ struct libxl__domain_create_state {
libxl__stub_dm_spawn_state dmss;
/* If we''re not doing stubdom, we use only dmss.dm,
* for the non-stubdom device model. */
+ libxl__save_helper_state shs;
};
/*----- Domain suspend (save) functions -----*/
@@ -1838,13 +1869,9 @@ _hidden void libxl__xc_domain_save(libxl__egc*,
libxl__domain_suspend_state*,
/* If rc==0 then retval is the return value from xc_domain_save
* and errnoval is the errno value it provided.
* If rc!=0, retval and errnoval are undefined. */
-_hidden void libxl__xc_domain_save_done(libxl__egc*,
- libxl__domain_suspend_state*,
+_hidden void libxl__xc_domain_save_done(libxl__egc*, void *dss_void,
int rc, int retval, int errnoval);
-_hidden int libxl__domain_suspend_common_callback(void *data);
-_hidden int libxl__domain_suspend_common_switch_qemu_logdirty
- (int domid, unsigned int enable, void *data);
_hidden int libxl__toolstack_save(uint32_t domid, uint8_t **buf,
uint32_t *len, void *data);
@@ -1853,13 +1880,11 @@ _hidden int libxl__toolstack_save(uint32_t domid,
uint8_t **buf,
_hidden void libxl__xc_domain_restore(libxl__egc *egc,
libxl__domain_create_state *dcs,
int hvm, int pae, int superpages,
- int no_incr_generationid,
- xc_toolstack_restore_cb*);
+ int no_incr_generationid);
/* If rc==0 then retval is the return value from xc_domain_save
* and errnoval is the errno value it provided.
* If rc!=0, retval and errnoval are undefined. */
-_hidden void libxl__xc_domain_restore_done(libxl__egc *egc,
- libxl__domain_create_state *dcs,
+_hidden void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void,
int rc, int retval, int errnoval);
diff --git a/tools/libxl/libxl_save_callout.c b/tools/libxl/libxl_save_callout.c
index 2355cc0..efc84d8 100644
--- a/tools/libxl/libxl_save_callout.c
+++ b/tools/libxl/libxl_save_callout.c
@@ -16,10 +16,22 @@
#include "libxl_internal.h"
+static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
+ const char *mode_arg, int data_fd,
+ const unsigned long *argnums, int num_argnums);
+
+static void helper_failed(libxl__egc*, libxl__save_helper_state *shs, int rc);
+static void helper_stdout_readable(libxl__egc *egc, libxl__ev_fd *ev,
+ int fd, short events, short revents);
+static void helper_exited(libxl__egc *egc, libxl__ev_child *ch,
+ pid_t pid, int status);
+static void helper_done(libxl__egc *egc, libxl__save_helper_state *shs);
+
+/*----- entrypoints -----*/
+
void libxl__xc_domain_restore(libxl__egc *egc, libxl__domain_create_state *dcs,
int hvm, int pae, int superpages,
- int no_incr_generationid,
- xc_toolstack_restore_cb *toolstack_restore)
+ int no_incr_generationid)
{
STATE_AO_GC(dcs->ao);
@@ -28,17 +40,23 @@ void libxl__xc_domain_restore(libxl__egc *egc,
libxl__domain_create_state *dcs,
const int restore_fd = dcs->restore_fd;
libxl__domain_build_state *const state = &dcs->build_state;
- struct restore_callbacks callbacks;
- callbacks.toolstack_restore = toolstack_restore;
- callbacks.data = gc;
+ const unsigned long argnums[] = {
+ restore_fd, domid,
+ state->store_port,
+ state->store_domid, state->console_port,
+ state->console_domid,
+ hvm, pae, superpages, no_incr_generationid,
+ };
- int r = xc_domain_restore(CTX->xch, restore_fd, domid,
- state->store_port, &state->store_mfn,
- state->store_domid, state->console_port,
- &state->console_mfn,
state->console_domid,
- hvm, pae, superpages, no_incr_generationid,
- &state->vm_generationid_addr,
&callbacks);
- libxl__xc_domain_restore_done(egc, dcs, 0, r, errno);
+ dcs->shs.ao = ao;
+ dcs->shs.domid = domid;
+ dcs->shs.recv_callback = libxl_srm_callout_received_restore;
+ dcs->shs.completion_callback = libxl__xc_domain_restore_done;
+ dcs->shs.caller_state = dcs;
+ dcs->shs.need_results = 1;
+
+ run_helper(egc, &dcs->shs, "--restore-domain", restore_fd,
+ argnums, ARRAY_SIZE(argnums));
}
void libxl__xc_domain_save(libxl__egc *egc, libxl__domain_suspend_state *dss,
@@ -46,17 +64,267 @@ void libxl__xc_domain_save(libxl__egc *egc,
libxl__domain_suspend_state *dss,
unsigned long vm_generationid_addr)
{
STATE_AO_GC(dss->ao);
- struct save_callbacks callbacks;
- int r;
-
- memset(&callbacks, 0, sizeof(callbacks));
- callbacks.suspend = libxl__domain_suspend_common_callback;
- callbacks.switch_qemu_logdirty -
libxl__domain_suspend_common_switch_qemu_logdirty;
- callbacks.toolstack_save = libxl__toolstack_save;
- callbacks.data = dss;
-
- r = xc_domain_save(CTX->xch, dss->fd, dss->domid, 0, 0, xcflags,
&callbacks,
- hvm, vm_generationid_addr);
- libxl__xc_domain_save_done(egc, dss, 0, r, errno);
+ int r, rc;
+ uint32_t toolstack_data_len;
+
+ /* Resources we need to free */
+ uint8_t *toolstack_data_buf = 0;
+ FILE *toolstack_data_file = 0;
+
+ r = libxl__toolstack_save(dss->domid, &toolstack_data_buf,
+ &toolstack_data_len, dss);
+ if (r) { rc = ERROR_FAIL; goto out; }
+
+ toolstack_data_file = tmpfile();
+ if (!toolstack_data_file) {
+ LOGE(ERROR, "cannot create toolstack data tmpfile");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ int toolstack_data_fd = fileno(toolstack_data_file);
+
+ r = libxl_write_exactly(CTX, toolstack_data_fd,
+ toolstack_data_buf, toolstack_data_len,
+ "toolstack data tmpfile", 0);
+ if (r) { rc = ERROR_FAIL; goto out; }
+
+ r = lseek(toolstack_data_fd, 0, SEEK_SET);
+ if (r) {
+ LOGE(ERROR, "rewind toolstack data tmpfile");
+ rc = ERROR_FAIL;
+ goto out;
+ }
+
+ const unsigned long argnums[] = {
+ dss->fd, dss->domid, 0, 0, xcflags, hvm, vm_generationid_addr,
+ toolstack_data_fd, toolstack_data_len,
+ };
+
+ dss->shs.ao = ao;
+ dss->shs.domid = dss->domid;
+ dss->shs.recv_callback = libxl_srm_callout_received_save;
+ dss->shs.completion_callback = libxl__xc_domain_save_done;
+ dss->shs.caller_state = dss;
+ dss->shs.need_results = 0;
+
+ run_helper(egc, &dss->shs, "--save-domain", dss->fd,
+ argnums, ARRAY_SIZE(argnums));
+ return;
+
+ out:
+ free(toolstack_data_buf);
+ if (toolstack_data_file) fclose(toolstack_data_file);
+
+ libxl__xc_domain_save_done(egc, dss, rc, 0, 0);
+}
+
+
+/*----- helper execution -----*/
+
+static void run_helper(libxl__egc *egc, libxl__save_helper_state *shs,
+ const char *mode_arg, int data_fd,
+ const unsigned long *argnums, int num_argnums)
+{
+ STATE_AO_GC(shs->ao);
+ const char *args[2 + num_argnums];
+ const char **arg = args;
+ int i, rc;
+
+ /* Resources we must free */
+ libxl__carefd *childs_pipes[2] = { 0,0 };
+
+ /* Convenience aliases */
+ const uint32_t domid = shs->domid;
+
+ shs->rc = 0;
+ shs->completed = 0;
+ shs->pipes[0] = shs->pipes[1] = 0;
+ libxl__ev_fd_init(&shs->readable);
+ libxl__ev_child_init(&shs->child);
+
+ shs->stdin_what = GCSPRINTF("domain %"PRIu32"
save/restore helper"
+ " stdin pipe", domid);
+ shs->stdout_what = GCSPRINTF("domain %"PRIu32"
save/restore helper"
+ " stdout pipe", domid);
+
+ *arg++ = LIBEXEC "/" "libxl-save-helper";
+ *arg++ = mode_arg;
+ for (i=0; i<num_argnums; i++)
+ *arg++ = GCSPRINTF("%lu", argnums[i]);
+ *arg++ = 0;
+ assert(arg == args + ARRAY_SIZE(args));
+
+ libxl__carefd_begin();
+ for (i=0; i<2; i++) {
+ int fds[2];
+ if (libxl_pipe(CTX,fds)) { rc = ERROR_FAIL; goto out; }
+ childs_pipes[i] = libxl__carefd_record(CTX, fds[i]);
+ shs->pipes[i] = libxl__carefd_record(CTX, fds[!i]);
+ }
+ libxl__carefd_unlock();
+
+ pid_t pid = libxl__ev_child_fork(gc, &shs->child, helper_exited);
+ if (!pid) {
+ libxl_fd_set_cloexec(CTX, data_fd, 0);
+ libxl__exec(libxl__carefd_fd(childs_pipes[0]),
+ libxl__carefd_fd(childs_pipes[1]),
+ -1,
+ args[0], (char**)args);
+ }
+
+ libxl__carefd_close(childs_pipes[0]);
+ libxl__carefd_close(childs_pipes[1]);
+
+ rc = libxl__ev_fd_register(gc, &shs->readable,
helper_stdout_readable,
+ libxl__carefd_fd(shs->pipes[1]),
POLLIN|POLLPRI);
+ if (rc) goto out;
+ return;
+
+ out:
+ libxl__carefd_close(childs_pipes[0]);
+ libxl__carefd_close(childs_pipes[1]);
+ helper_failed(egc, shs, rc);;
+}
+
+static void helper_failed(libxl__egc *egc, libxl__save_helper_state *shs,
+ int rc)
+{
+ STATE_AO_GC(shs->ao);
+
+ if (!shs->rc)
+ shs->rc = rc;
+
+ if (!libxl__ev_child_inuse(&shs->child)) {
+ helper_done(egc, shs);
+ return;
+ }
+
+ int r = kill(shs->child.pid, SIGKILL);
+ if (r) LOGE(WARN, "failed to kill save/restore helper [%lu]",
+ (unsigned long)shs->child.pid);
+}
+
+static void helper_stdout_readable(libxl__egc *egc, libxl__ev_fd *ev,
+ int fd, short events, short revents)
+{
+ libxl__save_helper_state *shs = CONTAINER_OF(ev, *shs, readable);
+ STATE_AO_GC(shs->ao);
+ int rc, errnoval;
+
+ if (revents & POLLPRI) {
+ LOG(ERROR, "%s signaled POLLERR", shs->stdout_what);
+ rc = ERROR_FAIL;
+ out:
+ /* this is here because otherwise we bypass the decl of msg[] */
+ helper_failed(egc, shs, rc);
+ return;
+ }
+
+ uint32_t msglen;
+ errnoval = libxl_read_exactly(CTX, fd, &msglen, sizeof(msglen),
+ shs->stdout_what, "ipc msg
header");
+ if (errnoval) { rc = ERROR_FAIL; goto out; }
+
+ unsigned char msg[msglen];
+ errnoval = libxl_read_exactly(CTX, fd, msg, msglen,
+ shs->stdout_what, "ipc msg
body");
+ if (errnoval) { rc = ERROR_FAIL; goto out; }
+
+ libxl__save_helper_callback_user user = { egc, shs, shs->caller_state };
+ shs->recv_callback(msg, msglen, &user);
+ return;
+}
+
+static void helper_exited(libxl__egc *egc, libxl__ev_child *ch,
+ pid_t pid, int status)
+{
+ libxl__save_helper_state *shs = CONTAINER_OF(ch, *shs, child);
+ STATE_AO_GC(shs->ao);
+
+ /* Convenience aliases */
+ const uint32_t domid = shs->domid;
+
+ const char *what + GCSPRINTF("domain %"PRIu32"
save/restore helper", domid);
+
+ if (status) {
+ libxl_report_child_exitstatus(CTX, XTL_ERROR, what, pid, status);
+ shs->rc = ERROR_FAIL;
+ }
+
+ if (shs->need_results) {
+ if (!shs->rc)
+ LOG(ERROR,"%s exited without providing results",what);
+ shs->rc = ERROR_FAIL;
+ }
+
+ if (!shs->completed) {
+ if (!shs->rc)
+ LOG(ERROR,"%s exited without signaling completion",what);
+ shs->rc = ERROR_FAIL;
+ }
+
+ helper_done(egc, shs);
+ return;
+}
+
+static void helper_done(libxl__egc *egc, libxl__save_helper_state *shs)
+{
+ STATE_AO_GC(shs->ao);
+
+ libxl__ev_fd_deregister(gc, &shs->readable);
+ libxl__carefd_close(shs->pipes[0]); shs->pipes[0] = 0;
+ libxl__carefd_close(shs->pipes[1]); shs->pipes[1] = 0;
+ assert(!libxl__ev_child_inuse(&shs->child));
+
+ shs->completion_callback(egc, shs->caller_state,
+ shs->rc, shs->retval, shs->errnoval);
+}
+
+/*----- generic helpers for the autogenerated code -----*/
+
+void libxl_srm_callout_sendreply(int r, void *user_void)
+{
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__save_helper_state *shs = user->shs;
+ libxl__egc *egc = user->egc;
+ STATE_AO_GC(shs->ao);
+ int errnoval;
+
+ errnoval = libxl_write_exactly(CTX, libxl__carefd_fd(shs->pipes[0]),
+ &r, sizeof(r), shs->stdin_what,
+ "callback return value");
+ if (errnoval)
+ helper_failed(egc, shs, ERROR_FAIL);
+}
+
+void libxl_srm_callout_callback_log(uint32_t level, uint32_t errnoval,
+ const char *context, const char *formatted, void *user_void)
+{
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__save_helper_state *shs = user->shs;
+ STATE_AO_GC(shs->ao);
+ xtl_log(CTX->lg, level, errnoval, context, "%s", formatted);
+}
+
+void libxl_srm_callout_callback_progress(const char *context,
+ const char *doing_what, unsigned long done,
+ unsigned long total, void *user_void)
+{
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__save_helper_state *shs = user->shs;
+ STATE_AO_GC(shs->ao);
+ xtl_progress(CTX->lg, context, doing_what, done, total);
+}
+
+void libxl_srm_callout_callback_complete(int retval, int errnoval,
+ void *user_void)
+{
+ libxl__save_helper_callback_user *user = user_void;
+ libxl__save_helper_state *shs = user->shs;
+ STATE_AO_GC(shs->ao);
+
+ shs->completed = 1;
+ shs->retval = retval;
+ shs->errnoval = errnoval;
}
diff --git a/tools/libxl/libxl_save_helper.c b/tools/libxl/libxl_save_helper.c
new file mode 100644
index 0000000..79081dd
--- /dev/null
+++ b/tools/libxl/libxl_save_helper.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2012 Citrix Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+/*
+ * The libxl-save-helper utility speaks a protocol to its caller for
+ * the callbacks. The protocol is as follows.
+ *
+ * The helper talks on stdin and stdout, in binary in machine
+ * endianness. The helper speaks first, and only when it has a
+ * callback to make. It writes a 32-bit number being the message
+ * length, and then the message body.
+ *
+ * Each message starts with a 32-bit number indicating which of the
+ * messages it is, and then some arguments in a binary marshalled form.
+ * If the callback does not need a reply (it returns void), the helper
+ * just continues. Otherwise the helper waits for its caller to send a
+ * single int which is to be the return value from the callback.
+ *
+ * Where feasible the stubs and callbacks have prototypes identical to
+ * those required by xc_domain_save and xc_domain_restore, so that the
+ * autogenerated functions can be used/provided directly.
+ *
+ * The actual messages are in the array @msgs in libxl_save_msgs_gen.pl
+ */
+
+#include "libxl_osdeps.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include "libxl.h"
+
+#include "xenctrl.h"
+#include "xenguest.h"
+#include "_libxl_save_msgs.h"
+
+/*----- globals -----*/
+
+static const char *program = "libxl-save-helper";
+static xentoollog_logger *logger;
+static xc_interface *xch;
+
+/*----- error handling -----*/
+
+static void fail(int errnoval, const char *fmt, ...)
+ __attribute__((noreturn,format(printf,2,3)));
+static void fail(int errnoval, const char *fmt, ...)
+{
+ va_list al;
+ va_start(al,fmt);
+ xtl_logv(logger,XTL_ERROR,errnoval,program,fmt,al);
+ exit(-1);
+}
+
+static int read_exactly(int fd, void *buf, size_t len)
+/* returns 0 if we get eof, even if we got it midway through; 1 if ok */
+{
+ while (len) {
+ ssize_t r = read(fd, buf, len);
+ if (r<=0) return r;
+ assert(r <= len);
+ len -= r;
+ buf = (char*)buf + r;
+ }
+ return 1;
+}
+
+static void *xmalloc(size_t sz)
+{
+ if (!sz) return 0;
+ void *r = malloc(sz);
+ if (!r) { perror("memory allocation failed"); exit(-1); }
+ return r;
+}
+
+/*----- logger -----*/
+
+typedef struct {
+ xentoollog_logger vtable;
+} xentoollog_logger_tellparent;
+
+static void tellparent_vmessage(xentoollog_logger *logger_in,
+ xentoollog_level level,
+ int errnoval,
+ const char *context,
+ const char *format,
+ va_list al)
+{
+ char *formatted;
+ int r = vasprintf(&formatted, format, al);
+ if (r < 0) { perror("memory allocation failed during
logging"); exit(-1); }
+ libxl_srm_helper_stub_log(level, errnoval, context, formatted, 0);
+ free(formatted);
+}
+
+static void tellparent_progress(struct xentoollog_logger *logger_in,
+ const char *context,
+ const char *doing_what, int percent,
+ unsigned long done, unsigned long total)
+{
+ libxl_srm_helper_stub_progress(context, doing_what, done, total, 0);
+}
+
+static void tellparent_destroy(struct xentoollog_logger *logger_in)
+{
+ abort();
+}
+
+static xentoollog_logger_tellparent *createlogger_tellparent(void)
+{
+ xentoollog_logger_tellparent newlogger;
+ return XTL_NEW_LOGGER(tellparent, newlogger);
+}
+
+/*----- helper functions called by autogenerated stubs -----*/
+
+unsigned char * libxl_srm_helper_allocbuf(int len, void *user)
+{
+ return xmalloc(len);
+}
+
+void libxl_srm_helper_transmit(unsigned char *msg_freed, int len, void *user)
+{
+ while (len) {
+ int r = write(1, msg_freed, len);
+ if (r<0) { perror("write"); exit(-1); }
+ assert(r >= 0);
+ assert(r <= len);
+ len -= r;
+ msg_freed += r;
+ }
+ free(msg_freed);
+}
+
+int libxl_srm_helper_getreply(void *user)
+{
+ int v;
+ int r = read_exactly(0, &v, sizeof(v));
+ if (r<=0) exit(-2);
+ return v;
+}
+
+/*----- other callbacks -----*/
+
+static int toolstack_save_fd;
+static uint32_t toolstack_save_len;
+
+static int toolstack_save_cb(uint32_t domid, uint8_t **buf,
+ uint32_t *len, void *data)
+{
+ assert(toolstack_save_fd > 0);
+
+ *buf = xmalloc(toolstack_save_len);
+ int r = read_exactly(toolstack_save_fd, *buf, toolstack_save_len);
+ if (r<0) fail(errno,"read toolstack data");
+ if (r==0) fail(0,"read toolstack data eof");
+
+ toolstack_save_fd = -1;
+ return 0;
+}
+
+static struct save_callbacks helper_save_callbacks = {
+ .suspend = libxl_srm_helper_stub_suspend,
+ .postcopy = libxl_srm_helper_stub_postcopy,
+ .checkpoint = libxl_srm_helper_stub_checkpoint,
+ .switch_qemu_logdirty = libxl_srm_helper_stub_switch_qemu_logdirty,
+ .toolstack_save = toolstack_save_cb,
+ .data = NULL,
+};
+
+static struct restore_callbacks helper_restore_callbacks = {
+ .toolstack_restore = libxl_srm_helper_stub_toolstack_restore,
+ .data = NULL,
+};
+
+static void startup(const char *op) {
+ xentoollog_logger *logger = (xentoollog_logger*)createlogger_tellparent();
+ if (!logger) {
+ fprintf(stderr, "%s: cannot initialise logger\n", program);
+ exit(-1);
+ }
+
+ xtl_log(logger,XTL_DEBUG,0,program,"starting %s",op);
+
+ xch = xc_interface_open(logger,logger,0);
+ if (!xch) fail(errno,"xc_interface_open failed");
+}
+
+static void complete(int retval) {
+ int errnoval = errno;
+ xtl_log(logger,XTL_DEBUG,errnoval,program,"complete
r=%d",retval);
+ libxl_srm_helper_stub_complete(retval,errnoval,0);
+ exit(0);
+}
+
+int main(int argc, char **argv)
+{
+ int r;
+
+#define NEXTARG (++argv, assert(*argv), *argv)
+
+ const char *mode = *++argv;
+ assert(mode);
+
+ if (!strcmp(mode,"--save-domain")) {
+
+ int io_fd = atoi(NEXTARG);
+ uint32_t dom = strtoul(NEXTARG,0,10);
+ uint32_t max_iters = strtoul(NEXTARG,0,10);
+ uint32_t max_factor = strtoul(NEXTARG,0,10);
+ uint32_t flags = strtoul(NEXTARG,0,10);
+ int hvm = atoi(NEXTARG);
+ unsigned long genidad = strtoul(NEXTARG,0,10);
+ toolstack_save_fd = atoi(NEXTARG);
+ toolstack_save_len = strtoul(NEXTARG,0,10);
+ assert(!*++argv);
+
+ startup("save");
+ r = xc_domain_save(xch, io_fd, dom, max_iters, max_factor, flags,
+ &helper_save_callbacks, hvm, genidad);
+ complete(r);
+
+ } else if (!strcmp(mode,"--restore-domain")) {
+
+ int io_fd = atoi(NEXTARG);
+ uint32_t dom = strtoul(NEXTARG,0,10);
+ unsigned store_evtchn = strtoul(NEXTARG,0,10);
+ domid_t store_domid = strtoul(NEXTARG,0,10);
+ unsigned console_evtchn = strtoul(NEXTARG,0,10);
+ domid_t console_domid = strtoul(NEXTARG,0,10);
+ unsigned int hvm = strtoul(NEXTARG,0,10);
+ unsigned int pae = strtoul(NEXTARG,0,10);
+ int superpages = strtoul(NEXTARG,0,10);
+ int no_incr_genidad = strtoul(NEXTARG,0,10);
+ assert(!*++argv);
+
+ unsigned long store_mfn = 0;
+ unsigned long console_mfn = 0;
+ unsigned long genidad = 0;
+
+ startup("restore");
+ r = xc_domain_restore(xch, io_fd, dom, store_evtchn, &store_mfn,
+ store_domid, console_evtchn, &console_mfn,
+ console_domid, hvm, pae, superpages,
+ no_incr_genidad, &genidad,
+ &helper_restore_callbacks);
+ libxl_srm_helper_stub_restore_results(store_mfn,console_mfn,genidad,0);
+ complete(r);
+
+ } else {
+ assert(!"unexpected mode argument");
+ }
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/libxl_save_msgs_gen.pl
b/tools/libxl/libxl_save_msgs_gen.pl
new file mode 100755
index 0000000..9960fdc
--- /dev/null
+++ b/tools/libxl/libxl_save_msgs_gen.pl
@@ -0,0 +1,305 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use POSIX;
+
+
+our @msgs = (
+ [ 1, ''sr'',0, "log",
[qw(uint32_t level
+ uint32_t errnoval
+ STRING context
+ STRING formatted)] ],
+ [ 2, ''sr'',0, "progress",
[qw(STRING context
+ STRING doing_what),
+ ''unsigned long'',
''done'',
+ ''unsigned long'',
''total''] ],
+ [ 3, ''s'', 1, "suspend", [] ],
+ [ 4, ''s'', 1, "postcopy", [] ],
+ [ 5, ''s'', 1, "checkpoint", [] ],
+ [ 6, ''s'', 1, "switch_qemu_logdirty", [qw(int
domid
+ unsigned enable)] ],
+ [ 7, ''r'', 1, "toolstack_restore",
[qw(uint32_t domid
+ BLOCK tsdata)] ],
+ [ 8, ''r'', 0, "restore_results",
[''unsigned long'', ''store_mfn'',
+ ''unsigned long'',
''console_mfn'',
+ ''unsigned long'',
''genidad''] ],
+ [ 9, ''sr'',0, "complete", [qw(int
retval
+ int errnoval)] ],
+);
+
+#----------------------------------------
+
+our %func;
+our %func_ah;
+our @outfuncs;
+our %out_decls;
+our %out_body;
+our %msgnum_used;
+
+die unless @ARGV==1;
+die if $ARGV[0] =~ m/^-/;
+
+our ($intendedout) = @ARGV;
+
+my $declprefix = '''';
+
+foreach my $ah (qw(callout helper)) {
+ $out_body{$ah} .= <<END;
+#include <assert.h>
+#include <string.h>
+#include <stdint.h>
+#include <limits.h>
+
+#include "_libxl_save_msgs.h"
+
+END
+}
+
+sub f_decl ($$$$) {
+ my ($name, $ah, $c_rtype, $c_decl) = @_;
+ $out_decls{$name} = "${declprefix}$c_rtype $name$c_decl;\n";
+ $func{$name} = "$c_rtype $name$c_decl\n{\n" . ($func{$name} ||
'''');
+ $func_ah{$name} = $ah;
+}
+
+sub f_more ($$) {
+ my ($name, $addbody) = @_;
+ $func{$name} ||= '''';
+ $func{$name} .= $addbody;
+ push @outfuncs, $name;
+}
+
+our $callback = "libxl_srm_callout_callback";
+our $receiveds = "libxl_srm_callout_received";
+our $sendreply = "libxl_srm_callout_sendreply";
+
+f_decl($sendreply, ''callout'', ''void'',
"(int r, void *user)");
+
+our $encode = "libxl_srm_helper_stub";
+our $allocbuf = "libxl_srm_helper_allocbuf";
+our $transmit = "libxl_srm_helper_transmit";
+our $getreply = "libxl_srm_helper_getreply";
+
+f_decl($allocbuf, ''helper'', ''unsigned char
*'', ''(int len, void *user)'');
+f_decl($transmit, ''helper'', ''void'',
+ ''(unsigned char *msg_freed, int len, void *user)'');
+f_decl($getreply, ''helper'', ''int'',
''(void *user)'');
+
+sub typeid ($) { my ($t) = @_; $t =~ s/\W/_/; return $t; };
+
+$out_body{''callout''} .= <<END;
+static int bytes_get(const unsigned char **msg,
+ const unsigned char *const endmsg,
+ void *result, int rlen)
+{
+ if (endmsg - *msg < rlen) return 0;
+ memcpy(result,*msg,rlen);
+ *msg += rlen;
+ return 1;
+}
+
+END
+$out_body{''helper''} .= <<END;
+static void bytes_put(unsigned char *const buf, int *len,
+ const void *value, int vlen)
+{
+ assert(vlen < INT_MAX/2 - *len);
+ if (buf)
+ memcpy(buf + *len, value, vlen);
+ *len += vlen;
+}
+
+END
+
+foreach my $simpletype (qw(int uint32_t unsigned), ''unsigned
long'') {
+ my $typeid = typeid($simpletype);
+ $out_body{''callout''} .= <<END;
+static int ${typeid}_get(const unsigned char **msg,
+ const unsigned char *const endmsg,
+ $simpletype *result)
+{
+ return bytes_get(msg, endmsg, result, sizeof(*result));
+}
+
+END
+ $out_body{''helper''} .= <<END;
+static void ${typeid}_put(unsigned char *const buf, int *len,
+ const $simpletype value)
+{
+ bytes_put(buf, len, &value, sizeof(value));
+}
+
+END
+}
+
+$out_body{''callout''} .= <<END;
+static int BLOCK_get(const unsigned char **msg,
+ const unsigned char *const endmsg,
+ const uint8_t **result, uint32_t *result_size)
+{
+ if (!uint32_t_get(msg,endmsg,result_size)) return 0;
+ if (endmsg - *msg < *result_size) return 0;
+ *result = (const void*)*msg;
+ *msg += *result_size;
+ return 1;
+}
+
+static int STRING_get(const unsigned char **msg,
+ const unsigned char *const endmsg,
+ const char **result)
+{
+ const uint8_t *data;
+ uint32_t datalen;
+ if (!BLOCK_get(msg,endmsg,&data,&datalen)) return 0;
+ if (datalen == 0) return 0;
+ if (data[datalen-1] != ''\\0'') return 0;
+ *result = (const void*)data;
+ return 1;
+}
+
+END
+$out_body{''helper''} .= <<END;
+static void BLOCK_put(unsigned char *const buf,
+ int *len,
+ const uint8_t *bytes, uint32_t size)
+{
+ uint32_t_put(buf, len, size);
+ bytes_put(buf, len, bytes, size);
+}
+
+static void STRING_put(unsigned char *const buf,
+ int *len,
+ const char *string)
+{
+ size_t slen = strlen(string);
+ assert(slen < INT_MAX / 4);
+ assert(slen < (uint32_t)0x40000000);
+ BLOCK_put(buf, len, (const void*)string, slen+1);
+}
+
+END
+
+foreach my $sr (qw(save restore)) {
+ f_decl("${receiveds}_${sr}", ''callout'',
''int'',
+ "(const unsigned char *msg, uint32_t len, void *user)");
+
+ f_more("${receiveds}_${sr}",
+" const unsigned char *const endmsg = msg + len;
+ uint32_t mtype;
+ if (!uint32_t_get(&msg,endmsg,&mtype)) return 0;
+ switch (mtype) {
+
+");
+}
+
+foreach my $msginfo (@msgs) {
+ my ($msgnum, $inwhich, $hasreply, $name, $args) = @$msginfo;
+ die if $msgnum_used{$msgnum}++;
+
+ my $f_more_recv = sub {
+ f_more("${receiveds}_save", $_[0]) if $inwhich =~ m/s/;
+ f_more("${receiveds}_restore", $_[0]) if $inwhich =~ m/r/;
+ };
+
+ $f_more_recv->(" case $msgnum: { /* $name */\n");
+ if ($hasreply) {
+ $f_more_recv->(" int r;\n");
+ }
+
+ my $c_rtype = $hasreply ? ''int'' :
''void'';
+ my $c_decl = ''('';
+ my $c_callback_args = '''';
+
+ f_more("${encode}_$name",
+" unsigned char *buf = 0;
+ int len = 0, allocd = 0;
+
+ for (;;) {
+ uint32_t_put(buf, &len, $msgnum /* $name */);
+");
+
+ my @args = @$args;
+ my $c_recv = '''';
+ my ($argtype, $arg);
+ while (($argtype, $arg, @args) = @args) {
+ my $typeid = typeid($argtype);
+ my $c_args = "$arg";
+ my $c_get_args = "&$arg";
+ if ($argtype eq ''STRING'') {
+ $c_decl .= "const char *$arg, ";
+ $f_more_recv->(" const char *$arg;\n");
+ } elsif ($argtype eq ''BLOCK'') {
+ $c_decl .= "const uint8_t *$arg, uint32_t ${arg}_size, ";
+ $c_args .= ", ${arg}_size";
+ $c_get_args .= ",&${arg}_size";
+ $f_more_recv->(" const uint8_t *$arg;\n".
+ " uint32_t ${arg}_size;\n");
+ } else {
+ $c_decl .= "$argtype $arg, ";
+ $f_more_recv->(" $argtype $arg;\n");
+ }
+ $c_callback_args .= "$c_args, ";
+ $c_recv.+ " if
(!${typeid}_get(&msg,endmsg,$c_get_args)) return 0;\n";
+ f_more("${encode}_$name", " ${typeid}_put(buf, &len,
$c_args);\n");
+ }
+ $f_more_recv->($c_recv);
+ $c_decl .= "void *user)";
+ $c_callback_args .= "user";
+
+ $f_more_recv->(" if (msg != endmsg) return 0;\n");
+ my $c_make_callback = "${callback}_$name($c_callback_args)";
+ if (!$hasreply) {
+ $f_more_recv->(" $c_make_callback;\n");
+ } else {
+ $f_more_recv->(" r = $c_make_callback;\n".
+ " $sendreply(r, user);\n");
+ f_decl($sendreply, ''callout'', ''void'',
''(int r, void *user)'');
+ }
+ $f_more_recv->(" return 1;\n }\n\n");
+ f_decl("${callback}_$name", ''callout'',
$c_rtype, $c_decl);
+ f_decl("${encode}_$name", ''helper'', $c_rtype,
$c_decl);
+ f_more("${encode}_$name",
+" if (buf) break;
+ buf = libxl_srm_helper_allocbuf(len, user);
+ assert(buf);
+ }
+ assert(len == allocd);
+ libxl_srm_helper_transmit(buf, len, user);
+");
+ if ($hasreply) {
+ f_more("${encode}_$name",
+ " return libxl_srm_helper_getreply(user);\n");
+ }
+}
+
+foreach my $sr (qw(save restore)) {
+f_more("${receiveds}_${sr}",
+" default:
+ return 0;
+ }
+");
+}
+
+foreach my $name (@outfuncs) {
+ next unless defined $func{$name};
+ $func{$name} .= "}\n";
+ $out_body{$func_ah{$name}} .= "$func{$name}\n";
+ delete $func{$name};
+}
+
+print "/* AUTOGENERATED by $0 DO NOT EDIT */\n\n" or die $!;
+
+if ($intendedout =~ m/\.h$/) {
+ foreach my $decl (sort values %out_decls) {
+ print $decl or die $!;
+ }
+} elsif ($intendedout =~ m/([a-z]+)\.c$/) {
+ die $1 unless defined $out_body{$1};
+ print $out_body{$1} or die $!;
+} else {
+ die;
+}
+
+close STDOUT or die $!;
--
1.7.2.5