[PATCH v5 03/10] libxl: convert libxl_domain_destroy to an async op

Roger Pau Monne
Wed, 30 May 2012 13:07:48 -0000

This change introduces some new structures, and breaks the mutual
dependency that libxl_domain_destroy and libxl__destroy_device_model
had. This is done by checking if the domid passed to
libxl_domain_destroy has a stubdom, and then having the bulk of the
destroy machinery in a separate function (libxl__destroy_domid) that
doesn''t check for stubdom presence, since we check for it in the upper
level function. The reason behind this change is the need to use
structures for ao operations, and it was impossible to have two
different self-referencing structs.

All uses of libxl_domain_destroy have been changed, and either
replaced by the new libxl_domain_destroy ao function or by the
internal libxl__domain_destroy that can be used inside an already
running ao.

Changes since v4:

 * Fixed spelling mistakes.

 * Always use "force = 1" in device destruction in
   libxl__destroy_domid function.

 * Changed name of domain destroy callbacks to include "destroy".

 * Changed the use of rc to catch syscall errors.

 * Use libxl__remove_file instead of unlink.

 * Changed variable name of number of devices returned by
   libxl__xs_directory.

 * Simplify libxl__ao_device_check_last return.

 * Correctly propagate error returned from libxl__num_devices.

 * Add a comment about the use of libxl__device_destroy to destroy the
   console.

 * Fixed some uses of LIBXL__LOG.

Changes since v3:

 * Fixed python bindings.

Changes since v2:

 * Remove printfs.

 * Replace aorm with aodev.

 * Define an auxiliary libxl__ao_device *aodev to avoid using the long
   expression: drs->aorm[numdev]...

 * Added a common callback for both domain and stubdomain destruction
   that checks if both domains are finished and handles errors
   correctly.

 * Change libxl__ao_device_check_last logic a bit and add a comment
   describing how does it work.

 * Fixed spelling mistakes.

 * Use a do-while for xs transaction in device_remove_callback.

Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>
Signed-off-by: Roger Pau Monne <roger.pau@citrix.com>
---
 tools/libxl/libxl.c               |  167 +++++++++++++++++++++++++++++++++++--
 tools/libxl/libxl.h               |    3 +-
 tools/libxl/libxl_create.c        |   29 ++++++-
 tools/libxl/libxl_device.c        |  160 +++++++++++++++++++++++++++++++----
 tools/libxl/libxl_dm.c            |   83 +++++++++---------
 tools/libxl/libxl_internal.h      |   94 ++++++++++++++++++++-
 tools/libxl/xl_cmdimpl.c          |   12 ++--
 tools/python/xen/lowlevel/xl/xl.c |    2 +-
 8 files changed, 469 insertions(+), 81 deletions(-)

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index c4e31e1..4044b9e 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -1112,11 +1112,133 @@ void libxl_evdisable_disk_eject(libxl_ctx *ctx,
libxl_evgen_disk_eject *evg) {
     GC_FREE;
 }

-int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid)
+/* Callbacks for libxl_domain_destroy */
+
+static void domain_destroy_cb(libxl__egc *egc, libxl__domain_destroy_state
*dds,
+                              int rc);
+
+int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid,
+                         const libxl_asyncop_how *ao_how)
 {
-    GC_INIT(ctx);
+    AO_CREATE(ctx, domid, ao_how);
+    libxl__domain_destroy_state *dds;
+
+    GCNEW(dds);
+    dds->ao = ao;
+    dds->domid = domid;
+    dds->callback = domain_destroy_cb;
+    libxl__domain_destroy(egc, dds);
+
+    return AO_INPROGRESS;
+}
+
+static void domain_destroy_cb(libxl__egc *egc, libxl__domain_destroy_state
*dds,
+                              int rc)
+{
+    STATE_AO_GC(dds->ao);
+
+    if (rc)
+        LOG(ERROR, "destruction of domain %u failed", dds->domid);
+
+    libxl__ao_complete(egc, ao, rc);
+}
+
+/* Callbacks for libxl__domain_destroy */
+
+static void stubdom_destroy_callback(libxl__egc *egc,
+                                     libxl__destroy_domid_state *dis,
+                                     int rc);
+
+static void domain_destroy_callback(libxl__egc *egc,
+                                    libxl__destroy_domid_state *dis,
+                                    int rc);
+
+static void destroy_finish_check(libxl__egc *egc,
+                                 libxl__domain_destroy_state *dds);
+
+void libxl__domain_destroy(libxl__egc *egc, libxl__domain_destroy_state *dds)
+{
+    STATE_AO_GC(dds->ao);
+    uint32_t stubdomid = libxl_get_stubdom_id(CTX, dds->domid);
+
+    if (stubdomid) {
+        dds->stubdom.ao = ao;
+        dds->stubdom.domid = stubdomid;
+        dds->stubdom.callback = stubdom_destroy_callback;
+        libxl__destroy_domid(egc, &dds->stubdom);
+    } else {
+        dds->stubdom_finished = 1;
+    }
+
+    dds->domain.ao = ao;
+    dds->domain.domid = dds->domid;
+    dds->domain.callback = domain_destroy_callback;
+    libxl__destroy_domid(egc, &dds->domain);
+}
+
+static void stubdom_destroy_callback(libxl__egc *egc,
+                                     libxl__destroy_domid_state *dis,
+                                     int rc)
+{
+    STATE_AO_GC(dis->ao);
+    libxl__domain_destroy_state *dds = CONTAINER_OF(dis, *dds, stubdom);
+    const char *savefile;
+
+    if (rc) {
+        LOG(ERROR, "unable to destroy stubdom with domid %u",
dis->domid);
+        dds->rc = rc;
+    }
+
+    dds->stubdom_finished = 1;
+    savefile = libxl__device_model_savefile(gc, dis->domid);
+    rc = libxl__remove_file(gc, savefile);
+    /*
+     * On suspend libxl__domain_save_device_model will have already
+     * unlinked the save file.
+     */
+    if (rc) {
+        LOG(ERROR, "failed to remove device-model savefile %s",
savefile);
+    }
+
+    destroy_finish_check(egc, dds);
+}
+
+static void domain_destroy_callback(libxl__egc *egc,
+                                    libxl__destroy_domid_state *dis,
+                                    int rc)
+{
+    STATE_AO_GC(dis->ao);
+    libxl__domain_destroy_state *dds = CONTAINER_OF(dis, *dds, domain);
+
+    if (rc) {
+        LOG(ERROR, "unable to destroy guest with domid %u",
dis->domid);
+        dds->rc = rc;
+    }
+
+    dds->domain_finished = 1;
+    destroy_finish_check(egc, dds);
+}
+
+static void destroy_finish_check(libxl__egc *egc,
+                                 libxl__domain_destroy_state *dds)
+{
+    if (!(dds->domain_finished && dds->stubdom_finished))
+        return;
+
+    dds->callback(egc, dds, dds->rc);
+}
+
+/* Callbacks for libxl__destroy_domid */
+static void devices_destroy_cb(libxl__egc *egc,
+                               libxl__devices_remove_state *drs,
+                               int rc);
+
+void libxl__destroy_domid(libxl__egc *egc, libxl__destroy_domid_state *dis)
+{
+    STATE_AO_GC(dis->ao);
+    libxl_ctx *ctx = CTX;
+    uint32_t domid = dis->domid;
     char *dom_path;
-    char *vm_path;
     char *pid;
     int rc, dm_present;

@@ -1127,7 +1249,7 @@ int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid)
     case ERROR_INVAL:
         LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "non-existant domain %d",
domid);
     default:
-        return rc;
+        goto out;
     }

     switch (libxl__domain_type(gc, domid)) {
@@ -1160,7 +1282,37 @@ int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid)

         libxl__qmp_cleanup(gc, domid);
     }
-    if (libxl__devices_destroy(gc, domid) < 0)
+    dis->drs.ao = ao;
+    dis->drs.domid = domid;
+    dis->drs.callback = devices_destroy_cb;
+    dis->drs.force = 1;
+    libxl__devices_destroy(egc, &dis->drs);
+    return;
+
+out:
+    assert(rc);
+    dis->callback(egc, dis, rc);
+    return;
+}
+
+static void devices_destroy_cb(libxl__egc *egc,
+                               libxl__devices_remove_state *drs,
+                               int rc)
+{
+    STATE_AO_GC(drs->ao);
+    libxl__destroy_domid_state *dis = CONTAINER_OF(drs, *dis, drs);
+    libxl_ctx *ctx = CTX;
+    uint32_t domid = dis->domid;
+    char *dom_path;
+    char *vm_path;
+
+    dom_path = libxl__xs_get_dompath(gc, domid);
+    if (!dom_path) {
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+    if (rc < 0)
         LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
                    "libxl__devices_destroy failed for %d", domid);

@@ -1183,9 +1335,10 @@ int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid)
         goto out;
     }
     rc = 0;
+
 out:
-    GC_FREE;
-    return rc;
+    dis->callback(egc, dis, rc);
+    return;
 }

 int libxl_console_exec(libxl_ctx *ctx, uint32_t domid, int cons_num,
libxl_console_type type)
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 6bd028f..d5dc2a0 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -526,7 +526,8 @@ int libxl_domain_suspend(libxl_ctx *ctx,
libxl_domain_suspend_info *info,
 int libxl_domain_resume(libxl_ctx *ctx, uint32_t domid, int suspend_cancel);
 int libxl_domain_shutdown(libxl_ctx *ctx, uint32_t domid);
 int libxl_domain_reboot(libxl_ctx *ctx, uint32_t domid);
-int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid);
+int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid,
+                         const libxl_asyncop_how *ao_how);
 int libxl_domain_preserve(libxl_ctx *ctx, uint32_t domid,
libxl_domain_create_info *info, const char *name_suffix, libxl_uuid new_uuid);

 /* get max. number of cpus supported by hypervisor */
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index e5999c0..1df116e 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -586,6 +586,12 @@ static void domcreate_complete(libxl__egc *egc,
                                libxl__domain_create_state *dcs,
                                int rc);

+/* If creation is not successful, this callback will be executed
+ * when domain destruction is finished */
+static void domcreate_destruction_cb(libxl__egc *egc,
+                                     libxl__domain_destroy_state *dds,
+                                     int rc);
+
 static void initiate_domain_create(libxl__egc *egc,
                                    libxl__domain_create_state *dcs)
 {
@@ -860,16 +866,31 @@ static void domcreate_complete(libxl__egc *egc,

     if (rc) {
         if (dcs->guest_domid) {
-            int rc2 = libxl_domain_destroy(CTX, dcs->guest_domid);
-            if (rc2)
-                LOG(ERROR, "unable to destroy domain %d following"
-                    " failed creation", dcs->guest_domid);
+            dcs->dds.ao = ao;
+            dcs->dds.domid = dcs->guest_domid;
+            dcs->dds.callback = domcreate_destruction_cb;
+            libxl__domain_destroy(egc, &dcs->dds);
+            return;
         }
         dcs->guest_domid = -1;
     }
     dcs->callback(egc, dcs, rc, dcs->guest_domid);
 }

+static void domcreate_destruction_cb(libxl__egc *egc,
+                                     libxl__domain_destroy_state *dds,
+                                     int rc)
+{
+    STATE_AO_GC(dds->ao);
+    libxl__domain_create_state *dcs = CONTAINER_OF(dds, *dcs, dds);
+
+    if (rc)
+        LOG(ERROR, "unable to destroy domain %u following failed
creation",
+                   dds->domid);
+
+    dcs->callback(egc, dcs, ERROR_FAIL, dcs->guest_domid);
+}
+
 /*----- application-facing domain creation interface -----*/

 typedef struct {
diff --git a/tools/libxl/libxl_device.c b/tools/libxl/libxl_device.c
index d898a89..882e647 100644
--- a/tools/libxl/libxl_device.c
+++ b/tools/libxl/libxl_device.c
@@ -58,6 +58,48 @@ int libxl__parse_backend_path(libxl__gc *gc,
     return libxl__device_kind_from_string(strkind, &dev->backend_kind);
 }

+static int libxl__num_devices(libxl__gc *gc, uint32_t domid)
+{
+    char *path;
+    unsigned int num_kinds, num_devs;
+    char **kinds = NULL, **devs = NULL;
+    int i, j, rc = 0;
+    libxl__device dev;
+    libxl__device_kind kind;
+    int numdevs = 0;
+
+    path = GCSPRINTF("/local/domain/%d/device", domid);
+    kinds = libxl__xs_directory(gc, XBT_NULL, path, &num_kinds);
+    if (!kinds) {
+        if (errno != ENOENT) {
+            LOGE(ERROR, "unable to get xenstore device listing %s",
path);
+            rc = ERROR_FAIL;
+            goto out;
+        }
+        num_kinds = 0;
+    }
+    for (i = 0; i < num_kinds; i++) {
+        if (libxl__device_kind_from_string(kinds[i], &kind))
+            continue;
+
+        path = GCSPRINTF("/local/domain/%d/device/%s", domid,
kinds[i]);
+        devs = libxl__xs_directory(gc, XBT_NULL, path, &num_devs);
+        if (!devs)
+            continue;
+        for (j = 0; j < num_devs; j++) {
+            path = GCSPRINTF("/local/domain/%d/device/%s/%s/backend",
+                             domid, kinds[i], devs[j]);
+            path = libxl__xs_read(gc, XBT_NULL, path);
+            if (path && libxl__parse_backend_path(gc, path, &dev)
== 0) {
+                numdevs++;
+            }
+        }
+    }
+out:
+    if (rc) return rc;
+    return numdevs;
+}
+
 int libxl__device_generic_add(libxl__gc *gc, libxl__device *device,
                              char **bents, char **fents)
 {
@@ -367,6 +409,23 @@ void libxl__prepare_ao_device(libxl__ao_device *aodev,
libxl__ao *ao,
     aodev->base = base;
 }

+int libxl__ao_device_check_last(libxl__gc *gc, libxl__ao_device *device,
+                                libxl__ao_device *list, int num)
+{
+    int i, error = 0;
+
+    device->active = 0;
+    for (i = 0; i < num; i++) {
+        if (list[i].active)
+            return +1;
+
+        if (list[i].rc)
+            error = list[i].rc;
+    }
+
+    return error;
+}
+
 int libxl__device_destroy(libxl__gc *gc, libxl__device *dev)
 {
     libxl_ctx *ctx = libxl__gc_owner(gc);
@@ -381,16 +440,35 @@ int libxl__device_destroy(libxl__gc *gc, libxl__device
*dev)
     return 0;
 }

-int libxl__devices_destroy(libxl__gc *gc, uint32_t domid)
+/* Callback for device destruction */
+
+static void device_remove_callback(libxl__egc *egc, libxl__ao_device *aodev);
+
+void libxl__devices_destroy(libxl__egc *egc, libxl__devices_remove_state *drs)
 {
+    STATE_AO_GC(drs->ao);
     libxl_ctx *ctx = libxl__gc_owner(gc);
+    uint32_t domid = drs->domid;
     char *path;
-    unsigned int num_kinds, num_devs;
+    unsigned int num_kinds, num_dev_xsentries;
     char **kinds = NULL, **devs = NULL;
-    int i, j;
-    libxl__device dev;
+    int i, j, numdev = 0, rc = 0;
+    libxl__device *dev;
+    libxl__ao_device *aodev;
     libxl__device_kind kind;

+    drs->num_devices = libxl__num_devices(gc, drs->domid);
+    if (drs->num_devices < 0) {
+        LOG(ERROR, "unable to get number of devices for domain %u",
drs->domid);
+        rc = drs->num_devices;
+        goto out;
+    }
+
+    GCNEW_ARRAY(drs->aodev, drs->num_devices);
+    for (i = 0; i < drs->num_devices; i++) {
+        libxl__prepare_ao_device(&drs->aodev[i], drs->ao,
&drs->aodev);
+    }
+
     path = libxl__sprintf(gc, "/local/domain/%d/device", domid);
     kinds = libxl__xs_directory(gc, XBT_NULL, path, &num_kinds);
     if (!kinds) {
@@ -406,19 +484,25 @@ int libxl__devices_destroy(libxl__gc *gc, uint32_t domid)
             continue;

         path = libxl__sprintf(gc, "/local/domain/%d/device/%s",
domid, kinds[i]);
-        devs = libxl__xs_directory(gc, XBT_NULL, path, &num_devs);
+        devs = libxl__xs_directory(gc, XBT_NULL, path, &num_dev_xsentries);
         if (!devs)
             continue;
-        for (j = 0; j < num_devs; j++) {
+        for (j = 0; j < num_dev_xsentries; j++) {
             path = libxl__sprintf(gc,
"/local/domain/%d/device/%s/%s/backend",
                                   domid, kinds[i], devs[j]);
             path = libxl__xs_read(gc, XBT_NULL, path);
-            if (path && libxl__parse_backend_path(gc, path, &dev)
== 0) {
-                dev.domid = domid;
-                dev.kind = kind;
-                dev.devid = atoi(devs[j]);
-
-                libxl__device_destroy(gc, &dev);
+            GCNEW(dev);
+            if (path && libxl__parse_backend_path(gc, path, dev) == 0)
{
+                aodev = &drs->aodev[numdev];
+                dev->domid = domid;
+                dev->kind = kind;
+                dev->devid = atoi(devs[j]);
+                aodev->action = DEVICE_DISCONNECT;
+                aodev->dev = dev;
+                aodev->callback = device_remove_callback;
+                aodev->force = drs->force;
+                libxl__initiate_device_remove(egc, aodev);
+                numdev++;
             }
         }
     }
@@ -426,17 +510,22 @@ int libxl__devices_destroy(libxl__gc *gc, uint32_t domid)
     /* console 0 frontend directory is not under
/local/domain/<domid>/device */
     path = libxl__sprintf(gc, "/local/domain/%d/console/backend",
domid);
     path = libxl__xs_read(gc, XBT_NULL, path);
+    GCNEW(dev);
     if (path && strcmp(path, "") &&
-        libxl__parse_backend_path(gc, path, &dev) == 0) {
-        dev.domid = domid;
-        dev.kind = LIBXL__DEVICE_KIND_CONSOLE;
-        dev.devid = 0;
-
-        libxl__device_destroy(gc, &dev);
+        libxl__parse_backend_path(gc, path, dev) == 0) {
+        dev->domid = domid;
+        dev->kind = LIBXL__DEVICE_KIND_CONSOLE;
+        dev->devid = 0;
+
+        /* Currently console devices can be destroyed synchronously by just
+         * removing xenstore entries, this is what libxl__device_destroy does.
+         */
+        libxl__device_destroy(gc, dev);
     }

 out:
-    return 0;
+    if (!numdev) drs->callback(egc, drs, rc);
+    return;
 }

 /* Callbacks for device related operations */
@@ -536,6 +625,39 @@ static void device_backend_cleanup(libxl__gc *gc,
libxl__ao_device *aodev)
     libxl__ev_devstate_cancel(gc, &aodev->ds);
 }

+static void device_remove_callback(libxl__egc *egc, libxl__ao_device *aodev)
+{
+    STATE_AO_GC(aodev->ao);
+    libxl__devices_remove_state *drs = CONTAINER_OF(aodev->base, *drs,
aodev);
+    char *be_path = libxl__device_backend_path(gc, aodev->dev);
+    char *fe_path = libxl__device_frontend_path(gc, aodev->dev);
+    xs_transaction_t t = 0;
+    int rc = 0, ret = 0;
+
+    if (aodev->action == DEVICE_DISCONNECT) {
+        do {
+            t = xs_transaction_start(CTX->xsh);
+            libxl__xs_path_cleanup(gc, t, fe_path);
+            libxl__xs_path_cleanup(gc, t, be_path);
+            ret = !xs_transaction_end(CTX->xsh, t, 0);
+        } while (ret && errno == EAGAIN);
+
+        if (ret) {
+            LOGE(ERROR, "unable to finish transaction");
+            rc = ERROR_FAIL;
+            goto out;
+        }
+    }
+
+out:
+    rc = libxl__ao_device_check_last(gc, aodev, drs->aodev,
+                                     drs->num_devices);
+    if (rc > 0) return;
+
+    drs->callback(egc, drs, rc);
+    return;
+}
+
 int libxl__wait_for_device_model(libxl__gc *gc,
                                  uint32_t domid, char *state,
                                  libxl__spawn_starting *spawning,
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index d987347..99059dc 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -673,6 +673,10 @@ static void spawn_stubdom_pvqemu_cb(libxl__egc *egc,
                                 libxl__dm_spawn_state *stubdom_dmss,
                                 int rc);

+static void spaw_stubdom_pvqemu_destroy_cb(libxl__egc *egc,
+                                           libxl__destroy_domid_state *dis,
+                                           int rc);
+
 void libxl__spawn_stub_dm(libxl__egc *egc, libxl__stub_dm_spawn_state *sdss)
 {
     STATE_AO_GC(sdss->dm.spawn.ao);
@@ -893,12 +897,31 @@ static void spawn_stubdom_pvqemu_cb(libxl__egc *egc,

  out:
     if (rc) {
-        if (dm_domid)
-            libxl_domain_destroy(CTX, dm_domid);
+        if (dm_domid) {
+            sdss->dis.ao = ao;
+            sdss->dis.domid = dm_domid;
+            sdss->dis.callback = spaw_stubdom_pvqemu_destroy_cb;
+            libxl__destroy_domid(egc, &sdss->dis);
+            return;
+        }
     }
     sdss->callback(egc, &sdss->dm, rc);
 }

+static void spaw_stubdom_pvqemu_destroy_cb(libxl__egc *egc,
+                                           libxl__destroy_domid_state *dis,
+                                           int rc)
+{
+    libxl__stub_dm_spawn_state *sdss = CONTAINER_OF(dis, *sdss, dis);
+    STATE_AO_GC(sdss->dis.ao);
+
+    if (rc)
+        LOG(ERROR, "destruction of domain %u after failed creation
failed",
+                   sdss->pvqemu.guest_domid);
+
+    sdss->callback(egc, &sdss->dm, rc);
+}
+
 /* callbacks passed to libxl__spawn_spawn */
 static void device_model_confirm(libxl__egc *egc, libxl__spawn_state *spawn,
                                  const char *xsdata);
@@ -1091,48 +1114,24 @@ int libxl__destroy_device_model(libxl__gc *gc, uint32_t
domid)
     int ret;

     pid = libxl__xs_read(gc, XBT_NULL, libxl__sprintf(gc,
"/local/domain/%d/image/device-model-pid", domid));
-    if (!pid) {
-        int stubdomid = libxl_get_stubdom_id(ctx, domid);
-        const char *savefile;
-
-        if (!stubdomid) {
-            LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "Couldn''t
find device model''s pid");
-            ret = ERROR_INVAL;
-            goto out;
-        }
-        LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "Device model is a stubdom,
domid=%d", stubdomid);
-        ret = libxl_domain_destroy(ctx, stubdomid);
-        if (ret)
-            goto out;
-
-        savefile = libxl__device_model_savefile(gc, domid);
-        ret = unlink(savefile);
-        /*
-         * On suspend libxl__domain_save_device_model will have already
-         * unlinked the save file.
-         */
-        if (ret && errno == ENOENT) ret = 0;
-        if (ret) {
-            LIBXL__LOG_ERRNO(ctx, XTL_ERROR,
-                             "failed to remove device-model savefile
%s\n",
-                             savefile);
-            goto out;
-        }
+    if (!pid || !atoi(pid)) {
+        LOG(ERROR, "could not find device-model''s pid for dom
%u", domid);
+        ret = ERROR_FAIL;
+        goto out;
+    }
+    ret = kill(atoi(pid), SIGHUP);
+    if (ret < 0 && errno == ESRCH) {
+        LOG(ERROR, "Device Model already exited");
+        ret = 0;
+    } else if (ret == 0) {
+        LOG(DEBUG, "Device Model signaled");
+        ret = 0;
     } else {
-        ret = kill(atoi(pid), SIGHUP);
-        if (ret < 0 && errno == ESRCH) {
-            LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "Device Model already
exited");
-            ret = 0;
-        } else if (ret == 0) {
-            LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "Device Model
signaled");
-            ret = 0;
-        } else {
-            LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "failed to kill Device
Model [%d]",
-                    atoi(pid));
-            ret = ERROR_FAIL;
-            goto out;
-        }
+        LOGE(ERROR, "failed to kill Device Model [%d]", atoi(pid));
+        ret = ERROR_FAIL;
+        goto out;
     }
+
     xs_rm(ctx->xsh, XBT_NULL, libxl__sprintf(gc,
"/local/domain/0/device-model/%d", domid));
     xs_rm(ctx->xsh, XBT_NULL, libxl__sprintf(gc,
"/local/domain/%d/hvmloader", domid));

diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 5605440..5c15ca5 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -817,7 +817,6 @@ _hidden char *libxl__device_frontend_path(libxl__gc *gc,
libxl__device *device);
 _hidden int libxl__parse_backend_path(libxl__gc *gc, const char *path,
                                       libxl__device *dev);
 _hidden int libxl__device_destroy(libxl__gc *gc, libxl__device *dev);
-_hidden int libxl__devices_destroy(libxl__gc *gc, uint32_t domid);
 _hidden int libxl__wait_for_backend(libxl__gc *gc, char *be_path, char *state);

 /*
@@ -1820,6 +1819,16 @@ typedef void libxl__device_callback(libxl__egc*,
libxl__ao_device*);
 _hidden void libxl__prepare_ao_device(libxl__ao_device *aodev, libxl__ao *ao,
                                       libxl__ao_device **base);

+/* Check if there are devices that have pending events in the array
+ * pointed to by the "list" parameter. Return values can be:
+ * < 0: All done, but error(s) found.
+ * 0: All done
+ * > 0: Not all done
+ * The passed libxl__ao_device struct in "device" is marked as
finished.
+ */
+_hidden int libxl__ao_device_check_last(libxl__gc *gc, libxl__ao_device
*device,
+                                        libxl__ao_device *list, int num);
+
 struct libxl__ao_device {
     /* filled in by user */
     libxl__ao *ao;
@@ -1847,6 +1856,86 @@ struct libxl__ao_device {
 _hidden void libxl__initiate_device_remove(libxl__egc *egc,
                                            libxl__ao_device *aodev);

+/*----- Domain destruction -----*/
+
+/* Domain destruction has been split into two functions:
+ *
+ * libxl__domain_destroy is the main destroy function, which detects
+ * stubdoms and calls libxl__destroy_domid on the domain and its
+ * stubdom if present, creating a different libxl__destroy_domid_state
+ * for each one of them.
+ *
+ * libxl__destroy_domid actually destroys the domain, but it
+ * doesn''t check for stubdomains, since that would involve
+ * recursion, which we want to avoid.
+ */
+
+typedef struct libxl__domain_destroy_state libxl__domain_destroy_state;
+typedef struct libxl__destroy_domid_state libxl__destroy_domid_state;
+typedef struct libxl__devices_remove_state libxl__devices_remove_state;
+
+typedef void libxl__domain_destroy_cb(libxl__egc *egc,
+                                      libxl__domain_destroy_state *dds,
+                                      int rc);
+
+typedef void libxl__domid_destroy_cb(libxl__egc *egc,
+                                     libxl__destroy_domid_state *dis,
+                                     int rc);
+
+typedef void libxl__devices_remove_callback(libxl__egc *egc,
+                                            libxl__devices_remove_state *drs,
+                                            int rc);
+
+struct libxl__devices_remove_state {
+    /* filled in by user */
+    libxl__ao *ao;
+    uint32_t domid;
+    libxl__devices_remove_callback *callback;
+    int force; /* libxl_device_TYPE_destroy rather than _remove */
+    /* private */
+    libxl__ao_device *aodev;
+    int num_devices;
+};
+
+struct libxl__destroy_domid_state {
+    /* filled in by user */
+    libxl__ao *ao;
+    uint32_t domid;
+    libxl__domid_destroy_cb *callback;
+    /* private to implementation */
+    libxl__devices_remove_state drs;
+};
+
+struct libxl__domain_destroy_state {
+    /* filled by the user */
+    libxl__ao *ao;
+    uint32_t domid;
+    libxl__domain_destroy_cb *callback;
+    /* Private */
+    int rc;
+    uint32_t stubdomid;
+    libxl__destroy_domid_state stubdom;
+    int stubdom_finished;
+    libxl__destroy_domid_state domain;
+    int domain_finished;
+};
+
+/*
+ * Entry point for domain destruction
+ * This function checks for stubdom presence and then calls
+ * libxl__destroy_domid on the passed domain and its stubdom if found.
+ */
+_hidden void libxl__domain_destroy(libxl__egc *egc,
+                                   libxl__domain_destroy_state *dds);
+
+/* Used to destroy a domain with the passed id (it doesn''t check for
stubs) */
+_hidden void libxl__destroy_domid(libxl__egc *egc,
+                                  libxl__destroy_domid_state *dis);
+
+/* Entry point for devices destruction */
+_hidden void libxl__devices_destroy(libxl__egc *egc,
+                                    libxl__devices_remove_state *drs);
+
 /*----- device model creation -----*/

 /* First layer; wraps libxl__spawn_spawn. */
@@ -1880,6 +1969,7 @@ typedef struct {
     libxl_domain_config dm_config;
     libxl__domain_build_state dm_state;
     libxl__dm_spawn_state pvqemu;
+    libxl__destroy_domid_state dis;
 } libxl__stub_dm_spawn_state;

 _hidden void libxl__spawn_stub_dm(libxl__egc *egc,
libxl__stub_dm_spawn_state*);
@@ -1906,6 +1996,8 @@ struct libxl__domain_create_state {
     libxl__stub_dm_spawn_state dmss;
         /* If we''re not doing stubdom, we use only dmss.dm,
          * for the non-stubdom device model. */
+    /* necessary if the domain creation failed and we have to destroy it */
+    libxl__domain_destroy_state dds;
 };

 /*
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index ca988d6..3220e89 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -1389,7 +1389,7 @@ static int handle_domain_death(uint32_t domid,
         /* fall-through */
     case LIBXL_ACTION_ON_SHUTDOWN_DESTROY:
         LOG("Domain %d needs to be cleaned up: destroying the
domain", domid);
-        libxl_domain_destroy(ctx, domid);
+        libxl_domain_destroy(ctx, domid, 0);
         break;

     case LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_DESTROY:
@@ -1987,7 +1987,7 @@ start:
 error_out:
     release_lock();
     if (libxl_domid_valid_guest(domid))
-        libxl_domain_destroy(ctx, domid);
+        libxl_domain_destroy(ctx, domid, 0);

 out:
     if (logfile != 2)
@@ -2550,7 +2550,7 @@ static void destroy_domain(const char *p)
         fprintf(stderr, "Cannot destroy privileged domain 0.\n\n");
         exit(-1);
     }
-    rc = libxl_domain_destroy(ctx, domid);
+    rc = libxl_domain_destroy(ctx, domid, 0);
     if (rc) { fprintf(stderr,"destroy failed (rc=%d)\n",rc);
exit(-1); }
 }

@@ -2824,7 +2824,7 @@ static int save_domain(const char *p, const char
*filename, int checkpoint,
     if (checkpoint)
         libxl_domain_resume(ctx, domid, 1);
     else
-        libxl_domain_destroy(ctx, domid);
+        libxl_domain_destroy(ctx, domid, 0);

     exit(0);
 }
@@ -3084,7 +3084,7 @@ static void migrate_domain(const char *domain_spec, const
char *rune,
     }

     fprintf(stderr, "migration sender: Target reports successful
startup.\n");
-    libxl_domain_destroy(ctx, domid); /* bang! */
+    libxl_domain_destroy(ctx, domid, 0); /* bang! */
     fprintf(stderr, "Migration successful.\n");
     exit(0);

@@ -3237,7 +3237,7 @@ static void migrate_receive(int debug, int daemonize, int
monitor,
     if (rc) {
         fprintf(stderr, "migration target: Failure, destroying our
copy.\n");

-        rc2 = libxl_domain_destroy(ctx, domid);
+        rc2 = libxl_domain_destroy(ctx, domid, 0);
         if (rc2) {
             fprintf(stderr, "migration target: Failed to destroy our
copy"
                     " (code %d).\n", rc2);
diff --git a/tools/python/xen/lowlevel/xl/xl.c
b/tools/python/xen/lowlevel/xl/xl.c
index 1db777d..4ff3120 100644
--- a/tools/python/xen/lowlevel/xl/xl.c
+++ b/tools/python/xen/lowlevel/xl/xl.c
@@ -437,7 +437,7 @@ static PyObject *pyxl_domain_destroy(XlObject *self,
PyObject *args)
     int domid;
     if ( !PyArg_ParseTuple(args, "i", &domid) )
         return NULL;
-    if ( libxl_domain_destroy(self->ctx, domid) ) {
+    if ( libxl_domain_destroy(self->ctx, domid, 0) ) {
         PyErr_SetString(xl_error_obj, "cannot destroy domain");
         return NULL;
     }
--
1.7.7.5 (Apple Git-26)