[Libguestfs] [PATCH libnbd 4/4] lib: Atomically update h->state when leaving the locked region.

Richard W.M. Jones
2019-06-05T11:15:37Z

Split h->state into:

 - h->state = the state on entry to the locked region

 - h->next_state = the current state and what the "publicly
visible"
   state will become when we leave the locked region

Some calls to get_state become calls to get_next_state depending on
which of these they are trying to read.  Calls to set_state become
set_next_state because that is what gets updated.

When we leave the locked region we update h->state.

The purpose of this patch is to make it easier to reason about the
state in lockless code.
---
 generator/generator | 23 +++++++++++++----------
 lib/connect.c       | 10 +++++-----
 lib/disconnect.c    |  8 ++++----
 lib/handle.c        |  2 +-
 lib/internal.h      | 15 +++++++++++++--
 lib/rw.c            |  4 ++--
 6 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/generator/generator b/generator/generator
index a3fbea9..d198007 100755
--- a/generator/generator
+++ b/generator/generator
@@ -2415,11 +2415,11 @@ let generate_lib_states_c ()        pr "  enum
state next_state = %s;\n" state_enum;
       pr "\n";
       pr "  r = _enter_%s (h, &next_state, blocked);\n"
state_enum;
-      pr "  if (get_state (h) != next_state) {\n";
+      pr "  if (get_next_state (h) != next_state) {\n";
       pr "    debug (h, \"transition: %%s -> %%s\",\n";
       pr "           \"%s\",\n" display_name;
       pr "           nbd_internal_state_short_string
(next_state));\n";
-      pr "    set_state (h, next_state);\n";
+      pr "    set_next_state (h, next_state);\n";
       pr "  }\n";
       pr "  return r;\n";
       pr "}\n";
@@ -2434,7 +2434,7 @@ let generate_lib_states_c ()    pr "  bool
blocked;\n";
   pr "\n";
   pr "  /* Validate and handle the external event. */\n";
-  pr "  switch (get_state (h))\n";
+  pr "  switch (get_next_state (h))\n";
   pr "  {\n";
   List.iter (
     fun ({ parsed = { display_name; state_enum; events } } as state) ->
@@ -2446,7 +2446,7 @@ let generate_lib_states_c ()            fun (e,
next_state) ->
             pr "    case %s:\n" (c_string_of_external_event e);
             if state != next_state then (
-              pr "      set_state (h, %s);\n"
next_state.parsed.state_enum;
+              pr "      set_next_state (h, %s);\n"
next_state.parsed.state_enum;
               pr "      debug (h, \"event %%s: %%s ->
%%s\",\n";
               pr "             \"%s\", \"%s\",
\"%s\");\n"
                  (string_of_external_event e)
@@ -2462,7 +2462,7 @@ let generate_lib_states_c ()    pr "  }\n";
   pr "\n";
   pr "  set_error (EINVAL, \"external event %%d is invalid in state
%%s\",\n";
-  pr "             ev, nbd_internal_state_short_string (get_state
(h)));\n";
+  pr "             ev, nbd_internal_state_short_string (get_next_state
(h)));\n";
   pr "  return -1;\n";
   pr "\n";
   pr " ok:\n";
@@ -2470,7 +2470,7 @@ let generate_lib_states_c ()    pr "    blocked =
true;\n";
   pr "\n";
   pr "    /* Run a single step. */\n";
-  pr "    switch (get_state (h))\n";
+  pr "    switch (get_next_state (h))\n";
   pr "    {\n";
   List.iter (
     fun { parsed = { state_enum } } ->
@@ -2496,7 +2496,7 @@ let generate_lib_states_c ()    pr "{\n";
   pr "  int r = 0;\n";
   pr "\n";
-  pr "  switch (get_state (h))\n";
+  pr "  switch (get_next_state (h))\n";
   pr "  {\n";
   List.iter (
     fun ({ parsed = { state_enum; events } }) ->
@@ -2542,7 +2542,7 @@ let generate_lib_states_c ()    pr "const char
*\n";
   pr "nbd_unlocked_connection_state (struct nbd_handle *h)\n";
   pr "{\n";
-  pr "  switch (get_state (h))\n";
+  pr "  switch (get_next_state (h))\n";
   pr "  {\n";
   List.iter (
     fun ({ comment; parsed = { display_name; state_enum } }) ->
@@ -2866,8 +2866,11 @@ let generate_lib_api_c ()      let argnames =
List.flatten (List.map name_of_arg args) in
     List.iter (pr ", %s") argnames;
     pr ");\n";
-    if is_locked then
-      pr "  pthread_mutex_unlock (&h->lock);\n";
+    if is_locked then (
+      pr "  if (h->state != h->next_state)\n";
+      pr "    h->state = h->next_state;\n";
+      pr "  pthread_mutex_unlock (&h->lock);\n"
+    );
     pr "  return ret;\n";
     pr "}\n";
     pr "\n";
diff --git a/lib/connect.c b/lib/connect.c
index b889f80..4e3141f 100644
--- a/lib/connect.c
+++ b/lib/connect.c
@@ -38,16 +38,16 @@
 static int
 error_unless_ready (struct nbd_handle *h)
 {
-  if (nbd_internal_is_state_ready (get_state (h)))
+  if (nbd_internal_is_state_ready (get_next_state (h)))
     return 0;

   /* Why did it fail? */
-  if (nbd_internal_is_state_closed (get_state (h))) {
+  if (nbd_internal_is_state_closed (get_next_state (h))) {
     set_error (0, "connection is closed");
     return -1;
   }

-  if (nbd_internal_is_state_dead (get_state (h)))
+  if (nbd_internal_is_state_dead (get_next_state (h)))
     /* Don't set the error here, keep the error set when
      * the connection died.
      */
@@ -55,14 +55,14 @@ error_unless_ready (struct nbd_handle *h)

   /* Should probably never happen. */
   set_error (0, "connection in an unexpected state (%s)",
-             nbd_internal_state_short_string (get_state (h)));
+             nbd_internal_state_short_string (get_next_state (h)));
   return -1;
 }

 static int
 wait_until_connected (struct nbd_handle *h)
 {
-  while (nbd_internal_is_state_connecting (get_state (h))) {
+  while (nbd_internal_is_state_connecting (get_next_state (h))) {
     if (nbd_unlocked_poll (h, -1) == -1)
       return -1;
   }
diff --git a/lib/disconnect.c b/lib/disconnect.c
index 423edaf..95e9a37 100644
--- a/lib/disconnect.c
+++ b/lib/disconnect.c
@@ -29,14 +29,14 @@
 int
 nbd_unlocked_shutdown (struct nbd_handle *h)
 {
-  if (nbd_internal_is_state_ready (get_state (h)) ||
-      nbd_internal_is_state_processing (get_state (h))) {
+  if (nbd_internal_is_state_ready (get_next_state (h)) ||
+      nbd_internal_is_state_processing (get_next_state (h))) {
     if (nbd_unlocked_aio_disconnect (h, 0) == -1)
       return -1;
   }

-  while (!nbd_internal_is_state_closed (get_state (h)) &&
-         !nbd_internal_is_state_dead (get_state (h))) {
+  while (!nbd_internal_is_state_closed (get_next_state (h)) &&
+         !nbd_internal_is_state_dead (get_next_state (h))) {
     if (nbd_unlocked_poll (h, -1) == -1)
       return -1;
   }
diff --git a/lib/handle.c b/lib/handle.c
index cc311ba..8bf5280 100644
--- a/lib/handle.c
+++ b/lib/handle.c
@@ -57,7 +57,7 @@ nbd_create (void)
   s = getenv ("LIBNBD_DEBUG");
   h->debug = s && strcmp (s, "1") == 0;

-  h->state = STATE_START;
+  h->state = h->next_state = STATE_START;
   h->pid = -1;

   h->export_name = strdup ("");
diff --git a/lib/internal.h b/lib/internal.h
index 7290247..1bef1c2 100644
--- a/lib/internal.h
+++ b/lib/internal.h
@@ -80,7 +80,17 @@ struct nbd_handle {
   /* Linked list of close callbacks. */
   struct close_callback *close_callbacks;

-  _Atomic enum state state;     /* State machine. */
+  /* State machine.
+   *
+   * The actual current state is ‘next_state’.  ‘state’ is updated
+   * before we release the lock.
+   *
+   * Note don't access these fields directly, use the SET_NEXT_STATE
+   * macro in generator/states* code, or the set_next_state,
+   * get_next_state and get_state macros in regular code.
+   */
+  _Atomic enum state state;
+  enum state next_state;

   bool structured_replies;      /* If we negotiated NBD_OPT_STRUCTURED_REPLY */

@@ -291,7 +301,8 @@ extern const char *nbd_internal_state_short_string (enum
state state);
 extern enum state_group nbd_internal_state_group (enum state state);
 extern enum state_group nbd_internal_state_group_parent (enum state_group
group);

-#define set_state(h,next_state) ((h)->state) = (next_state)
+#define set_next_state(h,_next_state) ((h)->next_state) = (_next_state)
+#define get_next_state(h) ((h)->next_state)
 #define get_state(h) ((h)->state)

 /* utils.c */
diff --git a/lib/rw.c b/lib/rw.c
index b38d95b..ad9c8a0 100644
--- a/lib/rw.c
+++ b/lib/rw.c
@@ -201,7 +201,7 @@ nbd_internal_command_common (struct nbd_handle *h,
    * be handled automatically on a future cycle around to READY.
    */
   if (h->cmds_to_issue != NULL) {
-    assert (nbd_internal_is_state_processing (get_state (h)));
+    assert (nbd_internal_is_state_processing (get_next_state (h)));
     prev_cmd = h->cmds_to_issue;
     while (prev_cmd->next)
       prev_cmd = prev_cmd->next;
@@ -209,7 +209,7 @@ nbd_internal_command_common (struct nbd_handle *h,
   }
   else {
     h->cmds_to_issue = cmd;
-    if (nbd_internal_is_state_ready (get_state (h)) &&
+    if (nbd_internal_is_state_ready (get_next_state (h)) &&
         nbd_internal_run (h, cmd_issue) == -1)
       return -1;
   }
--
2.21.0