Jeff Mahoney
2006-Feb-21 16:57 UTC
[Ocfs2-devel] [PATCH 06/14] ocfs2: add heartbeat operation registration interface
This patch allows heartbeat modules to register themselves with the
generic heartbeat implementation.
It adds a heartbeat operations structure, refcounted and protected. Currently
there may only be one heartbeat operations module active at any given time,
but this can be changed if multiple implementations are desired.
The registration process starts up o2net, since o2net can't start up
without
heartbeat information.
fs/ocfs2/cluster/Makefile | 6 +-
fs/ocfs2/cluster/disk_heartbeat.c | 62 +++++++++++++++------
fs/ocfs2/cluster/disk_heartbeat.h | 3 -
fs/ocfs2/cluster/heartbeat.c | 110 ++++++++++++++++++++++++++++++++------
fs/ocfs2/cluster/heartbeat.h | 11 +++
fs/ocfs2/cluster/nodemanager.c | 3 -
fs/ocfs2/cluster/quorum.c | 1
7 files changed, 153 insertions(+), 43 deletions(-)
Signed-off-by: Jeff Mahoney <jeffm at suse.com>
diff -ruNpX ../dontdiff
linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/disk_heartbeat.c
linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/disk_heartbeat.c
--- linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/disk_heartbeat.c 2006-02-21
11:44:36.000000000 -0500
+++ linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/disk_heartbeat.c 2006-02-21
11:44:36.000000000 -0500
@@ -849,18 +849,8 @@ static int o2hb_thread(void *data)
return 0;
}
-void o2hb_disk_heartbeat_init(void)
-{
- int i;
-
- for (i = 0; i < ARRAY_SIZE(o2hb_live_slots); i++)
- INIT_LIST_HEAD(&o2hb_live_slots[i]);
-
- memset(o2hb_live_node_bitmap, 0, sizeof(o2hb_live_node_bitmap));
-}
-
/* if we're already in a callback then we're already serialized by the
sem */
-void o2hb_fill_node_map_from_callback(unsigned long *map, unsigned bytes)
+static void o2hb_disk_fill_node_map(unsigned long *map, size_t bytes)
{
BUG_ON(bytes < (BITS_TO_LONGS(O2NM_MAX_NODES) * sizeof(unsigned long)));
@@ -1476,11 +1466,27 @@ static struct configfs_group_operations
.drop_item = o2hb_heartbeat_group_drop_item,
};
-struct config_item_type o2hb_heartbeat_group_type = {
- .ct_group_ops = &o2hb_heartbeat_group_group_ops,
- .ct_item_ops = &o2hb_hearbeat_group_item_ops,
- .ct_attrs = o2hb_heartbeat_group_attrs,
- .ct_owner = THIS_MODULE,
+static int o2hb_disk_heartbeat_group_init(struct o2hb_heartbeat_group *hs)
+{
+ return o2quo_init();
+}
+
+static void o2hb_disk_heartbeat_group_exit(struct o2hb_heartbeat_group *hs)
+{
+ o2quo_exit();
+}
+
+static struct o2hb_heartbeat_group disk_heartbeat_group = {
+ .hs_type = {
+ .ct_group_ops = &o2hb_heartbeat_group_group_ops,
+ .ct_item_ops = &o2hb_hearbeat_group_item_ops,
+ .ct_attrs = o2hb_heartbeat_group_attrs,
+ .ct_owner = THIS_MODULE,
+ },
+ .hs_name = "disk",
+ .init = o2hb_disk_heartbeat_group_init,
+ .exit = o2hb_disk_heartbeat_group_exit,
+ .fill_node_map = o2hb_disk_fill_node_map,
};
/*
@@ -1500,4 +1506,26 @@ void o2hb_stop_all_regions(void)
spin_unlock(&o2hb_live_lock);
}
-EXPORT_SYMBOL_GPL(o2hb_stop_all_regions);
+
+static int __init o2hb_disk_heartbeat_init(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(o2hb_live_slots); i++)
+ INIT_LIST_HEAD(&o2hb_live_slots[i]);
+
+ memset(o2hb_live_node_bitmap, 0, sizeof(o2hb_live_node_bitmap));
+ return o2hb_register_heartbeat_group(&disk_heartbeat_group);
+}
+
+static void __exit o2hb_disk_heartbeat_exit(void)
+{
+ o2hb_unregister_heartbeat_group(&disk_heartbeat_group);
+}
+
+MODULE_AUTHOR("Oracle");
+MODULE_LICENSE("GPL");
+
+module_init(o2hb_disk_heartbeat_init);
+module_exit(o2hb_disk_heartbeat_exit);
+
diff -ruNpX ../dontdiff
linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/disk_heartbeat.h
linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/disk_heartbeat.h
--- linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/disk_heartbeat.h 2006-02-21
11:44:36.000000000 -0500
+++ linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/disk_heartbeat.h 2006-02-21
11:44:36.000000000 -0500
@@ -36,9 +36,6 @@ extern unsigned int o2hb_dead_threshold;
#define O2HB_MIN_DEAD_THRESHOLD 2
#define O2HB_MAX_WRITE_TIMEOUT_MS (O2HB_REGION_TIMEOUT_MS *
(o2hb_dead_threshold - 1))
void o2hb_stop_all_regions(void);
-void o2hb_disk_heartbeat_init(void);
-void o2hb_fill_node_map_from_callback(unsigned long *map, unsigned bytes);
void o2hb_dead_threshold_set(unsigned int threshold);
-extern struct config_item_type o2hb_heartbeat_group_type;
#endif /* O2CLUSTER_DISK_HEARTBEAT_H */
diff -ruNpX ../dontdiff
linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/heartbeat.c
linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/heartbeat.c
--- linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/heartbeat.c 2006-02-21
11:44:36.000000000 -0500
+++ linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/heartbeat.c 2006-02-21
11:44:36.000000000 -0500
@@ -35,7 +35,6 @@
#include <linux/time.h>
#include "heartbeat.h"
-#include "disk_heartbeat.h"
#include "tcp.h"
#include "nodemanager.h"
#include "quorum.h"
@@ -58,6 +57,9 @@ static DECLARE_RWSEM(o2hb_callback_sem);
spinlock_t o2hb_live_lock = SPIN_LOCK_UNLOCKED;
EXPORT_SYMBOL_GPL(o2hb_live_lock);
static LIST_HEAD(o2hb_node_events);
+static DEFINE_SPINLOCK(o2hb_group_lock);
+static LIST_HEAD(o2hb_group_list);
+static struct o2hb_heartbeat_group *o2hb_active_group;
static struct o2hb_callback {
struct list_head list;
@@ -125,6 +127,7 @@ void o2hb_run_event_list(struct o2hb_nod
up_write(&o2hb_callback_sem);
}
+EXPORT_SYMBOL_GPL(o2hb_run_event_list);
void o2hb_queue_node_event(struct o2hb_node_event *event,
enum o2hb_callback_type type,
@@ -142,6 +145,7 @@ void o2hb_queue_node_event(struct o2hb_n
list_add_tail(&event->hn_item, &o2hb_node_events);
}
+EXPORT_SYMBOL_GPL(o2hb_queue_node_event);
void o2hb_init(void)
{
@@ -151,8 +155,11 @@ void o2hb_init(void)
INIT_LIST_HEAD(&o2hb_callbacks[i].list);
INIT_LIST_HEAD(&o2hb_node_events);
+}
- o2hb_disk_heartbeat_init();
+void o2hb_fill_node_map_from_callback(unsigned long *map, unsigned bytes)
+{
+ o2hb_active_group->fill_node_map(map, bytes);
}
/*
@@ -171,31 +178,74 @@ void o2hb_fill_node_map(unsigned long *m
EXPORT_SYMBOL_GPL(o2hb_fill_node_map);
+const char *o2hb_heartbeat_mode(void)
+{
+ const char *ret = "";
+ spin_lock(&o2hb_group_lock);
+ if (o2hb_active_group)
+ ret = o2hb_active_group->hs_name;
+ spin_unlock(&o2hb_group_lock);
+ return ret;
+}
+
+int o2hb_set_heartbeat_mode(const char *type, size_t count)
+{
+ int err = -EINVAL;
+ spin_lock(&o2hb_group_lock);
+ if (o2hb_active_group &&
atomic_read(&o2hb_active_group->hs_count)) {
+ err = -EBUSY;
+ } else {
+ struct o2hb_heartbeat_group *hs;
+ list_for_each_entry(hs, &o2hb_group_list, hs_list) {
+ if (!strncmp(hs->hs_name, type, count - 1)) {
+ o2hb_active_group = hs;
+ err = count;
+ break;
+ }
+ }
+ }
+ spin_unlock(&o2hb_group_lock);
+ return err;
+}
+
/* this is just here to avoid touching group in heartbeat.h which the
* entire damn world #includes */
struct config_group *o2hb_alloc_hb_set(void)
{
struct o2hb_heartbeat_group *hs = NULL;
- struct config_group *ret = NULL;
-
- hs = kcalloc(1, sizeof(struct o2hb_heartbeat_group), GFP_KERNEL);
- if (hs == NULL)
- goto out;
-
- config_group_init_type_name(&hs->hs_group, "heartbeat",
- &o2hb_heartbeat_group_type);
+ int err = 0;
- ret = &hs->hs_group;
-out:
- if (ret == NULL)
- kfree(hs);
- return ret;
+ spin_lock(&o2hb_group_lock);
+ hs = o2hb_active_group;
+ if (hs == NULL) {
+ printk("o2hb: No group types have been associated with "
+ "heartbeat. Please load a group type module.\n");
+ spin_unlock(&o2hb_group_lock);
+ return NULL;
+ }
+ if ((err = try_module_get(hs->hs_type.ct_owner)))
+ atomic_inc(&hs->hs_count);
+ spin_unlock(&o2hb_group_lock);
+
+ if (err == 0)
+ return NULL;
+
+ config_group_init_type_name(&hs->hs_group, "heartbeat",
&hs->hs_type);
+ if (hs->init)
+ err = hs->init(hs);
+ return &hs->hs_group;
}
void o2hb_free_hb_set(struct config_group *group)
{
- struct o2hb_heartbeat_group *hs = to_o2hb_heartbeat_group(group);
- kfree(hs);
+ if (group) {
+ struct o2hb_heartbeat_group *hs = to_o2hb_heartbeat_group(group);
+ if (hs->exit)
+ hs->exit(hs);
+
+ atomic_dec(&hs->hs_count);
+ module_put(hs->hs_type.ct_owner);
+ }
}
/* hb callback registration and issueing */
@@ -280,6 +330,32 @@ int o2hb_unregister_callback(struct o2hb
}
EXPORT_SYMBOL_GPL(o2hb_unregister_callback);
+int o2hb_register_heartbeat_group(struct o2hb_heartbeat_group *group)
+{
+ spin_lock(&o2hb_group_lock);
+ if (list_empty(&o2hb_group_list))
+ o2hb_active_group = group;
+ list_add(&group->hs_list, &o2hb_group_list);
+ atomic_set(&group->hs_count, 0);
+ spin_unlock(&o2hb_group_lock);
+
+ printk("o2cb heartbeat: registered %s mode\n", group->hs_name);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(o2hb_register_heartbeat_group);
+
+int o2hb_unregister_heartbeat_group(struct o2hb_heartbeat_group *group)
+{
+ spin_lock(&o2hb_group_lock);
+ if (o2hb_active_group == group)
+ o2hb_active_group = NULL;
+ list_del_init(&group->hs_list);
+ spin_unlock(&o2hb_group_lock);
+ printk("o2cb heartbeat: unregistered %s mode\n", group->hs_name);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(o2hb_unregister_heartbeat_group);
+
int o2hb_check_node_heartbeating(u8 node_num)
{
unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)];
diff -ruNpX ../dontdiff
linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/heartbeat.h
linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/heartbeat.h
--- linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/heartbeat.h 2006-02-21
11:44:36.000000000 -0500
+++ linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/heartbeat.h 2006-02-21
11:44:36.000000000 -0500
@@ -43,7 +43,13 @@ enum o2hb_callback_type {
struct o2hb_heartbeat_group {
struct config_group hs_group;
- /* some stuff? */
+ struct config_item_type hs_type;
+ const char *hs_name;
+ int (*init)(struct o2hb_heartbeat_group *hs);
+ void (*exit)(struct o2hb_heartbeat_group *hs);
+ void (*fill_node_map)(unsigned long *map, size_t bytes);
+ atomic_t hs_count;
+ struct list_head hs_list;
};
struct o2hb_heartbeat_resource {
@@ -75,6 +81,9 @@ void o2hb_queue_node_event(struct o2hb_n
struct o2nm_node *node, int node_num);
void o2hb_run_event_list(struct o2hb_node_event *queued_event);
+int o2hb_register_heartbeat_group(struct o2hb_heartbeat_group *group);
+int o2hb_unregister_heartbeat_group(struct o2hb_heartbeat_group *group);
+
struct config_group *o2hb_alloc_hb_set(void);
void o2hb_free_hb_set(struct config_group *group);
diff -ruNpX ../dontdiff
linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/Makefile
linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/Makefile
--- linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/Makefile 2006-02-21
11:44:36.000000000 -0500
+++ linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/Makefile 2006-02-21
11:44:36.000000000 -0500
@@ -1,5 +1,7 @@
-obj-$(CONFIG_OCFS2_FS) += ocfs2_nodemanager.o
+obj-$(CONFIG_OCFS2_FS) += ocfs2_nodemanager.o ocfs2_disk_heartbeat.o
ocfs2_nodemanager-objs := nodemanager.o heartbeat.o tcp.o net_proc.o \
- masklog.o quorum.o ver.o disk_heartbeat.o
+ masklog.o ver.o
+
+ocfs2_disk_heartbeat-objs := disk_heartbeat.o quorum.o
diff -ruNpX ../dontdiff
linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/nodemanager.c
linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/nodemanager.c
--- linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/nodemanager.c 2006-02-21
11:44:31.000000000 -0500
+++ linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/nodemanager.c 2006-02-21
11:44:36.000000000 -0500
@@ -755,8 +755,6 @@ static void __exit exit_o2nm(void)
mlog_remove_proc(o2nm_proc);
o2net_proc_exit(o2nm_proc);
remove_proc_entry(O2NM_PROC_PATH, NULL);
-
- o2quo_exit();
o2net_exit();
}
@@ -857,7 +855,6 @@ static int __init init_o2nm(void)
cluster_print_version();
o2hb_init();
- o2quo_init();
o2net_init();
ocfs2_table_header = register_sysctl_table(ocfs2_root_table, 0);
diff -ruNpX ../dontdiff
linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/quorum.c
linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/quorum.c
--- linux-2.6.16-rc4.ocfs2-staging1/fs/ocfs2/cluster/quorum.c 2006-02-21
11:44:31.000000000 -0500
+++ linux-2.6.16-rc4.ocfs2-staging2/fs/ocfs2/cluster/quorum.c 2006-02-21
11:44:36.000000000 -0500
@@ -48,6 +48,7 @@
#include <linux/workqueue.h>
#include "heartbeat.h"
+#include "disk_heartbeat.h"
#include "tcp.h"
#include "nodemanager.h"
#define MLOG_MASK_PREFIX ML_QUORUM