Ocfs2 devel - Jan 2006 - [PATCH 11/11] ocfs2: add userspace clustering

This patch adds userspace clustering support as well as separate
 configurability of disk and userspace hearbeat modes.

 fs/Kconfig                        |   22 +++
 fs/ocfs2/cluster/Makefile         |    5 
 fs/ocfs2/cluster/nodemanager.c    |    5 
 fs/ocfs2/cluster/nodemanager.h    |    5 
 fs/ocfs2/cluster/user_heartbeat.c |  213 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 244 insertions(+), 6 deletions(-)

Signed-off-by: Jeff Mahoney <jeffm at suse.com>
diff -ruNpX dontdiff linux-2.6.15-staging1/fs/Kconfig
linux-2.6.15-staging2/fs/Kconfig
--- linux-2.6.15-staging1/fs/Kconfig	2006-01-08 19:23:29.639398856 -0500
+++ linux-2.6.15-staging2/fs/Kconfig	2006-01-08 19:12:12.846287048 -0500
@@ -368,6 +368,28 @@ config OCFS2_FS
 	          - POSIX ACLs
 	          - readpages / writepages (not user visible)
 
+config OCFS2_FS_O2CB
+	tristate "O2CB Kernelspace Clustering"
+	depends on OCFS2_FS
+	default y
+	help
+	  OCFS2 includes a simple kernelspace clustering package that requires
+	  a very small userspace complement to configure it. However, it is
+	  not as flexible as an external cluster manager, and is only
+	  limited to maintaining a cluster for OCFS2 file systems.
+	  If you are not using an external clustering package, choose
+	  this option.
+
+config OCFS2_FS_USERSPACE_CLUSTER
+	tristate "Userspace Clustering"
+	depends on OCFS2_FS
+	default y
+	help
+	  This option will allow OCFS2 to use userspace clustering services
+	  via configfs and sysfs. Currently, the only supported userspace
+	  clustering agent is the Linux-HA project from http://www.linux-ha.org.
+	  If you are using a userspace cluster manager, choose this option.
+
 config MINIX_FS
 	tristate "Minix fs support"
 	help
diff -ruNpX dontdiff linux-2.6.15-staging1/fs/ocfs2/cluster/Makefile
linux-2.6.15-staging2/fs/ocfs2/cluster/Makefile
--- linux-2.6.15-staging1/fs/ocfs2/cluster/Makefile	2006-01-08
20:11:03.024618368 -0500
+++ linux-2.6.15-staging2/fs/ocfs2/cluster/Makefile	2006-01-08
19:12:12.846287048 -0500
@@ -1,6 +1,9 @@
-obj-$(CONFIG_OCFS2_FS) += ocfs2_nodemanager.o ocfs2_disk_heartbeat.o
+obj-$(CONFIG_OCFS2_FS) += ocfs2_nodemanager.o
+obj-$(CONFIG_OCFS2_FS_O2CB) += ocfs2_disk_heartbeat.o
+obj-$(CONFIG_OCFS2_FS_USERSPACE_CLUSTER) += ocfs2_user_heartbeat.o
 
 ocfs2_nodemanager-objs := heartbeat.o masklog.o sys.o nodemanager.o \
 	tcp.o ver.o
 
 ocfs2_disk_heartbeat-objs := disk_heartbeat.o quorum.o
+ocfs2_user_heartbeat-objs := user_heartbeat.o
diff -ruNpX dontdiff linux-2.6.15-staging1/fs/ocfs2/cluster/nodemanager.c
linux-2.6.15-staging2/fs/ocfs2/cluster/nodemanager.c
--- linux-2.6.15-staging1/fs/ocfs2/cluster/nodemanager.c	2006-01-08
20:11:08.002861560 -0500
+++ linux-2.6.15-staging2/fs/ocfs2/cluster/nodemanager.c	2006-01-08
20:07:52.077646720 -0500
@@ -217,11 +217,6 @@ static struct o2nm_cluster *to_o2nm_clus
 		: NULL;
 }
 
-static struct o2nm_node *to_o2nm_node(struct config_item *item)
-{
-	return item ? container_of(item, struct o2nm_node, nd_item) : NULL;
-}
-
 static void o2nm_node_release(struct config_item *item)
 {
 	struct o2nm_node *node = to_o2nm_node(item);
diff -ruNpX dontdiff linux-2.6.15-staging1/fs/ocfs2/cluster/nodemanager.h
linux-2.6.15-staging2/fs/ocfs2/cluster/nodemanager.h
--- linux-2.6.15-staging1/fs/ocfs2/cluster/nodemanager.h	2006-01-08
20:11:10.510480344 -0500
+++ linux-2.6.15-staging2/fs/ocfs2/cluster/nodemanager.h	2006-01-08
19:12:12.847286896 -0500
@@ -74,4 +74,9 @@ struct o2nm_node *o2nm_get_node_by_ip(__
 void o2nm_node_get(struct o2nm_node *node);
 void o2nm_node_put(struct o2nm_node *node);
 
+static inline struct o2nm_node *to_o2nm_node(struct config_item *item)
+{
+	return container_of(item, struct o2nm_node, nd_item);
+}
+
 #endif /* O2CLUSTER_NODEMANAGER_H */
diff -ruNpX dontdiff linux-2.6.15-staging1/fs/ocfs2/cluster/user_heartbeat.c
linux-2.6.15-staging2/fs/ocfs2/cluster/user_heartbeat.c
--- linux-2.6.15-staging1/fs/ocfs2/cluster/user_heartbeat.c	1969-12-31
19:00:00.000000000 -0500
+++ linux-2.6.15-staging2/fs/ocfs2/cluster/user_heartbeat.c	2006-01-08
19:53:20.452153936 -0500
@@ -0,0 +1,213 @@
+
+/* The userspace cluster heartbeat directory will be populated with symbolic
+ * links to the nodes who are heartbeating in the given group */
+
+#include <linux/configfs.h>
+#include "heartbeat.h"
+#include "tcp.h"
+#include "nodemanager.h"
+
+struct o2hb_user_region {
+	struct o2hb_heartbeat_resource hr_res;
+	unsigned long hr_live_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)];
+};
+
+static inline struct o2hb_user_region *to_o2hb_user_region(struct
o2hb_heartbeat_resource *hbres)
+{
+	return container_of(hbres, struct o2hb_user_region, hr_res);
+}
+
+static inline struct o2hb_user_region *item_to_o2hb_user_region(struct
config_item *item)
+{
+	return to_o2hb_user_region(to_o2hb_heartbeat_resource(item));
+}
+
+static inline void o2hb_user_region_get(struct o2hb_user_region *region)
+{
+	o2hb_heartbeat_resource_get(&region->hr_res);
+}
+
+static inline void o2hb_user_region_put(struct o2hb_user_region *region)
+{
+	o2hb_heartbeat_resource_put(&region->hr_res);
+}
+static int o2hb_user_group_allow_link(struct config_item *src,
+                                      struct config_item *target)
+{
+	struct o2nm_node *node = to_o2nm_node(target);
+	struct o2hb_user_region *hbr = item_to_o2hb_user_region(src);
+
+	struct o2hb_node_event event = {
+		.hn_item = LIST_HEAD_INIT(event.hn_item),
+		.hn_res = &hbr->hr_res,
+	};
+
+	/* Make sure the target is within the same cluster */
+	if (src->ci_parent->ci_parent != target->ci_parent->ci_parent)
+		return -EPERM;
+
+	printk ("Node %s is up in group %s\n", target->ci_name,
src->ci_name);
+
+	spin_lock(&o2hb_live_lock);
+	set_bit(node->nd_num, hbr->hr_live_bitmap);
+
+	/* also add a link back to the node */
+
+	/* Notify listeners that this node is up */
+	o2hb_queue_node_event(&event, O2HB_NODE_UP_CB, node, node->nd_num);
+	spin_unlock(&o2hb_live_lock);
+
+	o2hb_run_event_list (&event);
+
+	return 0;
+}
+
+static int o2hb_user_group_drop_link(struct config_item *src,
+                                     struct config_item *target)
+{
+	struct o2nm_node *node = to_o2nm_node(target);
+	struct o2hb_user_region *hbr = item_to_o2hb_user_region(src);
+
+	struct o2hb_node_event event = {
+		.hn_item = LIST_HEAD_INIT(event.hn_item),
+		.hn_res = &hbr->hr_res,
+	};
+
+	/* Somehow notify listeners that this node is down */
+	printk ("Node %s is down in group %s\n", target->ci_name,
+	                                         src->ci_name);
+
+	spin_lock(&o2hb_live_lock);
+	clear_bit(node->nd_num, hbr->hr_live_bitmap);
+
+	/* Notify listeners that this node is down */
+	o2hb_queue_node_event (&event, O2HB_NODE_DOWN_CB, node, node->nd_num);
+	spin_unlock(&o2hb_live_lock);
+
+	o2hb_run_event_list (&event);
+
+	return 0;
+}
+
+static struct configfs_item_operations o2hb_user_region_ops = {
+	.allow_link	= o2hb_user_group_allow_link,
+	.drop_link	= o2hb_user_group_drop_link,
+};
+
+struct config_item_type o2hb_user_region_type = {
+	.ct_item_ops	= &o2hb_user_region_ops,
+	.ct_owner	= THIS_MODULE,
+};
+
+/* How to create a heartbeat region */
+static struct config_item *o2hb_heartbeat_group_make_item(struct config_group
*group,
+                                                          const char *name)
+{
+	struct o2hb_user_region *region;
+	struct config_item *ret = NULL;
+
+	region = kzalloc(sizeof (*region), GFP_KERNEL);
+	if (region == NULL)
+		goto out;
+
+	/* mkdir <fs uuid> */
+	config_item_init_type_name(&region->hr_res.hr_item, name,
+	                            &o2hb_user_region_type);
+
+	ret = &region->hr_res.hr_item;
+
+out:
+	if (ret == NULL)
+		kfree(region);
+	return ret;
+}
+
+/* How to remove a heartbeat region */
+static void o2hb_heartbeat_group_drop_item(struct config_group *group,
+                                           struct config_item *item)
+{
+	config_item_put (item);
+}
+
+static struct configfs_group_operations o2hb_user_heartbeat_root_ops = {
+	.make_item	= o2hb_heartbeat_group_make_item,
+	.drop_item	= o2hb_heartbeat_group_drop_item,
+};
+
+static inline struct o2hb_user_region *o2hb_user_region_get_by_name(const char
*resource)
+{
+	struct o2hb_heartbeat_resource *hbr;
+	struct o2hb_user_region *region = NULL;
+
+	hbr = o2hb_heartbeat_resource_get_by_name(resource);
+	if (hbr != NULL)
+		region = to_o2hb_user_region(hbr);
+
+	return region;
+}
+
+static int fill_node_map(const char *resource, unsigned long *map, size_t len)
+{
+	struct o2hb_user_region *region;
+	if (resource == NULL)
+		return -EINVAL;
+
+	if (len > BITS_TO_LONGS(O2NM_MAX_NODES) * sizeof (*map))
+		return -EFAULT;
+
+	region = o2hb_user_region_get_by_name(resource);
+	if (region == NULL)
+		return -ENOENT;
+
+	memcpy (map, region->hr_live_bitmap, len);
+
+	o2hb_user_region_put(region);
+
+	return 0;
+}
+
+static int check_node_status(const char *resource, u8 node_num)
+{
+	int ret = 0;
+
+	if (resource) {
+		struct o2hb_user_region *region;
+		region = o2hb_user_region_get_by_name(resource);
+		if (region == NULL)
+			goto out;
+
+		ret = test_bit(node_num, region->hr_live_bitmap);
+		o2hb_user_region_put(region);
+	} else {
+		struct o2nm_node *node = o2nm_get_node_by_num(node_num);
+		ret = (node && atomic_read(&node->nd_count));
+	}
+out:
+	return ret;
+}
+
+static struct o2hb_heartbeat_group user_heartbeat_group = {
+	.hs_type = {
+		.ct_group_ops	= &o2hb_user_heartbeat_root_ops,
+		.ct_owner	= THIS_MODULE,
+	},
+	.hs_name		= "user",
+	.fill_node_map		= fill_node_map,
+	.check_node_status	= check_node_status,
+};
+
+static int __init init_user_heartbeat(void)
+{
+	return o2hb_register_heartbeat_group(&user_heartbeat_group);
+}
+
+static void __exit exit_user_heartbeat(void)
+{
+	o2hb_unregister_heartbeat_group(&user_heartbeat_group);
+}
+
+MODULE_AUTHOR("Novell");
+MODULE_LICENSE("GPL");
+
+module_init(init_user_heartbeat);
+module_exit(exit_user_heartbeat);