1/* -*- mode: c; c-basic-offset: 8; -*-
2 * vim: noexpandtab sw=8 ts=8 sts=0:
3 *
4 * Copyright (C) 2004, 2005, 2012 Oracle.  All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 * General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public
17 * License along with this program; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 021110-1307, USA.
20 */
21
22#include <linux/kernel.h>
23#include <linux/module.h>
24#include <linux/configfs.h>
25
26#include "heartbeat.h"
27#include "tcp.h"
28#include "nodemanager.h"
29
30#include "masklog.h"
31
32/*
33 * The first heartbeat pass had one global thread that would serialize all hb
34 * callback calls.  This global serializing sem should only be removed once
35 * we've made sure that all callees can deal with being called concurrently
36 * from multiple hb region threads.
37 */
38static DECLARE_RWSEM(r2hb_callback_sem);
39
40/*
41 * multiple hb threads are watching multiple regions.  A node is live
42 * whenever any of the threads sees activity from the node in its region.
43 */
44static DEFINE_SPINLOCK(r2hb_live_lock);
45static unsigned long r2hb_live_node_bitmap[BITS_TO_LONGS(R2NM_MAX_NODES)];
46
47static struct r2hb_callback {
48	struct list_head list;
49} r2hb_callbacks[R2HB_NUM_CB];
50
51enum r2hb_heartbeat_modes {
52	R2HB_HEARTBEAT_LOCAL		= 0,
53	R2HB_HEARTBEAT_GLOBAL,
54	R2HB_HEARTBEAT_NUM_MODES,
55};
56
57char *r2hb_heartbeat_mode_desc[R2HB_HEARTBEAT_NUM_MODES] = {
58		"local",	/* R2HB_HEARTBEAT_LOCAL */
59		"global",	/* R2HB_HEARTBEAT_GLOBAL */
60};
61
62unsigned int r2hb_dead_threshold = R2HB_DEFAULT_DEAD_THRESHOLD;
63unsigned int r2hb_heartbeat_mode = R2HB_HEARTBEAT_LOCAL;
64
65/* Only sets a new threshold if there are no active regions.
66 *
67 * No locking or otherwise interesting code is required for reading
68 * r2hb_dead_threshold as it can't change once regions are active and
69 * it's not interesting to anyone until then anyway. */
70static void r2hb_dead_threshold_set(unsigned int threshold)
71{
72	if (threshold > R2HB_MIN_DEAD_THRESHOLD) {
73		spin_lock(&r2hb_live_lock);
74		r2hb_dead_threshold = threshold;
75		spin_unlock(&r2hb_live_lock);
76	}
77}
78
79static int r2hb_global_hearbeat_mode_set(unsigned int hb_mode)
80{
81	int ret = -1;
82
83	if (hb_mode < R2HB_HEARTBEAT_NUM_MODES) {
84		spin_lock(&r2hb_live_lock);
85		r2hb_heartbeat_mode = hb_mode;
86		ret = 0;
87		spin_unlock(&r2hb_live_lock);
88	}
89
90	return ret;
91}
92
93void r2hb_exit(void)
94{
95}
96
97int r2hb_init(void)
98{
99	int i;
100
101	for (i = 0; i < ARRAY_SIZE(r2hb_callbacks); i++)
102		INIT_LIST_HEAD(&r2hb_callbacks[i].list);
103
104	memset(r2hb_live_node_bitmap, 0, sizeof(r2hb_live_node_bitmap));
105
106	return 0;
107}
108
109/* if we're already in a callback then we're already serialized by the sem */
110static void r2hb_fill_node_map_from_callback(unsigned long *map,
111					     unsigned bytes)
112{
113	BUG_ON(bytes < (BITS_TO_LONGS(R2NM_MAX_NODES) * sizeof(unsigned long)));
114
115	memcpy(map, &r2hb_live_node_bitmap, bytes);
116}
117
118/*
119 * get a map of all nodes that are heartbeating in any regions
120 */
121void r2hb_fill_node_map(unsigned long *map, unsigned bytes)
122{
123	/* callers want to serialize this map and callbacks so that they
124	 * can trust that they don't miss nodes coming to the party */
125	down_read(&r2hb_callback_sem);
126	spin_lock(&r2hb_live_lock);
127	r2hb_fill_node_map_from_callback(map, bytes);
128	spin_unlock(&r2hb_live_lock);
129	up_read(&r2hb_callback_sem);
130}
131EXPORT_SYMBOL_GPL(r2hb_fill_node_map);
132
133/*
134 * heartbeat configfs bits.  The heartbeat set is a default set under
135 * the cluster set in nodemanager.c.
136 */
137
138/* heartbeat set */
139
140struct r2hb_hb_group {
141	struct config_group hs_group;
142	/* some stuff? */
143};
144
145static struct r2hb_hb_group *to_r2hb_hb_group(struct config_group *group)
146{
147	return group ?
148		container_of(group, struct r2hb_hb_group, hs_group)
149		: NULL;
150}
151
152static struct config_item r2hb_config_item;
153
154static struct config_item *r2hb_hb_group_make_item(struct config_group *group,
155							  const char *name)
156{
157	int ret;
158
159	if (strlen(name) > R2HB_MAX_REGION_NAME_LEN) {
160		ret = -ENAMETOOLONG;
161		goto free;
162	}
163
164	config_item_put(&r2hb_config_item);
165
166	return &r2hb_config_item;
167free:
168	return ERR_PTR(ret);
169}
170
171static void r2hb_hb_group_drop_item(struct config_group *group,
172					   struct config_item *item)
173{
174	if (r2hb_global_heartbeat_active()) {
175		printk(KERN_NOTICE "ramster: Heartbeat %s "
176			"on region %s (%s)\n",
177			"stopped/aborted", config_item_name(item),
178			"no region");
179	}
180
181	config_item_put(item);
182}
183
184struct r2hb_hb_group_attribute {
185	struct configfs_attribute attr;
186	ssize_t (*show)(struct r2hb_hb_group *, char *);
187	ssize_t (*store)(struct r2hb_hb_group *, const char *, size_t);
188};
189
190static ssize_t r2hb_hb_group_show(struct config_item *item,
191					 struct configfs_attribute *attr,
192					 char *page)
193{
194	struct r2hb_hb_group *reg = to_r2hb_hb_group(to_config_group(item));
195	struct r2hb_hb_group_attribute *r2hb_hb_group_attr =
196		container_of(attr, struct r2hb_hb_group_attribute, attr);
197	ssize_t ret = 0;
198
199	if (r2hb_hb_group_attr->show)
200		ret = r2hb_hb_group_attr->show(reg, page);
201	return ret;
202}
203
204static ssize_t r2hb_hb_group_store(struct config_item *item,
205					  struct configfs_attribute *attr,
206					  const char *page, size_t count)
207{
208	struct r2hb_hb_group *reg = to_r2hb_hb_group(to_config_group(item));
209	struct r2hb_hb_group_attribute *r2hb_hb_group_attr =
210		container_of(attr, struct r2hb_hb_group_attribute, attr);
211	ssize_t ret = -EINVAL;
212
213	if (r2hb_hb_group_attr->store)
214		ret = r2hb_hb_group_attr->store(reg, page, count);
215	return ret;
216}
217
218static ssize_t r2hb_hb_group_threshold_show(struct r2hb_hb_group *group,
219						     char *page)
220{
221	return sprintf(page, "%u\n", r2hb_dead_threshold);
222}
223
224static ssize_t r2hb_hb_group_threshold_store(struct r2hb_hb_group *group,
225						    const char *page,
226						    size_t count)
227{
228	unsigned long tmp;
229	char *p = (char *)page;
230	int err;
231
232	err = kstrtoul(p, 10, &tmp);
233	if (err)
234		return err;
235
236	/* this will validate ranges for us. */
237	r2hb_dead_threshold_set((unsigned int) tmp);
238
239	return count;
240}
241
242static
243ssize_t r2hb_hb_group_mode_show(struct r2hb_hb_group *group,
244				       char *page)
245{
246	return sprintf(page, "%s\n",
247		       r2hb_heartbeat_mode_desc[r2hb_heartbeat_mode]);
248}
249
250static
251ssize_t r2hb_hb_group_mode_store(struct r2hb_hb_group *group,
252					const char *page, size_t count)
253{
254	unsigned int i;
255	int ret;
256	size_t len;
257
258	len = (page[count - 1] == '\n') ? count - 1 : count;
259	if (!len)
260		return -EINVAL;
261
262	for (i = 0; i < R2HB_HEARTBEAT_NUM_MODES; ++i) {
263		if (strnicmp(page, r2hb_heartbeat_mode_desc[i], len))
264			continue;
265
266		ret = r2hb_global_hearbeat_mode_set(i);
267		if (!ret)
268			printk(KERN_NOTICE "ramster: Heartbeat mode "
269						"set to %s\n",
270			       r2hb_heartbeat_mode_desc[i]);
271		return count;
272	}
273
274	return -EINVAL;
275
276}
277
278static struct r2hb_hb_group_attribute r2hb_hb_group_attr_threshold = {
279	.attr	= { .ca_owner = THIS_MODULE,
280		    .ca_name = "dead_threshold",
281		    .ca_mode = S_IRUGO | S_IWUSR },
282	.show	= r2hb_hb_group_threshold_show,
283	.store	= r2hb_hb_group_threshold_store,
284};
285
286static struct r2hb_hb_group_attribute r2hb_hb_group_attr_mode = {
287	.attr   = { .ca_owner = THIS_MODULE,
288		.ca_name = "mode",
289		.ca_mode = S_IRUGO | S_IWUSR },
290	.show   = r2hb_hb_group_mode_show,
291	.store  = r2hb_hb_group_mode_store,
292};
293
294static struct configfs_attribute *r2hb_hb_group_attrs[] = {
295	&r2hb_hb_group_attr_threshold.attr,
296	&r2hb_hb_group_attr_mode.attr,
297	NULL,
298};
299
300static struct configfs_item_operations r2hb_hearbeat_group_item_ops = {
301	.show_attribute		= r2hb_hb_group_show,
302	.store_attribute	= r2hb_hb_group_store,
303};
304
305static struct configfs_group_operations r2hb_hb_group_group_ops = {
306	.make_item	= r2hb_hb_group_make_item,
307	.drop_item	= r2hb_hb_group_drop_item,
308};
309
310static struct config_item_type r2hb_hb_group_type = {
311	.ct_group_ops	= &r2hb_hb_group_group_ops,
312	.ct_item_ops	= &r2hb_hearbeat_group_item_ops,
313	.ct_attrs	= r2hb_hb_group_attrs,
314	.ct_owner	= THIS_MODULE,
315};
316
317/* this is just here to avoid touching group in heartbeat.h which the
318 * entire damn world #includes */
319struct config_group *r2hb_alloc_hb_set(void)
320{
321	struct r2hb_hb_group *hs = NULL;
322	struct config_group *ret = NULL;
323
324	hs = kzalloc(sizeof(struct r2hb_hb_group), GFP_KERNEL);
325	if (hs == NULL)
326		goto out;
327
328	config_group_init_type_name(&hs->hs_group, "heartbeat",
329				    &r2hb_hb_group_type);
330
331	ret = &hs->hs_group;
332out:
333	if (ret == NULL)
334		kfree(hs);
335	return ret;
336}
337
338void r2hb_free_hb_set(struct config_group *group)
339{
340	struct r2hb_hb_group *hs = to_r2hb_hb_group(group);
341	kfree(hs);
342}
343
344/* hb callback registration and issuing */
345
346static struct r2hb_callback *hbcall_from_type(enum r2hb_callback_type type)
347{
348	if (type == R2HB_NUM_CB)
349		return ERR_PTR(-EINVAL);
350
351	return &r2hb_callbacks[type];
352}
353
354void r2hb_setup_callback(struct r2hb_callback_func *hc,
355			 enum r2hb_callback_type type,
356			 r2hb_cb_func *func,
357			 void *data,
358			 int priority)
359{
360	INIT_LIST_HEAD(&hc->hc_item);
361	hc->hc_func = func;
362	hc->hc_data = data;
363	hc->hc_priority = priority;
364	hc->hc_type = type;
365	hc->hc_magic = R2HB_CB_MAGIC;
366}
367EXPORT_SYMBOL_GPL(r2hb_setup_callback);
368
369int r2hb_register_callback(const char *region_uuid,
370			   struct r2hb_callback_func *hc)
371{
372	struct r2hb_callback_func *tmp;
373	struct list_head *iter;
374	struct r2hb_callback *hbcall;
375	int ret;
376
377	BUG_ON(hc->hc_magic != R2HB_CB_MAGIC);
378	BUG_ON(!list_empty(&hc->hc_item));
379
380	hbcall = hbcall_from_type(hc->hc_type);
381	if (IS_ERR(hbcall)) {
382		ret = PTR_ERR(hbcall);
383		goto out;
384	}
385
386	down_write(&r2hb_callback_sem);
387
388	list_for_each(iter, &hbcall->list) {
389		tmp = list_entry(iter, struct r2hb_callback_func, hc_item);
390		if (hc->hc_priority < tmp->hc_priority) {
391			list_add_tail(&hc->hc_item, iter);
392			break;
393		}
394	}
395	if (list_empty(&hc->hc_item))
396		list_add_tail(&hc->hc_item, &hbcall->list);
397
398	up_write(&r2hb_callback_sem);
399	ret = 0;
400out:
401	mlog(ML_CLUSTER, "returning %d on behalf of %p for funcs %p\n",
402	     ret, __builtin_return_address(0), hc);
403	return ret;
404}
405EXPORT_SYMBOL_GPL(r2hb_register_callback);
406
407void r2hb_unregister_callback(const char *region_uuid,
408			      struct r2hb_callback_func *hc)
409{
410	BUG_ON(hc->hc_magic != R2HB_CB_MAGIC);
411
412	mlog(ML_CLUSTER, "on behalf of %p for funcs %p\n",
413	     __builtin_return_address(0), hc);
414
415	/* XXX Can this happen _with_ a region reference? */
416	if (list_empty(&hc->hc_item))
417		return;
418
419	down_write(&r2hb_callback_sem);
420
421	list_del_init(&hc->hc_item);
422
423	up_write(&r2hb_callback_sem);
424}
425EXPORT_SYMBOL_GPL(r2hb_unregister_callback);
426
427int r2hb_check_node_heartbeating_from_callback(u8 node_num)
428{
429	unsigned long testing_map[BITS_TO_LONGS(R2NM_MAX_NODES)];
430
431	r2hb_fill_node_map_from_callback(testing_map, sizeof(testing_map));
432	if (!test_bit(node_num, testing_map)) {
433		mlog(ML_HEARTBEAT,
434		     "node (%u) does not have heartbeating enabled.\n",
435		     node_num);
436		return 0;
437	}
438
439	return 1;
440}
441EXPORT_SYMBOL_GPL(r2hb_check_node_heartbeating_from_callback);
442
443void r2hb_stop_all_regions(void)
444{
445}
446EXPORT_SYMBOL_GPL(r2hb_stop_all_regions);
447
448/*
449 * this is just a hack until we get the plumbing which flips file systems
450 * read only and drops the hb ref instead of killing the node dead.
451 */
452int r2hb_global_heartbeat_active(void)
453{
454	return (r2hb_heartbeat_mode == R2HB_HEARTBEAT_GLOBAL);
455}
456EXPORT_SYMBOL(r2hb_global_heartbeat_active);
457
458/* added for RAMster */
459void r2hb_manual_set_node_heartbeating(int node_num)
460{
461	if (node_num < R2NM_MAX_NODES)
462		set_bit(node_num, r2hb_live_node_bitmap);
463}
464EXPORT_SYMBOL(r2hb_manual_set_node_heartbeating);
465