1/* -*- mode: c; c-basic-offset: 8; -*- 2 * vim: noexpandtab sw=8 ts=8 sts=0: 3 * 4 * Copyright (C) 2004, 2005, 2012 Oracle. All rights reserved. 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public 17 * License along with this program; if not, write to the 18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 19 * Boston, MA 021110-1307, USA. 20 */ 21 22#include <linux/kernel.h> 23#include <linux/module.h> 24#include <linux/configfs.h> 25 26#include "heartbeat.h" 27#include "tcp.h" 28#include "nodemanager.h" 29 30#include "masklog.h" 31 32/* 33 * The first heartbeat pass had one global thread that would serialize all hb 34 * callback calls. This global serializing sem should only be removed once 35 * we've made sure that all callees can deal with being called concurrently 36 * from multiple hb region threads. 37 */ 38static DECLARE_RWSEM(r2hb_callback_sem); 39 40/* 41 * multiple hb threads are watching multiple regions. A node is live 42 * whenever any of the threads sees activity from the node in its region. 43 */ 44static DEFINE_SPINLOCK(r2hb_live_lock); 45static unsigned long r2hb_live_node_bitmap[BITS_TO_LONGS(R2NM_MAX_NODES)]; 46 47static struct r2hb_callback { 48 struct list_head list; 49} r2hb_callbacks[R2HB_NUM_CB]; 50 51enum r2hb_heartbeat_modes { 52 R2HB_HEARTBEAT_LOCAL = 0, 53 R2HB_HEARTBEAT_GLOBAL, 54 R2HB_HEARTBEAT_NUM_MODES, 55}; 56 57char *r2hb_heartbeat_mode_desc[R2HB_HEARTBEAT_NUM_MODES] = { 58 "local", /* R2HB_HEARTBEAT_LOCAL */ 59 "global", /* R2HB_HEARTBEAT_GLOBAL */ 60}; 61 62unsigned int r2hb_dead_threshold = R2HB_DEFAULT_DEAD_THRESHOLD; 63unsigned int r2hb_heartbeat_mode = R2HB_HEARTBEAT_LOCAL; 64 65/* Only sets a new threshold if there are no active regions. 66 * 67 * No locking or otherwise interesting code is required for reading 68 * r2hb_dead_threshold as it can't change once regions are active and 69 * it's not interesting to anyone until then anyway. */ 70static void r2hb_dead_threshold_set(unsigned int threshold) 71{ 72 if (threshold > R2HB_MIN_DEAD_THRESHOLD) { 73 spin_lock(&r2hb_live_lock); 74 r2hb_dead_threshold = threshold; 75 spin_unlock(&r2hb_live_lock); 76 } 77} 78 79static int r2hb_global_hearbeat_mode_set(unsigned int hb_mode) 80{ 81 int ret = -1; 82 83 if (hb_mode < R2HB_HEARTBEAT_NUM_MODES) { 84 spin_lock(&r2hb_live_lock); 85 r2hb_heartbeat_mode = hb_mode; 86 ret = 0; 87 spin_unlock(&r2hb_live_lock); 88 } 89 90 return ret; 91} 92 93void r2hb_exit(void) 94{ 95} 96 97int r2hb_init(void) 98{ 99 int i; 100 101 for (i = 0; i < ARRAY_SIZE(r2hb_callbacks); i++) 102 INIT_LIST_HEAD(&r2hb_callbacks[i].list); 103 104 memset(r2hb_live_node_bitmap, 0, sizeof(r2hb_live_node_bitmap)); 105 106 return 0; 107} 108 109/* if we're already in a callback then we're already serialized by the sem */ 110static void r2hb_fill_node_map_from_callback(unsigned long *map, 111 unsigned bytes) 112{ 113 BUG_ON(bytes < (BITS_TO_LONGS(R2NM_MAX_NODES) * sizeof(unsigned long))); 114 115 memcpy(map, &r2hb_live_node_bitmap, bytes); 116} 117 118/* 119 * get a map of all nodes that are heartbeating in any regions 120 */ 121void r2hb_fill_node_map(unsigned long *map, unsigned bytes) 122{ 123 /* callers want to serialize this map and callbacks so that they 124 * can trust that they don't miss nodes coming to the party */ 125 down_read(&r2hb_callback_sem); 126 spin_lock(&r2hb_live_lock); 127 r2hb_fill_node_map_from_callback(map, bytes); 128 spin_unlock(&r2hb_live_lock); 129 up_read(&r2hb_callback_sem); 130} 131EXPORT_SYMBOL_GPL(r2hb_fill_node_map); 132 133/* 134 * heartbeat configfs bits. The heartbeat set is a default set under 135 * the cluster set in nodemanager.c. 136 */ 137 138/* heartbeat set */ 139 140struct r2hb_hb_group { 141 struct config_group hs_group; 142 /* some stuff? */ 143}; 144 145static struct r2hb_hb_group *to_r2hb_hb_group(struct config_group *group) 146{ 147 return group ? 148 container_of(group, struct r2hb_hb_group, hs_group) 149 : NULL; 150} 151 152static struct config_item r2hb_config_item; 153 154static struct config_item *r2hb_hb_group_make_item(struct config_group *group, 155 const char *name) 156{ 157 int ret; 158 159 if (strlen(name) > R2HB_MAX_REGION_NAME_LEN) { 160 ret = -ENAMETOOLONG; 161 goto free; 162 } 163 164 config_item_put(&r2hb_config_item); 165 166 return &r2hb_config_item; 167free: 168 return ERR_PTR(ret); 169} 170 171static void r2hb_hb_group_drop_item(struct config_group *group, 172 struct config_item *item) 173{ 174 if (r2hb_global_heartbeat_active()) { 175 printk(KERN_NOTICE "ramster: Heartbeat %s " 176 "on region %s (%s)\n", 177 "stopped/aborted", config_item_name(item), 178 "no region"); 179 } 180 181 config_item_put(item); 182} 183 184struct r2hb_hb_group_attribute { 185 struct configfs_attribute attr; 186 ssize_t (*show)(struct r2hb_hb_group *, char *); 187 ssize_t (*store)(struct r2hb_hb_group *, const char *, size_t); 188}; 189 190static ssize_t r2hb_hb_group_show(struct config_item *item, 191 struct configfs_attribute *attr, 192 char *page) 193{ 194 struct r2hb_hb_group *reg = to_r2hb_hb_group(to_config_group(item)); 195 struct r2hb_hb_group_attribute *r2hb_hb_group_attr = 196 container_of(attr, struct r2hb_hb_group_attribute, attr); 197 ssize_t ret = 0; 198 199 if (r2hb_hb_group_attr->show) 200 ret = r2hb_hb_group_attr->show(reg, page); 201 return ret; 202} 203 204static ssize_t r2hb_hb_group_store(struct config_item *item, 205 struct configfs_attribute *attr, 206 const char *page, size_t count) 207{ 208 struct r2hb_hb_group *reg = to_r2hb_hb_group(to_config_group(item)); 209 struct r2hb_hb_group_attribute *r2hb_hb_group_attr = 210 container_of(attr, struct r2hb_hb_group_attribute, attr); 211 ssize_t ret = -EINVAL; 212 213 if (r2hb_hb_group_attr->store) 214 ret = r2hb_hb_group_attr->store(reg, page, count); 215 return ret; 216} 217 218static ssize_t r2hb_hb_group_threshold_show(struct r2hb_hb_group *group, 219 char *page) 220{ 221 return sprintf(page, "%u\n", r2hb_dead_threshold); 222} 223 224static ssize_t r2hb_hb_group_threshold_store(struct r2hb_hb_group *group, 225 const char *page, 226 size_t count) 227{ 228 unsigned long tmp; 229 char *p = (char *)page; 230 int err; 231 232 err = kstrtoul(p, 10, &tmp); 233 if (err) 234 return err; 235 236 /* this will validate ranges for us. */ 237 r2hb_dead_threshold_set((unsigned int) tmp); 238 239 return count; 240} 241 242static 243ssize_t r2hb_hb_group_mode_show(struct r2hb_hb_group *group, 244 char *page) 245{ 246 return sprintf(page, "%s\n", 247 r2hb_heartbeat_mode_desc[r2hb_heartbeat_mode]); 248} 249 250static 251ssize_t r2hb_hb_group_mode_store(struct r2hb_hb_group *group, 252 const char *page, size_t count) 253{ 254 unsigned int i; 255 int ret; 256 size_t len; 257 258 len = (page[count - 1] == '\n') ? count - 1 : count; 259 if (!len) 260 return -EINVAL; 261 262 for (i = 0; i < R2HB_HEARTBEAT_NUM_MODES; ++i) { 263 if (strnicmp(page, r2hb_heartbeat_mode_desc[i], len)) 264 continue; 265 266 ret = r2hb_global_hearbeat_mode_set(i); 267 if (!ret) 268 printk(KERN_NOTICE "ramster: Heartbeat mode " 269 "set to %s\n", 270 r2hb_heartbeat_mode_desc[i]); 271 return count; 272 } 273 274 return -EINVAL; 275 276} 277 278static struct r2hb_hb_group_attribute r2hb_hb_group_attr_threshold = { 279 .attr = { .ca_owner = THIS_MODULE, 280 .ca_name = "dead_threshold", 281 .ca_mode = S_IRUGO | S_IWUSR }, 282 .show = r2hb_hb_group_threshold_show, 283 .store = r2hb_hb_group_threshold_store, 284}; 285 286static struct r2hb_hb_group_attribute r2hb_hb_group_attr_mode = { 287 .attr = { .ca_owner = THIS_MODULE, 288 .ca_name = "mode", 289 .ca_mode = S_IRUGO | S_IWUSR }, 290 .show = r2hb_hb_group_mode_show, 291 .store = r2hb_hb_group_mode_store, 292}; 293 294static struct configfs_attribute *r2hb_hb_group_attrs[] = { 295 &r2hb_hb_group_attr_threshold.attr, 296 &r2hb_hb_group_attr_mode.attr, 297 NULL, 298}; 299 300static struct configfs_item_operations r2hb_hearbeat_group_item_ops = { 301 .show_attribute = r2hb_hb_group_show, 302 .store_attribute = r2hb_hb_group_store, 303}; 304 305static struct configfs_group_operations r2hb_hb_group_group_ops = { 306 .make_item = r2hb_hb_group_make_item, 307 .drop_item = r2hb_hb_group_drop_item, 308}; 309 310static struct config_item_type r2hb_hb_group_type = { 311 .ct_group_ops = &r2hb_hb_group_group_ops, 312 .ct_item_ops = &r2hb_hearbeat_group_item_ops, 313 .ct_attrs = r2hb_hb_group_attrs, 314 .ct_owner = THIS_MODULE, 315}; 316 317/* this is just here to avoid touching group in heartbeat.h which the 318 * entire damn world #includes */ 319struct config_group *r2hb_alloc_hb_set(void) 320{ 321 struct r2hb_hb_group *hs = NULL; 322 struct config_group *ret = NULL; 323 324 hs = kzalloc(sizeof(struct r2hb_hb_group), GFP_KERNEL); 325 if (hs == NULL) 326 goto out; 327 328 config_group_init_type_name(&hs->hs_group, "heartbeat", 329 &r2hb_hb_group_type); 330 331 ret = &hs->hs_group; 332out: 333 if (ret == NULL) 334 kfree(hs); 335 return ret; 336} 337 338void r2hb_free_hb_set(struct config_group *group) 339{ 340 struct r2hb_hb_group *hs = to_r2hb_hb_group(group); 341 kfree(hs); 342} 343 344/* hb callback registration and issuing */ 345 346static struct r2hb_callback *hbcall_from_type(enum r2hb_callback_type type) 347{ 348 if (type == R2HB_NUM_CB) 349 return ERR_PTR(-EINVAL); 350 351 return &r2hb_callbacks[type]; 352} 353 354void r2hb_setup_callback(struct r2hb_callback_func *hc, 355 enum r2hb_callback_type type, 356 r2hb_cb_func *func, 357 void *data, 358 int priority) 359{ 360 INIT_LIST_HEAD(&hc->hc_item); 361 hc->hc_func = func; 362 hc->hc_data = data; 363 hc->hc_priority = priority; 364 hc->hc_type = type; 365 hc->hc_magic = R2HB_CB_MAGIC; 366} 367EXPORT_SYMBOL_GPL(r2hb_setup_callback); 368 369int r2hb_register_callback(const char *region_uuid, 370 struct r2hb_callback_func *hc) 371{ 372 struct r2hb_callback_func *tmp; 373 struct list_head *iter; 374 struct r2hb_callback *hbcall; 375 int ret; 376 377 BUG_ON(hc->hc_magic != R2HB_CB_MAGIC); 378 BUG_ON(!list_empty(&hc->hc_item)); 379 380 hbcall = hbcall_from_type(hc->hc_type); 381 if (IS_ERR(hbcall)) { 382 ret = PTR_ERR(hbcall); 383 goto out; 384 } 385 386 down_write(&r2hb_callback_sem); 387 388 list_for_each(iter, &hbcall->list) { 389 tmp = list_entry(iter, struct r2hb_callback_func, hc_item); 390 if (hc->hc_priority < tmp->hc_priority) { 391 list_add_tail(&hc->hc_item, iter); 392 break; 393 } 394 } 395 if (list_empty(&hc->hc_item)) 396 list_add_tail(&hc->hc_item, &hbcall->list); 397 398 up_write(&r2hb_callback_sem); 399 ret = 0; 400out: 401 mlog(ML_CLUSTER, "returning %d on behalf of %p for funcs %p\n", 402 ret, __builtin_return_address(0), hc); 403 return ret; 404} 405EXPORT_SYMBOL_GPL(r2hb_register_callback); 406 407void r2hb_unregister_callback(const char *region_uuid, 408 struct r2hb_callback_func *hc) 409{ 410 BUG_ON(hc->hc_magic != R2HB_CB_MAGIC); 411 412 mlog(ML_CLUSTER, "on behalf of %p for funcs %p\n", 413 __builtin_return_address(0), hc); 414 415 /* XXX Can this happen _with_ a region reference? */ 416 if (list_empty(&hc->hc_item)) 417 return; 418 419 down_write(&r2hb_callback_sem); 420 421 list_del_init(&hc->hc_item); 422 423 up_write(&r2hb_callback_sem); 424} 425EXPORT_SYMBOL_GPL(r2hb_unregister_callback); 426 427int r2hb_check_node_heartbeating_from_callback(u8 node_num) 428{ 429 unsigned long testing_map[BITS_TO_LONGS(R2NM_MAX_NODES)]; 430 431 r2hb_fill_node_map_from_callback(testing_map, sizeof(testing_map)); 432 if (!test_bit(node_num, testing_map)) { 433 mlog(ML_HEARTBEAT, 434 "node (%u) does not have heartbeating enabled.\n", 435 node_num); 436 return 0; 437 } 438 439 return 1; 440} 441EXPORT_SYMBOL_GPL(r2hb_check_node_heartbeating_from_callback); 442 443void r2hb_stop_all_regions(void) 444{ 445} 446EXPORT_SYMBOL_GPL(r2hb_stop_all_regions); 447 448/* 449 * this is just a hack until we get the plumbing which flips file systems 450 * read only and drops the hb ref instead of killing the node dead. 451 */ 452int r2hb_global_heartbeat_active(void) 453{ 454 return (r2hb_heartbeat_mode == R2HB_HEARTBEAT_GLOBAL); 455} 456EXPORT_SYMBOL(r2hb_global_heartbeat_active); 457 458/* added for RAMster */ 459void r2hb_manual_set_node_heartbeating(int node_num) 460{ 461 if (node_num < R2NM_MAX_NODES) 462 set_bit(node_num, r2hb_live_node_bitmap); 463} 464EXPORT_SYMBOL(r2hb_manual_set_node_heartbeating); 465