14e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#ifndef _LINUX_LIST_BL_H 24e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define _LINUX_LIST_BL_H 34e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 44e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#include <linux/list.h> 51879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig#include <linux/bit_spinlock.h> 64e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 74e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin/* 84e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * Special version of lists, where head of the list has a lock in the lowest 94e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * bit. This is useful for scalable hash tables without increasing memory 104e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * footprint overhead. 114e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * 124e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * For modification operations, the 0 bit of hlist_bl_head->first 134e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * pointer must be set. 144e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * 154e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * With some small modifications, this can easily be adapted to store several 164e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * arbitrary bits (not just a single lock bit), if the need arises to store 174e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * some fast and compact auxiliary data. 184e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin */ 194e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 2032385c7cf60a78375b63afc4f02001df84dfd1a0Russell King#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) 214e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define LIST_BL_LOCKMASK 1UL 224e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#else 234e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define LIST_BL_LOCKMASK 0UL 244e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#endif 254e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 264e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#ifdef CONFIG_DEBUG_LIST 274e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define LIST_BL_BUG_ON(x) BUG_ON(x) 284e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#else 294e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define LIST_BL_BUG_ON(x) 304e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#endif 314e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 324e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 334e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstruct hlist_bl_head { 344e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin struct hlist_bl_node *first; 354e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin}; 364e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 374e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstruct hlist_bl_node { 384e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin struct hlist_bl_node *next, **pprev; 394e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin}; 404e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define INIT_HLIST_BL_HEAD(ptr) \ 414e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin ((ptr)->first = NULL) 424e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 434e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) 444e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 454e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin h->next = NULL; 464e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin h->pprev = NULL; 474e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 484e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 494e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member) 504e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 514e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline int hlist_bl_unhashed(const struct hlist_bl_node *h) 524e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 534e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin return !h->pprev; 544e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 554e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 564e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline struct hlist_bl_node *hlist_bl_first(struct hlist_bl_head *h) 574e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 584e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin return (struct hlist_bl_node *) 594e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin ((unsigned long)h->first & ~LIST_BL_LOCKMASK); 604e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 614e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 624e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline void hlist_bl_set_first(struct hlist_bl_head *h, 634e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin struct hlist_bl_node *n) 644e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 654e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK); 662c6755988afc003a0332406a134fb6a1626f9b28Nick Piggin LIST_BL_BUG_ON(((unsigned long)h->first & LIST_BL_LOCKMASK) != 672c6755988afc003a0332406a134fb6a1626f9b28Nick Piggin LIST_BL_LOCKMASK); 684e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin h->first = (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK); 694e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 704e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 714e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline int hlist_bl_empty(const struct hlist_bl_head *h) 724e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 734e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin return !((unsigned long)h->first & ~LIST_BL_LOCKMASK); 744e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 754e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 764e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline void hlist_bl_add_head(struct hlist_bl_node *n, 774e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin struct hlist_bl_head *h) 784e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 794e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin struct hlist_bl_node *first = hlist_bl_first(h); 804e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 814e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin n->next = first; 824e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin if (first) 834e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin first->pprev = &n->next; 844e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin n->pprev = &h->first; 854e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin hlist_bl_set_first(h, n); 864e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 874e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 884e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline void __hlist_bl_del(struct hlist_bl_node *n) 894e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 904e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin struct hlist_bl_node *next = n->next; 914e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin struct hlist_bl_node **pprev = n->pprev; 924e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 934e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK); 944e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 954e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin /* pprev may be `first`, so be careful not to lose the lock bit */ 964e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin *pprev = (struct hlist_bl_node *) 974e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin ((unsigned long)next | 984e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin ((unsigned long)*pprev & LIST_BL_LOCKMASK)); 994e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin if (next) 1004e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin next->pprev = pprev; 1014e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 1024e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 1034e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline void hlist_bl_del(struct hlist_bl_node *n) 1044e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 1054e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin __hlist_bl_del(n); 1064e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin n->next = LIST_POISON1; 1074e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin n->pprev = LIST_POISON2; 1084e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 1094e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 1104e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Pigginstatic inline void hlist_bl_del_init(struct hlist_bl_node *n) 1114e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin{ 1124e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin if (!hlist_bl_unhashed(n)) { 1134e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin __hlist_bl_del(n); 1144e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin INIT_HLIST_BL_NODE(n); 1154e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin } 1164e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin} 1174e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 1181879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwigstatic inline void hlist_bl_lock(struct hlist_bl_head *b) 1191879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig{ 1201879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig bit_spin_lock(0, (unsigned long *)b); 1211879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig} 1221879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig 1231879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwigstatic inline void hlist_bl_unlock(struct hlist_bl_head *b) 1241879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig{ 1251879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig __bit_spin_unlock(0, (unsigned long *)b); 1261879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig} 1271879fd6a26571fd4e8e1f4bb3e7537bc936b1fe7Christoph Hellwig 12849d0de082c31de34cc896c14eec5f1c2ade0415aSteven Whitehousestatic inline bool hlist_bl_is_locked(struct hlist_bl_head *b) 12949d0de082c31de34cc896c14eec5f1c2ade0415aSteven Whitehouse{ 13049d0de082c31de34cc896c14eec5f1c2ade0415aSteven Whitehouse return bit_spin_is_locked(0, (unsigned long *)b); 13149d0de082c31de34cc896c14eec5f1c2ade0415aSteven Whitehouse} 13249d0de082c31de34cc896c14eec5f1c2ade0415aSteven Whitehouse 1334e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin/** 1344e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * hlist_bl_for_each_entry - iterate over list of given type 1354e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @tpos: the type * to use as a loop cursor. 1364e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @pos: the &struct hlist_node to use as a loop cursor. 1374e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @head: the head for your list. 1384e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @member: the name of the hlist_node within the struct. 1394e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * 1404e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin */ 1414e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define hlist_bl_for_each_entry(tpos, pos, head, member) \ 1424e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin for (pos = hlist_bl_first(head); \ 1434e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin pos && \ 1444e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \ 1454e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin pos = pos->next) 1464e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 1474e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin/** 1484e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * hlist_bl_for_each_entry_safe - iterate over list of given type safe against removal of list entry 1494e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @tpos: the type * to use as a loop cursor. 1504e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @pos: the &struct hlist_node to use as a loop cursor. 1514e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @n: another &struct hlist_node to use as temporary storage 1524e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @head: the head for your list. 1534e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin * @member: the name of the hlist_node within the struct. 1544e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin */ 1554e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#define hlist_bl_for_each_entry_safe(tpos, pos, n, head, member) \ 1564e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin for (pos = hlist_bl_first(head); \ 1574e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin pos && ({ n = pos->next; 1; }) && \ 1584e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1;}); \ 1594e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin pos = n) 1604e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin 1614e35e6070b1ceed89c3bba2af4216c286fb1dafdNick Piggin#endif 162