1bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron/* 2bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron * jump label support 3bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron * 4bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 5d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com> 6bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron * 7bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron */ 8bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#include <linux/memory.h> 9bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#include <linux/uaccess.h> 10bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#include <linux/module.h> 11bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#include <linux/list.h> 12bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#include <linux/slab.h> 13bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#include <linux/sort.h> 14bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#include <linux/err.h> 15c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar#include <linux/static_key.h> 16851cf6e7d6366195d4ee033cdc7787df1a649a14Andrew Jones#include <linux/jump_label_ratelimit.h> 17bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 18bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#ifdef HAVE_JUMP_LABEL 19bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 20bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron/* mutex to protect coming/going of the the jump_label table */ 21bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baronstatic DEFINE_MUTEX(jump_label_mutex); 22bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 2391bad2f8d3057482b9afb599f14421b007136960Jason Baronvoid jump_label_lock(void) 2491bad2f8d3057482b9afb599f14421b007136960Jason Baron{ 2591bad2f8d3057482b9afb599f14421b007136960Jason Baron mutex_lock(&jump_label_mutex); 2691bad2f8d3057482b9afb599f14421b007136960Jason Baron} 2791bad2f8d3057482b9afb599f14421b007136960Jason Baron 2891bad2f8d3057482b9afb599f14421b007136960Jason Baronvoid jump_label_unlock(void) 2991bad2f8d3057482b9afb599f14421b007136960Jason Baron{ 3091bad2f8d3057482b9afb599f14421b007136960Jason Baron mutex_unlock(&jump_label_mutex); 3191bad2f8d3057482b9afb599f14421b007136960Jason Baron} 3291bad2f8d3057482b9afb599f14421b007136960Jason Baron 33bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baronstatic int jump_label_cmp(const void *a, const void *b) 34bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 35bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron const struct jump_entry *jea = a; 36bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron const struct jump_entry *jeb = b; 37bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 38bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron if (jea->key < jeb->key) 39bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron return -1; 40bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 41bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron if (jea->key > jeb->key) 42bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron return 1; 43bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 44bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron return 0; 45bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 46bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 47bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baronstatic void 48d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronjump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) 49bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 50bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron unsigned long size; 51bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 52bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron size = (((unsigned long)stop - (unsigned long)start) 53bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron / sizeof(struct jump_entry)); 54bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); 55bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 56bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 57c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarstatic void jump_label_update(struct static_key *key, int enable); 58bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 59c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarvoid static_key_slow_inc(struct static_key *key) 60bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 61c4b2c0c5f647aa1093e8f9097a30c17ce0f94d4dHannes Frederic Sowa STATIC_KEY_CHECK_USE(); 62d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (atomic_inc_not_zero(&key->enabled)) 63d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return; 64bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 65d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jump_label_lock(); 66c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar if (atomic_read(&key->enabled) == 0) { 67c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar if (!jump_label_get_branch_default(key)) 68c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar jump_label_update(key, JUMP_LABEL_ENABLE); 69c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar else 70c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar jump_label_update(key, JUMP_LABEL_DISABLE); 71c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar } 72bbbf7af4bf8fc69bc751818cf30521080fa47dcbGleb Natapov atomic_inc(&key->enabled); 73d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jump_label_unlock(); 74bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 75c5905afb0ee6550b42c49213da1c22d67316c194Ingo MolnarEXPORT_SYMBOL_GPL(static_key_slow_inc); 76bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 77c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarstatic void __static_key_slow_dec(struct static_key *key, 78b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov unsigned long rate_limit, struct delayed_work *work) 79bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 80fadf0464b83f91ba021a358c0238a0810c0d2a0bJason Baron if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) { 81fadf0464b83f91ba021a358c0238a0810c0d2a0bJason Baron WARN(atomic_read(&key->enabled) < 0, 82fadf0464b83f91ba021a358c0238a0810c0d2a0bJason Baron "jump label: negative count!\n"); 83d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return; 84fadf0464b83f91ba021a358c0238a0810c0d2a0bJason Baron } 85bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 86b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov if (rate_limit) { 87b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov atomic_inc(&key->enabled); 88b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov schedule_delayed_work(work, rate_limit); 89c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar } else { 90c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar if (!jump_label_get_branch_default(key)) 91c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar jump_label_update(key, JUMP_LABEL_DISABLE); 92c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar else 93c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar jump_label_update(key, JUMP_LABEL_ENABLE); 94c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar } 9591bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_unlock(); 96bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 97bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 98b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapovstatic void jump_label_update_timeout(struct work_struct *work) 99b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov{ 100c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key_deferred *key = 101c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar container_of(work, struct static_key_deferred, work.work); 102c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar __static_key_slow_dec(&key->key, 0, NULL); 103b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov} 104b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov 105c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarvoid static_key_slow_dec(struct static_key *key) 106b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov{ 107c4b2c0c5f647aa1093e8f9097a30c17ce0f94d4dHannes Frederic Sowa STATIC_KEY_CHECK_USE(); 108c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar __static_key_slow_dec(key, 0, NULL); 109b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov} 110c5905afb0ee6550b42c49213da1c22d67316c194Ingo MolnarEXPORT_SYMBOL_GPL(static_key_slow_dec); 111b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov 112c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarvoid static_key_slow_dec_deferred(struct static_key_deferred *key) 113b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov{ 114c4b2c0c5f647aa1093e8f9097a30c17ce0f94d4dHannes Frederic Sowa STATIC_KEY_CHECK_USE(); 115c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar __static_key_slow_dec(&key->key, key->timeout, &key->work); 116b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov} 117c5905afb0ee6550b42c49213da1c22d67316c194Ingo MolnarEXPORT_SYMBOL_GPL(static_key_slow_dec_deferred); 118b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov 119c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarvoid jump_label_rate_limit(struct static_key_deferred *key, 120b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov unsigned long rl) 121b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov{ 122c4b2c0c5f647aa1093e8f9097a30c17ce0f94d4dHannes Frederic Sowa STATIC_KEY_CHECK_USE(); 123b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov key->timeout = rl; 124b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov INIT_DELAYED_WORK(&key->work, jump_label_update_timeout); 125b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov} 126a181dc14ed23f7a499542ff4c78532b5f24bb18fGleb NatapovEXPORT_SYMBOL_GPL(jump_label_rate_limit); 127b202952075f62603bea9bfb6ebc6b0420db11949Gleb Natapov 1284c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baronstatic int addr_conflict(struct jump_entry *entry, void *start, void *end) 1294c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron{ 1304c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron if (entry->code <= (unsigned long)end && 1314c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) 1324c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron return 1; 1334c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron 1344c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron return 0; 1354c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron} 1364c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron 137d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronstatic int __jump_label_text_reserved(struct jump_entry *iter_start, 138d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_stop, void *start, void *end) 1394c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron{ 1404c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron struct jump_entry *iter; 1414c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron 1424c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron iter = iter_start; 1434c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron while (iter < iter_stop) { 144d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (addr_conflict(iter, start, end)) 145d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return 1; 1464c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron iter++; 1474c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron } 1484c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron 149d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return 0; 150d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron} 151d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 15220284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge/* 15320284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge * Update code which is definitely not currently executing. 15420284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge * Architectures which need heavyweight synchronization to modify 15520284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge * running code can override this to make the non-live update case 15620284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge * cheaper. 15720284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge */ 1589cdbe1cbac4ec318037297175587a0080acc9d11Peter Zijlstravoid __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry, 15920284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge enum jump_label_type type) 16020284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge{ 16120284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge arch_jump_label_transform(entry, type); 16220284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge} 16320284aa77c0f6227da4783a920b72dc61d4bcc09Jeremy Fitzhardinge 164c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarstatic void __jump_label_update(struct static_key *key, 1657cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa struct jump_entry *entry, 1667cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa struct jump_entry *stop, int enable) 167d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron{ 1687cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa for (; (entry < stop) && 1697cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa (entry->key == (jump_label_t)(unsigned long)key); 1707cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa entry++) { 171d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron /* 172d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * entry->code set to 0 invalidates module init text sections 173d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * kernel_text_address() verifies we are not in core kernel 174d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * init code, see jump_label_invalidate_module_init(). 175d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron */ 176d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (entry->code && kernel_text_address(entry->code)) 177d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron arch_jump_label_transform(entry, enable); 178d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron } 1794c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron} 1804c3ef6d79328c0e23ade60cbfc8d496123a6855cJason Baron 181c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarstatic enum jump_label_type jump_label_type(struct static_key *key) 182c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar{ 183c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar bool true_branch = jump_label_get_branch_default(key); 184c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar bool state = static_key_enabled(key); 185c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar 186c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar if ((!true_branch && state) || (true_branch && !state)) 187c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar return JUMP_LABEL_ENABLE; 188c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar 189c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar return JUMP_LABEL_DISABLE; 190c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar} 191c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar 19297ce2c88f9ad42e3c60a9beb9fca87abf3639faaJeremy Fitzhardingevoid __init jump_label_init(void) 193bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 194bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron struct jump_entry *iter_start = __start___jump_table; 195bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron struct jump_entry *iter_stop = __stop___jump_table; 196c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key *key = NULL; 197bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron struct jump_entry *iter; 198bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 19991bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_lock(); 200d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jump_label_sort_entries(iter_start, iter_stop); 201d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 202d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron for (iter = iter_start; iter < iter_stop; iter++) { 203c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key *iterk; 20437348804e0289087d21ae8bff4c0732030a3c6acJeremy Fitzhardinge 205c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar iterk = (struct static_key *)(unsigned long)iter->key; 206c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar arch_jump_label_transform_static(iter, jump_label_type(iterk)); 20737348804e0289087d21ae8bff4c0732030a3c6acJeremy Fitzhardinge if (iterk == key) 208d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron continue; 209d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 21037348804e0289087d21ae8bff4c0732030a3c6acJeremy Fitzhardinge key = iterk; 211c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar /* 212c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. 213c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar */ 214c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar *((unsigned long *)&key->entries) += (unsigned long)iter; 215d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron#ifdef CONFIG_MODULES 216d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron key->next = NULL; 217d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron#endif 218bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron } 219c4b2c0c5f647aa1093e8f9097a30c17ce0f94d4dHannes Frederic Sowa static_key_initialized = true; 22091bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_unlock(); 221bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 222bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 223bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#ifdef CONFIG_MODULES 224bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 225c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarstruct static_key_mod { 226c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key_mod *next; 227d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *entries; 228d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct module *mod; 229d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron}; 230d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 231d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronstatic int __jump_label_mod_text_reserved(void *start, void *end) 232d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron{ 233d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct module *mod; 234d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 235d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron mod = __module_text_address((unsigned long)start); 236d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (!mod) 237d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return 0; 238d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 239d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); 240d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 241d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return __jump_label_text_reserved(mod->jump_entries, 242d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron mod->jump_entries + mod->num_jump_entries, 243d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron start, end); 244d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron} 245d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 246c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarstatic void __jump_label_mod_update(struct static_key *key, int enable) 247d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron{ 248c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key_mod *mod = key->next; 249d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 250d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron while (mod) { 2517cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa struct module *m = mod->mod; 2527cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa 2537cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa __jump_label_update(key, mod->entries, 2547cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa m->jump_entries + m->num_jump_entries, 2557cbc5b8d4a775a43875a09e29c49a2a8195b5b2dJiri Olsa enable); 256d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron mod = mod->next; 257d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron } 258d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron} 259d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 260d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron/*** 261d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() 262d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * @mod: module to patch 263d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * 264d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * Allow for run-time selection of the optimal nops. Before the module 265d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * loads patch these with arch_get_jump_label_nop(), which is specified by 266d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * the arch specific jump label code. 267d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron */ 268d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronvoid jump_label_apply_nops(struct module *mod) 269bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 270d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_start = mod->jump_entries; 271d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 272d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter; 273d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 274d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron /* if the module doesn't have jump label entries, just return */ 275d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (iter_start == iter_stop) 276d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return; 277d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 278ac99b862fb98a36929831791da31714f709c2aa8Peter Zijlstra for (iter = iter_start; iter < iter_stop; iter++) { 279c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); 280ac99b862fb98a36929831791da31714f709c2aa8Peter Zijlstra } 281bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 282bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 283d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronstatic int jump_label_add_module(struct module *mod) 284bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 285d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_start = mod->jump_entries; 286d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 287d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter; 288c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key *key = NULL; 289c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key_mod *jlm; 290bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 291bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron /* if the module doesn't have jump label entries, just return */ 292d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (iter_start == iter_stop) 293bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron return 0; 294bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 295d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jump_label_sort_entries(iter_start, iter_stop); 296d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 297d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron for (iter = iter_start; iter < iter_stop; iter++) { 298c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key *iterk; 299d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 300c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar iterk = (struct static_key *)(unsigned long)iter->key; 301c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar if (iterk == key) 302c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar continue; 303d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 304c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar key = iterk; 305d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (__module_address(iter->key) == mod) { 306c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar /* 307c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. 308c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar */ 309c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar *((unsigned long *)&key->entries) += (unsigned long)iter; 310d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron key->next = NULL; 311d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron continue; 312bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron } 313c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL); 314d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (!jlm) 315d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return -ENOMEM; 316d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jlm->mod = mod; 317d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jlm->entries = iter; 318d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jlm->next = key->next; 319d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron key->next = jlm; 320d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 321c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar if (jump_label_type(key) == JUMP_LABEL_ENABLE) 322ac99b862fb98a36929831791da31714f709c2aa8Peter Zijlstra __jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE); 323bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron } 324d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 325bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron return 0; 326bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 327bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 328d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronstatic void jump_label_del_module(struct module *mod) 329bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 330d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_start = mod->jump_entries; 331d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 332d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter; 333c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key *key = NULL; 334c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct static_key_mod *jlm, **prev; 335bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 336d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron for (iter = iter_start; iter < iter_stop; iter++) { 337d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (iter->key == (jump_label_t)(unsigned long)key) 338d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron continue; 339d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 340c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar key = (struct static_key *)(unsigned long)iter->key; 341d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 342d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (__module_address(iter->key) == mod) 343d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron continue; 344d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 345d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron prev = &key->next; 346d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jlm = key->next; 347bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 348d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron while (jlm && jlm->mod != mod) { 349d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron prev = &jlm->next; 350d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jlm = jlm->next; 351d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron } 352d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 353d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (jlm) { 354d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron *prev = jlm->next; 355d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron kfree(jlm); 356bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron } 357bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron } 358bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 359bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 360d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronstatic void jump_label_invalidate_module_init(struct module *mod) 361b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron{ 362d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_start = mod->jump_entries; 363d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; 364b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron struct jump_entry *iter; 365b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron 366d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron for (iter = iter_start; iter < iter_stop; iter++) { 367d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (within_module_init(iter->code, mod)) 368d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron iter->code = 0; 369b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron } 370b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron} 371b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron 372bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baronstatic int 373bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baronjump_label_module_notify(struct notifier_block *self, unsigned long val, 374bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron void *data) 375bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 376bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron struct module *mod = data; 377bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron int ret = 0; 378bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 379bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron switch (val) { 380bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron case MODULE_STATE_COMING: 38191bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_lock(); 382d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron ret = jump_label_add_module(mod); 383bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron if (ret) 384d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jump_label_del_module(mod); 38591bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_unlock(); 386bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron break; 387bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron case MODULE_STATE_GOING: 38891bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_lock(); 389d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jump_label_del_module(mod); 39091bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_unlock(); 391bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron break; 392b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron case MODULE_STATE_LIVE: 39391bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_lock(); 394d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron jump_label_invalidate_module_init(mod); 39591bad2f8d3057482b9afb599f14421b007136960Jason Baron jump_label_unlock(); 396b842f8faf6c7dc2005c6a70631c1a91bac02f180Jason Baron break; 397bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron } 398bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 399d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return notifier_from_errno(ret); 400bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 401bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 402bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baronstruct notifier_block jump_label_module_nb = { 403bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron .notifier_call = jump_label_module_notify, 404d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron .priority = 1, /* higher than tracepoints */ 405bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron}; 406bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 407d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronstatic __init int jump_label_init_module(void) 408bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron{ 409bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron return register_module_notifier(&jump_label_module_nb); 410bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron} 411d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronearly_initcall(jump_label_init_module); 412bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 413bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#endif /* CONFIG_MODULES */ 414bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron 415d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron/*** 416d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * jump_label_text_reserved - check if addr range is reserved 417d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * @start: start text addr 418d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * @end: end text addr 419d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * 420d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * checks if the text addr located between @start and @end 421d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * overlaps with any of the jump label patch addresses. Code 422d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * that wants to modify kernel text should first verify that 423d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * it does not overlap with any of the jump label addresses. 424d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * Caller must hold jump_label_mutex. 425d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * 426d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron * returns 1 if there is an overlap, 0 otherwise 427d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron */ 428d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baronint jump_label_text_reserved(void *start, void *end) 429d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron{ 430d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron int ret = __jump_label_text_reserved(__start___jump_table, 431d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron __stop___jump_table, start, end); 432d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 433d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron if (ret) 434d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return ret; 435d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 436d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron#ifdef CONFIG_MODULES 437d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron ret = __jump_label_mod_text_reserved(start, end); 438d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron#endif 439d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron return ret; 440d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron} 441d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 442c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnarstatic void jump_label_update(struct static_key *key, int enable) 443d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron{ 444c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct jump_entry *stop = __stop___jump_table; 445c5905afb0ee6550b42c49213da1c22d67316c194Ingo Molnar struct jump_entry *entry = jump_label_get_entries(key); 446d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 447d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron#ifdef CONFIG_MODULES 448a746e3cc984b0aa5b620dd07c1a433283b1835cfJason Baron struct module *mod = __module_address((unsigned long)key); 449140fe3b1ab9c082182ef13359fab4ddba95c24c3Xiao Guangrong 450d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron __jump_label_mod_update(key, enable); 451140fe3b1ab9c082182ef13359fab4ddba95c24c3Xiao Guangrong 452140fe3b1ab9c082182ef13359fab4ddba95c24c3Xiao Guangrong if (mod) 453140fe3b1ab9c082182ef13359fab4ddba95c24c3Xiao Guangrong stop = mod->jump_entries + mod->num_jump_entries; 454d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron#endif 455140fe3b1ab9c082182ef13359fab4ddba95c24c3Xiao Guangrong /* if there are no users, entry can be NULL */ 456140fe3b1ab9c082182ef13359fab4ddba95c24c3Xiao Guangrong if (entry) 457140fe3b1ab9c082182ef13359fab4ddba95c24c3Xiao Guangrong __jump_label_update(key, entry, stop, enable); 458d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron} 459d430d3d7e646eb1eac2bb4aa244a644312e67c76Jason Baron 460bf5438fca2950b03c21ad868090cc1a8fcd49536Jason Baron#endif 461