1c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifndef X86_64_PDA_H 2c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define X86_64_PDA_H 3c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 4c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifndef __ASSEMBLY__ 5c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <linux/stddef.h> 6c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <linux/types.h> 7c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <linux/cache.h> 8c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#include <asm/page.h> 9c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 10c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* Per processor datastructure. %gs points to it while the kernel runs */ 11c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Querustruct x8664_pda { 12c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru struct task_struct *pcurrent; /* 0 Current process */ 13c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long data_offset; /* 8 Per cpu data offset from linker 14c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru address */ 15c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long kernelstack; /* 16 top of kernel stack for current */ 16c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long oldrsp; /* 24 user rsp for system call */ 17c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru int irqcount; /* 32 Irq nesting counter. Starts with -1 */ 18c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru int cpunumber; /* 36 Logical CPU number */ 19c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#ifdef CONFIG_CC_STACKPROTECTOR 20c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned long stack_canary; /* 40 stack canary value */ 21c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru /* gcc-ABI: this canary MUST be at 22c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru offset 40!!! */ 23c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif 24c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru char *irqstackptr; 25c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru int nodenumber; /* number of current node */ 26c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned int __softirq_pending; 27c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned int __nmi_count; /* number of NMI on this CPUs */ 28c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru short mmu_state; 29c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru short isidle; 30c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru struct mm_struct *active_mm; 31c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned apic_timer_irqs; 32c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned irq0_irqs; 33c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned irq_resched_count; 34c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned irq_call_count; 35c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned irq_tlb_count; 36c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned irq_thermal_count; 37c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned irq_threshold_count; 38c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru unsigned irq_spurious_count; 39c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru} ____cacheline_aligned_in_smp; 40c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 41c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queruextern struct x8664_pda *_cpu_pda[]; 42c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queruextern struct x8664_pda boot_cpu_pda[]; 43c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 44c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define cpu_pda(i) (_cpu_pda[i]) 45c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 46c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* 47c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * There is no fast way to get the base address of the PDA, all the accesses 48c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * have to mention %fs/%gs. So it needs to be done this Torvaldian way. 49c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 50c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queruextern void __bad_pda_field(void) __attribute__((noreturn)); 51c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 52c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* 53c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * proxy_pda doesn't actually exist, but tell gcc it is accessed for 54c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru * all PDA accesses so it gets read/write dependencies right. 55c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru */ 56c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queruextern struct x8664_pda _proxy_pda; 57c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 58c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define pda_offset(field) offsetof(struct x8664_pda, field) 59c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 60c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define pda_to_op(op,field,val) do { \ 61c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru typedef typeof(_proxy_pda.field) T__; \ 62c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru if (0) { T__ tmp__; tmp__ = (val); } /* type checking */ \ 63c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru switch (sizeof(_proxy_pda.field)) { \ 64c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru case 2: \ 65c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru asm(op "w %1,%%gs:%c2" : \ 66c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "+m" (_proxy_pda.field) : \ 67c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "ri" ((T__)val), \ 68c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "i"(pda_offset(field))); \ 69c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru break; \ 70c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru case 4: \ 71c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru asm(op "l %1,%%gs:%c2" : \ 72c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "+m" (_proxy_pda.field) : \ 73c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "ri" ((T__)val), \ 74c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "i" (pda_offset(field))); \ 75c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru break; \ 76c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru case 8: \ 77c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru asm(op "q %1,%%gs:%c2": \ 78c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "+m" (_proxy_pda.field) : \ 79c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "ri" ((T__)val), \ 80c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "i"(pda_offset(field))); \ 81c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru break; \ 82c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru default: \ 83c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __bad_pda_field(); \ 84c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } \ 85c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } while (0) 86c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 87c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define pda_from_op(op,field) ({ \ 88c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru typeof(_proxy_pda.field) ret__; \ 89c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru switch (sizeof(_proxy_pda.field)) { \ 90c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru case 2: \ 91c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru asm(op "w %%gs:%c1,%0" : \ 92c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "=r" (ret__) : \ 93c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "i" (pda_offset(field)), \ 94c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "m" (_proxy_pda.field)); \ 95c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru break; \ 96c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru case 4: \ 97c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru asm(op "l %%gs:%c1,%0": \ 98c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "=r" (ret__): \ 99c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "i" (pda_offset(field)), \ 100c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "m" (_proxy_pda.field)); \ 101c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru break; \ 102c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru case 8: \ 103c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru asm(op "q %%gs:%c1,%0": \ 104c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "=r" (ret__) : \ 105c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "i" (pda_offset(field)), \ 106c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru "m" (_proxy_pda.field)); \ 107c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru break; \ 108c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru default: \ 109c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru __bad_pda_field(); \ 110c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru } \ 111c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru ret__; }) 112c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 113c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define read_pda(field) pda_from_op("mov",field) 114c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define write_pda(field,val) pda_to_op("mov",field,val) 115c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define add_pda(field,val) pda_to_op("add",field,val) 116c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define sub_pda(field,val) pda_to_op("sub",field,val) 117c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define or_pda(field,val) pda_to_op("or",field,val) 118c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 119c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru/* This is not atomic against other CPUs -- CPU preemption needs to be off */ 120c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define test_and_clear_bit_pda(bit,field) ({ \ 121c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru int old__; \ 122c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru asm volatile("btr %2,%%gs:%c3\n\tsbbl %0,%0" \ 123c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru : "=r" (old__), "+m" (_proxy_pda.field) \ 124c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru : "dIr" (bit), "i" (pda_offset(field)) : "memory"); \ 125c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru old__; \ 126c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru}) 127c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 128c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif 129c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 130c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#define PDA_STACKOFFSET (5*8) 131c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru 132c559cd81139f97cecad1ad91a0b2e25a5936d53Jean-Baptiste Queru#endif 133