1749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall/* 2749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * Copyright (C) 2012 - Virtual Open Systems and Columbia University 3749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * Author: Christoffer Dall <c.dall@virtualopensystems.com> 4749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * 5749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * This program is free software; you can redistribute it and/or modify 6749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * it under the terms of the GNU General Public License, version 2, as 7749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * published by the Free Software Foundation. 8749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * 9749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * This program is distributed in the hope that it will be useful, 10749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * but WITHOUT ANY WARRANTY; without even the implied warranty of 11749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * GNU General Public License for more details. 13749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * 14749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * You should have received a copy of the GNU General Public License 15749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * along with this program; if not, write to the Free Software 16749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall */ 18342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 19342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <linux/linkage.h> 20342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <linux/const.h> 21342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <asm/unified.h> 22342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <asm/page.h> 23f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#include <asm/ptrace.h> 24749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall#include <asm/asm-offsets.h> 25749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall#include <asm/kvm_asm.h> 26342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <asm/kvm_arm.h> 27f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#include <asm/vfpmacros.h> 28f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#include "interrupts_head.S" 29342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 30342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall .text 31342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 32342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall__kvm_hyp_code_start: 33342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall .globl __kvm_hyp_code_start 34342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 35342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall/******************************************************************** 36342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * Flush per-VMID TLBs 37f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 3848762767e1c150d58c250650f8202b7d4ad65ec4Marc Zyngier * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); 39f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 40f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * We rely on the hardware to broadcast the TLB invalidation to all CPUs 41f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * inside the inner-shareable domain (which is the case for all v7 42f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * implementations). If we come across a non-IS SMP implementation, we'll 43f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * have to use an IPI based mechanism. Until then, we stick to the simple 44f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * hardware assisted version. 4548762767e1c150d58c250650f8202b7d4ad65ec4Marc Zyngier * 4648762767e1c150d58c250650f8202b7d4ad65ec4Marc Zyngier * As v7 does not support flushing per IPA, just nuke the whole TLB 4748762767e1c150d58c250650f8202b7d4ad65ec4Marc Zyngier * instead, ignoring the ipa value. 48342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall */ 4948762767e1c150d58c250650f8202b7d4ad65ec4Marc ZyngierENTRY(__kvm_tlb_flush_vmid_ipa) 50f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {r2, r3} 51f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 52479c5ae2f8a55509b691494cd13691d3dc31d102Marc Zyngier dsb ishst 53f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall add r0, r0, #KVM_VTTBR 54f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldrd r2, r3, [r0] 5519b0e60a63f758a28329aa40f4270a6c98c2dcb7Victor Kamensky mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR 56f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall isb 57f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored) 58e3ab547f57bd626201d4b715b696c80ad1ef4ba2Will Deacon dsb ish 59f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall isb 60f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r2, #0 61f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r3, #0 62f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcrr p15, 6, r2, r3, c2 @ Back to VMID #0 63f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall isb @ Not necessary if followed by eret 64f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 65f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall pop {r2, r3} 66d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall bx lr 6748762767e1c150d58c250650f8202b7d4ad65ec4Marc ZyngierENDPROC(__kvm_tlb_flush_vmid_ipa) 68d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 69d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall/******************************************************************** 70f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Flush TLBs and instruction caches of all CPUs inside the inner-shareable 71f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * domain, for all VMIDs 72f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 73f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * void __kvm_flush_vm_context(void); 74d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall */ 75342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer DallENTRY(__kvm_flush_vm_context) 76f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r0, #0 @ rn parameter for c15 flushes is SBZ 77f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 78f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */ 79f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 4, r0, c8, c3, 4 80f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */ 81f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 0, r0, c7, c1, 0 82e3ab547f57bd626201d4b715b696c80ad1ef4ba2Will Deacon dsb ish 83f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall isb @ Not necessary if followed by eret 84f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 85342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall bx lr 86342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer DallENDPROC(__kvm_flush_vm_context) 87342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 88f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 89342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall/******************************************************************** 90342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * Hypervisor world-switch code 91f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 92f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 93f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * int __kvm_vcpu_run(struct kvm_vcpu *vcpu) 94342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall */ 95342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer DallENTRY(__kvm_vcpu_run) 96f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Save the vcpu pointer 97f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR 98f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 99f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall save_host_regs 100f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 1011a89dd9113badd7487313410a5f2e09b2944f92bMarc Zyngier restore_vgic_state 10253e724067a4ee9373972079e225d0d5f683b9c5aMarc Zyngier restore_timer_state 1031a89dd9113badd7487313410a5f2e09b2944f92bMarc Zyngier 104f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Store hardware CP15 state and load guest state 105f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall read_cp15_state store_to_vcpu = 0 106f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall write_cp15_state read_from_vcpu = 1 107f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 108f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ If the host kernel has not been configured with VFPv3 support, 109f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ then it is safer if we deny guests from using it as well. 110f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#ifdef CONFIG_VFPv3 111f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Set FPEXC_EN so the guest doesn't trap floating point instructions 112f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall VFPFMRX r2, FPEXC @ VMRS 113f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {r2} 114f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall orr r2, r2, #FPEXC_EN 115f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall VFPFMXR FPEXC, r2 @ VMSR 116f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#endif 117f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 118f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Configure Hyp-role 119f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall configure_hyp_role vmentry 120f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 121f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Trap coprocessor CRx accesses 122f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall set_hstr vmentry 123f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)) 124f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall set_hdcr vmentry 125f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 126f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Write configured ID register into MIDR alias 127f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldr r1, [vcpu, #VCPU_MIDR] 128f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 4, r1, c0, c0, 0 129f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 130f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Write guest view of MPIDR into VMPIDR 131f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)] 132f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 4, r1, c0, c0, 5 133f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 134f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Set up guest memory translation 135f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldr r1, [vcpu, #VCPU_KVM] 136f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall add r1, r1, #KVM_VTTBR 137f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldrd r2, r3, [r1] 13819b0e60a63f758a28329aa40f4270a6c98c2dcb7Victor Kamensky mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR 139f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 140f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ We're all done, just restore the GPRs and go to the guest 141f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall restore_guest_regs 142f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall clrex @ Clear exclusive monitor 143f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall eret 144f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 145f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall__kvm_vcpu_return: 146f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* 147f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * return convention: 148f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * guest r0, r1, r2 saved on the stack 149f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * r0: vcpu pointer 150f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * r1: exception code 151f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall */ 152f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall save_guest_regs 153f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 154f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Set VMID == 0 155f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r2, #0 156f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r3, #0 157f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcrr p15, 6, r2, r3, c2 @ Write VTTBR 158f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 159f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Don't trap coprocessor accesses for host kernel 160f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall set_hstr vmexit 161f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall set_hdcr vmexit 162f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)) 163f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 164f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#ifdef CONFIG_VFPv3 165f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Save floating point registers we if let guest use them. 166f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall tst r2, #(HCPTR_TCP(10) | HCPTR_TCP(11)) 167f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bne after_vfp_restore 168f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 169f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Switch VFP/NEON hardware state to the host's 170f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall add r7, vcpu, #VCPU_VFP_GUEST 171f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall store_vfp_state r7 172f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall add r7, vcpu, #VCPU_VFP_HOST 173f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldr r7, [r7] 174f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall restore_vfp_state r7 175f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 176f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallafter_vfp_restore: 177f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Restore FPEXC_EN which we clobbered on entry 178f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall pop {r2} 179f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall VFPFMXR FPEXC, r2 180f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#endif 181f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 182f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Reset Hyp-role 183f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall configure_hyp_role vmexit 184f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 185f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Let host read hardware MIDR 186f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 0, r2, c0, c0, 0 187f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 4, r2, c0, c0, 0 188f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 189f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Back to hardware MPIDR 190f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 0, r2, c0, c0, 5 191f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 4, r2, c0, c0, 5 192f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 193f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Store guest CP15 state and restore host state 194f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall read_cp15_state store_to_vcpu = 1 195f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall write_cp15_state read_from_vcpu = 0 196f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 19753e724067a4ee9373972079e225d0d5f683b9c5aMarc Zyngier save_timer_state 1981a89dd9113badd7487313410a5f2e09b2944f92bMarc Zyngier save_vgic_state 1991a89dd9113badd7487313410a5f2e09b2944f92bMarc Zyngier 200f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall restore_host_regs 201f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall clrex @ Clear exclusive monitor 2026d7311b520864531c81f0e0237e96146d8057d77Victor Kamensky#ifndef CONFIG_CPU_ENDIAN_BE8 203f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r0, r1 @ Return the return code 204f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r1, #0 @ Clear upper bits in return value 2056d7311b520864531c81f0e0237e96146d8057d77Victor Kamensky#else 2066d7311b520864531c81f0e0237e96146d8057d77Victor Kamensky @ r1 already has return code 2076d7311b520864531c81f0e0237e96146d8057d77Victor Kamensky mov r0, #0 @ Clear upper bits in return value 2086d7311b520864531c81f0e0237e96146d8057d77Victor Kamensky#endif /* CONFIG_CPU_ENDIAN_BE8 */ 209f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bx lr @ return to IOCTL 210342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 211342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall/******************************************************************** 212342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * Call function in Hyp mode 213342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * 214342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * 215342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * u64 kvm_call_hyp(void *hypfn, ...); 216342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * 217342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * This is not really a variadic function in the classic C-way and care must 218342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * be taken when calling this to ensure parameters are passed in registers 219342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * only, since the stack will change between the caller and the callee. 220342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * 221342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * Call the function with the first argument containing a pointer to the 222342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * function you wish to call in Hyp mode, and subsequent arguments will be 223342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the 224342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * function pointer can be passed). The function being called must be mapped 225342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are 226342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * passed in r0 and r1. 227342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * 228b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier * A function pointer with a value of 0xffffffff has a special meaning, 229b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier * and is used to implement __hyp_get_vectors in the same way as in 230b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier * arch/arm/kernel/hyp_stub.S. 231b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier * 232342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * The calling convention follows the standard AAPCS: 233342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * r0 - r3: caller save 234342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * r12: caller save 235342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * rest: callee save 236342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall */ 237342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer DallENTRY(kvm_call_hyp) 238342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall hvc #0 239342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall bx lr 240342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 241342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall/******************************************************************** 242342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * Hypervisor exception vector and handlers 243f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 244f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 245f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * The KVM/ARM Hypervisor ABI is defined as follows: 246f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 247f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Entry to Hyp mode from the host kernel will happen _only_ when an HVC 248f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * instruction is issued since all traps are disabled when running the host 249f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * kernel as per the Hyp-mode initialization at boot time. 250f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 2510b5e3bac30c545720f7e6b026241b5f8dd832df2Jonghwan Choi * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc 252f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * below) when the HVC instruction is called from SVC mode (i.e. a guest or the 2530b5e3bac30c545720f7e6b026241b5f8dd832df2Jonghwan Choi * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC 254f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * instructions are called from within Hyp-mode. 255f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 256f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode): 257f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Switching to Hyp mode is done through a simple HVC #0 instruction. The 258f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * exception vector code will check that the HVC comes from VMID==0 and if 259f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * so will push the necessary state (SPSR, lr_usr) on the Hyp stack. 260f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * - r0 contains a pointer to a HYP function 261f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * - r1, r2, and r3 contain arguments to the above function. 262f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * - The HYP function will be called with its arguments in r0, r1 and r2. 263f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * On HYP function return, we return directly to SVC. 264f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 265f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Note that the above is used to execute code in Hyp-mode from a host-kernel 266f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * point of view, and is a different concept from performing a world-switch and 267f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * executing guest code SVC mode (with a VMID != 0). 268342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall */ 269342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 270f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall/* Handle undef, svc, pabt, or dabt by crashing with a user notice */ 271f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall.macro bad_exception exception_code, panic_str 272f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {r0-r2} 273f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrrc p15, 6, r0, r1, c2 @ Read VTTBR 274f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall lsr r1, r1, #16 275f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ands r1, r1, #0xff 276f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall beq 99f 277f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 278f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall load_vcpu @ Load VCPU pointer 279f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .if \exception_code == ARM_EXCEPTION_DATA_ABORT 280f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 4, r2, c5, c2, 0 @ HSR 281f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 4, r1, c6, c0, 0 @ HDFAR 282f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall str r2, [vcpu, #VCPU_HSR] 283f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall str r1, [vcpu, #VCPU_HxFAR] 284f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .endif 285f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .if \exception_code == ARM_EXCEPTION_PREF_ABORT 286f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 4, r2, c5, c2, 0 @ HSR 287f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 4, r1, c6, c0, 2 @ HIFAR 288f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall str r2, [vcpu, #VCPU_HSR] 289f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall str r1, [vcpu, #VCPU_HxFAR] 290f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .endif 291f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r1, #\exception_code 292f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall b __kvm_vcpu_return 293f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 294f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ We were in the host already. Let's craft a panic-ing return to SVC. 295f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall99: mrs r2, cpsr 296f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bic r2, r2, #MODE_MASK 297f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall orr r2, r2, #SVC_MODE 298f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer DallTHUMB( orr r2, r2, #PSR_T_BIT ) 299f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall msr spsr_cxsf, r2 300f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrs r1, ELR_hyp 301f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldr r2, =BSYM(panic) 302f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall msr ELR_hyp, r2 303f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldr r0, =\panic_str 30422cfbb6d730ca2fda236b507d9fba17bf002736cMarc Zyngier clrex @ Clear exclusive monitor 305f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall eret 306f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall.endm 307f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 308f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .text 309f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 310342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall .align 5 311342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall__kvm_hyp_vector: 312342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall .globl __kvm_hyp_vector 313f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 314f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Hyp-mode exception vector 315f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_reset 316f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_undef 317f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_svc 318f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_pabt 319f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_dabt 320f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_hvc 321f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_irq 322f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall W(b) hyp_fiq 323f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 324f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 325f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_reset: 326f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall b hyp_reset 327f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 328f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 329f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_undef: 330f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str 331f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 332f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 333f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_svc: 334f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bad_exception ARM_EXCEPTION_HVC, svc_die_str 335f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 336f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 337f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_pabt: 338f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str 339f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 340f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 341f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_dabt: 342f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str 343f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 344f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 345f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_hvc: 346f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* 347f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Getting here is either becuase of a trap from a guest or from calling 348f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * HVC from the host kernel, which means "switch to Hyp mode". 349f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall */ 350f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {r0, r1, r2} 351f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 352f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Check syndrome register 353f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 4, r1, c5, c2, 0 @ HSR 354f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall lsr r0, r1, #HSR_EC_SHIFT 355f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#ifdef CONFIG_VFPv3 356f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall cmp r0, #HSR_EC_CP_0_13 357f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall beq switch_to_guest_vfp 358f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#endif 359f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall cmp r0, #HSR_EC_HVC 360f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bne guest_trap @ Not HVC instr. 361f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 362f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* 363f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Let's check if the HVC came from VMID 0 and allow simple 364f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * switch to Hyp mode 365f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall */ 366f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrrc p15, 6, r0, r2, c2 367f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall lsr r2, r2, #16 368f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall and r2, r2, #0xff 369f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall cmp r2, #0 370f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bne guest_trap @ Guest called HVC 371f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 372f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhost_switch_to_hyp: 373f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall pop {r0, r1, r2} 374f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 375b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier /* Check for __hyp_get_vectors */ 376b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier cmp r0, #-1 377b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR 378b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier beq 1f 379b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier 380f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {lr} 381f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrs lr, SPSR 382f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {lr} 383f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 384f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov lr, r0 385f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r0, r1 386f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r1, r2 387f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r2, r3 388f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 389f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer DallTHUMB( orr lr, #1) 390f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall blx lr @ Call the HYP function 391f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 392f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall pop {lr} 393f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall msr SPSR_csxf, lr 394f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall pop {lr} 395b20c9f29c5c25921c6ad18b50d4b61e6d181c3ccMarc Zyngier1: eret 396f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 397f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallguest_trap: 398f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall load_vcpu @ Load VCPU pointer to r0 399f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall str r1, [vcpu, #VCPU_HSR] 400f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 401f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Check if we need the fault information 402f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall lsr r1, r1, #HSR_EC_SHIFT 403f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall cmp r1, #HSR_EC_IABT 404f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrceq p15, 4, r2, c6, c0, 2 @ HIFAR 405f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall beq 2f 406f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall cmp r1, #HSR_EC_DABT 407f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bne 1f 408f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 4, r2, c6, c0, 0 @ HDFAR 409f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 410f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall2: str r2, [vcpu, #VCPU_HxFAR] 411f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 412f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* 413f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode: 414f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 415f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Abort on the stage 2 translation for a memory access from a 416f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Non-secure PL1 or PL0 mode: 417f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * 418f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * For any Access flag fault or Translation fault, and also for any 419f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * Permission fault on the stage 2 translation of a memory access 420f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * made as part of a translation table walk for a stage 1 translation, 421f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR 422f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * is UNKNOWN. 423f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall */ 424f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 425f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* Check for permission fault, and S1PTW */ 426f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrc p15, 4, r1, c5, c2, 0 @ HSR 427f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall and r0, r1, #HSR_FSC_TYPE 428f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall cmp r0, #FSC_PERM 429f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall tsteq r1, #(1 << 7) @ S1PTW 430f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrcne p15, 4, r2, c6, c0, 4 @ HPFAR 431f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bne 3f 432f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 4336a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier /* Preserve PAR */ 4346a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier mrrc p15, 0, r0, r1, c7 @ PAR 4356a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier push {r0, r1} 4366a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier 437f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall /* Resolve IPA using the xFAR */ 438f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR 439f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall isb 440f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mrrc p15, 0, r0, r1, c7 @ PAR 441f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall tst r0, #1 442f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall bne 4f @ Failed translation 443f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ubfx r2, r0, #12, #20 444f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall lsl r2, r2, #4 445f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall orr r2, r2, r1, lsl #24 446f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 4476a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier /* Restore PAR */ 4486a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier pop {r0, r1} 4496a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier mcrr p15, 0, r0, r1, c7 @ PAR 4506a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier 451f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall3: load_vcpu @ Load VCPU pointer to r0 452f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall str r2, [r0, #VCPU_HPFAR] 453f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 454f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall1: mov r1, #ARM_EXCEPTION_HVC 455f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall b __kvm_vcpu_return 456f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 4576a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier4: pop {r0, r1} @ Failed translation, return to guest 4586a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier mcrr p15, 0, r0, r1, c7 @ PAR 45922cfbb6d730ca2fda236b507d9fba17bf002736cMarc Zyngier clrex 4606a077e4ab9cbfbf279fb955bae05b03781c97013Marc Zyngier pop {r0, r1, r2} 461f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall eret 462f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 463f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall/* 464f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * If VFPv3 support is not available, then we will not switch the VFP 465f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * registers; however cp10 and cp11 accesses will still trap and fallback 466f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * to the regular coprocessor emulation code, which currently will 467f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall * inject an undefined exception to the guest. 468f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall */ 469f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#ifdef CONFIG_VFPv3 470f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallswitch_to_guest_vfp: 471f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall load_vcpu @ Load VCPU pointer to r0 472f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {r3-r7} 473f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 474f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ NEON/VFP used. Turn on VFP access. 475f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11)) 476f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 477f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall @ Switch VFP/NEON hardware state to the guest's 478f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall add r7, r0, #VCPU_VFP_HOST 479f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall ldr r7, [r7] 480f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall store_vfp_state r7 481f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall add r7, r0, #VCPU_VFP_GUEST 482f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall restore_vfp_state r7 483f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 484f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall pop {r3-r7} 485f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall pop {r0-r2} 48622cfbb6d730ca2fda236b507d9fba17bf002736cMarc Zyngier clrex 487f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall eret 488f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall#endif 489f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 490f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 491f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_irq: 492f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall push {r0, r1, r2} 493f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall mov r1, #ARM_EXCEPTION_IRQ 494f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall load_vcpu @ Load VCPU pointer to r0 495f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall b __kvm_vcpu_return 496f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 497f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .align 498f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallhyp_fiq: 499f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall b hyp_fiq 500f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 501f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .ltorg 502342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 503342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall__kvm_hyp_code_end: 504342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall .globl __kvm_hyp_code_end 505f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 506f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall .section ".rodata" 507f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dall 508f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallund_die_str: 5091fe40f6d39d23f39e643607a3e1883bfc74f1244Christoffer Dall .ascii "unexpected undefined exception in Hyp mode at: %#08x\n" 510f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallpabt_die_str: 5111fe40f6d39d23f39e643607a3e1883bfc74f1244Christoffer Dall .ascii "unexpected prefetch abort in Hyp mode at: %#08x\n" 512f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dalldabt_die_str: 5131fe40f6d39d23f39e643607a3e1883bfc74f1244Christoffer Dall .ascii "unexpected data abort in Hyp mode at: %#08x\n" 514f7ed45be3ba524e06a6d933f0517dc7ad2d06703Christoffer Dallsvc_die_str: 5151fe40f6d39d23f39e643607a3e1883bfc74f1244Christoffer Dall .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x\n" 516