1749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall/* 2749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * Copyright (C) 2012 - Virtual Open Systems and Columbia University 3749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * Author: Christoffer Dall <c.dall@virtualopensystems.com> 4749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * 5749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * This program is free software; you can redistribute it and/or modify 6749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * it under the terms of the GNU General Public License, version 2, as 7749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * published by the Free Software Foundation. 8749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * 9749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * This program is distributed in the hope that it will be useful, 10749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * but WITHOUT ANY WARRANTY; without even the implied warranty of 11749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * GNU General Public License for more details. 13749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * 14749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * You should have received a copy of the GNU General Public License 15749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * along with this program; if not, write to the Free Software 16749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 17749cf76c5a363e1383108a914ea09530bfa0bd43Christoffer Dall */ 18342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 19342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <linux/mman.h> 20342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <linux/kvm_host.h> 21342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <linux/io.h> 22ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall#include <linux/hugetlb.h> 2345e96ea6b369539a37040a8df9c59a39f073d9d6Christoffer Dall#include <trace/events/kvm.h> 24342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <asm/pgalloc.h> 2594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall#include <asm/cacheflush.h> 26342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <asm/kvm_arm.h> 27342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall#include <asm/kvm_mmu.h> 2845e96ea6b369539a37040a8df9c59a39f073d9d6Christoffer Dall#include <asm/kvm_mmio.h> 29d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall#include <asm/kvm_asm.h> 3094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall#include <asm/kvm_emulate.h> 31d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 32d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall#include "trace.h" 33342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 34342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallextern char __hyp_idmap_text_start[], __hyp_idmap_text_end[]; 35342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 365a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngierstatic pgd_t *boot_hyp_pgd; 372fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngierstatic pgd_t *hyp_pgd; 38342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallstatic DEFINE_MUTEX(kvm_hyp_pgd_mutex); 39342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 405a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngierstatic void *init_bounce_page; 415a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngierstatic unsigned long hyp_idmap_start; 425a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngierstatic unsigned long hyp_idmap_end; 435a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngierstatic phys_addr_t hyp_idmap_vector; 445a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 4538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall#define hyp_pgd_order get_order(PTRS_PER_PGD * sizeof(pgd_t)) 465d4e08c45a6cf8f1ab3c7fa375007635ac569165Mark Salter 479b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall#define kvm_pmd_huge(_x) (pmd_huge(_x) || pmd_trans_huge(_x)) 48ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 4948762767e1c150d58c250650f8202b7d4ad65ec4Marc Zyngierstatic void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa) 50d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 51d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier /* 52d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier * This function also gets called when dealing with HYP page 53d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier * tables. As HYP doesn't have an associated struct kvm (and 54d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier * the HYP page tables are fairly static), we don't do 55d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier * anything there. 56d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier */ 57d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier if (kvm) 58d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa); 59d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 60d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 61d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallstatic int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, 62d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall int min, int max) 63d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 64d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall void *page; 65d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 66d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall BUG_ON(max > KVM_NR_MEM_OBJS); 67d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (cache->nobjs >= min) 68d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 69d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall while (cache->nobjs < max) { 70d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall page = (void *)__get_free_page(PGALLOC_GFP); 71d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (!page) 72d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return -ENOMEM; 73d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall cache->objects[cache->nobjs++] = page; 74d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall } 75d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 76d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 77d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 78d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallstatic void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) 79d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 80d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall while (mc->nobjs) 81d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall free_page((unsigned long)mc->objects[--mc->nobjs]); 82d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 83d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 84d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallstatic void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) 85d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 86d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall void *p; 87d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 88d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall BUG_ON(!mc || !mc->nobjs); 89d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall p = mc->objects[--mc->nobjs]; 90d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return p; 91d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 92d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 934f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dallstatic void clear_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr) 94979acd5e18c3e5cb7e3308c699d79553af5af8c6Marc Zyngier{ 954f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pud_t *pud_table __maybe_unused = pud_offset(pgd, 0); 964f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pgd_clear(pgd); 974f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall kvm_tlb_flush_vmid_ipa(kvm, addr); 984f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pud_free(NULL, pud_table); 994f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall put_page(virt_to_page(pgd)); 100979acd5e18c3e5cb7e3308c699d79553af5af8c6Marc Zyngier} 101979acd5e18c3e5cb7e3308c699d79553af5af8c6Marc Zyngier 102d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngierstatic void clear_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr) 103342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 1044f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pmd_t *pmd_table = pmd_offset(pud, 0); 1054f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall VM_BUG_ON(pud_huge(*pud)); 1064f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pud_clear(pud); 1074f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall kvm_tlb_flush_vmid_ipa(kvm, addr); 1084f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pmd_free(NULL, pmd_table); 1094f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier put_page(virt_to_page(pud)); 1104f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier} 111342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 112d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngierstatic void clear_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr) 1134f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier{ 1144f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pte_t *pte_table = pte_offset_kernel(pmd, 0); 1154f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall VM_BUG_ON(kvm_pmd_huge(*pmd)); 1164f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pmd_clear(pmd); 1174f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall kvm_tlb_flush_vmid_ipa(kvm, addr); 1184f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pte_free_kernel(NULL, pte_table); 1194f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier put_page(virt_to_page(pmd)); 1204f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier} 1214f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier 1224f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dallstatic void unmap_ptes(struct kvm *kvm, pmd_t *pmd, 1234f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t addr, phys_addr_t end) 1244f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier{ 1254f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t start_addr = addr; 1264f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pte_t *pte, *start_pte; 1274f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall 1284f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall start_pte = pte = pte_offset_kernel(pmd, addr); 1294f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall do { 1304f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall if (!pte_none(*pte)) { 1314f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall kvm_set_pte(pte, __pte(0)); 1324f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall put_page(virt_to_page(pte)); 1334f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall kvm_tlb_flush_vmid_ipa(kvm, addr); 1344f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } 1354f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } while (pte++, addr += PAGE_SIZE, addr != end); 1364f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall 13738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (kvm_pte_table_empty(kvm, start_pte)) 1384f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall clear_pmd_entry(kvm, pmd, start_addr); 139342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 140342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 1414f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dallstatic void unmap_pmds(struct kvm *kvm, pud_t *pud, 1424f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t addr, phys_addr_t end) 143000d399625b4b302935508f2fc9ce93ff1bd1ba4Marc Zyngier{ 1444f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t next, start_addr = addr; 1454f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pmd_t *pmd, *start_pmd; 146000d399625b4b302935508f2fc9ce93ff1bd1ba4Marc Zyngier 1474f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall start_pmd = pmd = pmd_offset(pud, addr); 1484f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall do { 1494f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall next = kvm_pmd_addr_end(addr, end); 1504f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall if (!pmd_none(*pmd)) { 1514f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall if (kvm_pmd_huge(*pmd)) { 1524f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pmd_clear(pmd); 1534f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall kvm_tlb_flush_vmid_ipa(kvm, addr); 1544f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall put_page(virt_to_page(pmd)); 1554f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } else { 1564f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall unmap_ptes(kvm, pmd, addr, next); 1574f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } 158ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall } 1594f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } while (pmd++, addr = next, addr != end); 160ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 16138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (kvm_pmd_table_empty(kvm, start_pmd)) 1624f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall clear_pud_entry(kvm, pud, start_addr); 1634f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall} 164000d399625b4b302935508f2fc9ce93ff1bd1ba4Marc Zyngier 1654f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dallstatic void unmap_puds(struct kvm *kvm, pgd_t *pgd, 1664f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t addr, phys_addr_t end) 1674f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall{ 1684f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t next, start_addr = addr; 1694f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pud_t *pud, *start_pud; 1704f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier 1714f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall start_pud = pud = pud_offset(pgd, addr); 1724f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall do { 1734f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall next = kvm_pud_addr_end(addr, end); 1744f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall if (!pud_none(*pud)) { 1754f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall if (pud_huge(*pud)) { 1764f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pud_clear(pud); 1774f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall kvm_tlb_flush_vmid_ipa(kvm, addr); 1784f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall put_page(virt_to_page(pud)); 1794f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } else { 1804f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall unmap_pmds(kvm, pud, addr, next); 1814f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier } 1824f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier } 1834f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } while (pud++, addr = next, addr != end); 1844f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier 18538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (kvm_pud_table_empty(kvm, start_pud)) 1864f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall clear_pgd_entry(kvm, pgd, start_addr); 1874f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall} 1884f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall 1894f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall 1904f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dallstatic void unmap_range(struct kvm *kvm, pgd_t *pgdp, 1914f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t start, u64 size) 1924f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall{ 1934f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pgd_t *pgd; 1944f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t addr = start, end = start + size; 1954f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall phys_addr_t next; 1964f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall 1974f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall pgd = pgdp + pgd_index(addr); 1984f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall do { 1994f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall next = kvm_pgd_addr_end(addr, end); 2007cbb87d67e38cfc55680290a706fd7517f10050dMark Rutland if (!pgd_none(*pgd)) 2017cbb87d67e38cfc55680290a706fd7517f10050dMark Rutland unmap_puds(kvm, pgd, addr, next); 2024f853a714bf16338ff5261128e6c7ae2569e9505Christoffer Dall } while (pgd++, addr = next, addr != end); 203000d399625b4b302935508f2fc9ce93ff1bd1ba4Marc Zyngier} 204000d399625b4b302935508f2fc9ce93ff1bd1ba4Marc Zyngier 2059d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngierstatic void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd, 2069d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t addr, phys_addr_t end) 2079d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier{ 2089d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pte_t *pte; 2099d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2109d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pte = pte_offset_kernel(pmd, addr); 2119d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier do { 2129d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier if (!pte_none(*pte)) { 2139d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT); 2149d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier kvm_flush_dcache_to_poc((void*)hva, PAGE_SIZE); 2159d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } 2169d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } while (pte++, addr += PAGE_SIZE, addr != end); 2179d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier} 2189d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2199d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngierstatic void stage2_flush_pmds(struct kvm *kvm, pud_t *pud, 2209d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t addr, phys_addr_t end) 2219d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier{ 2229d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pmd_t *pmd; 2239d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t next; 2249d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2259d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pmd = pmd_offset(pud, addr); 2269d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier do { 2279d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier next = kvm_pmd_addr_end(addr, end); 2289d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier if (!pmd_none(*pmd)) { 2299d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier if (kvm_pmd_huge(*pmd)) { 2309d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT); 2319d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier kvm_flush_dcache_to_poc((void*)hva, PMD_SIZE); 2329d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } else { 2339d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier stage2_flush_ptes(kvm, pmd, addr, next); 2349d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } 2359d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } 2369d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } while (pmd++, addr = next, addr != end); 2379d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier} 2389d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2399d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngierstatic void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd, 2409d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t addr, phys_addr_t end) 2419d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier{ 2429d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pud_t *pud; 2439d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t next; 2449d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2459d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pud = pud_offset(pgd, addr); 2469d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier do { 2479d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier next = kvm_pud_addr_end(addr, end); 2489d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier if (!pud_none(*pud)) { 2499d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier if (pud_huge(*pud)) { 2509d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT); 2519d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier kvm_flush_dcache_to_poc((void*)hva, PUD_SIZE); 2529d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } else { 2539d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier stage2_flush_pmds(kvm, pud, addr, next); 2549d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } 2559d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } 2569d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } while (pud++, addr = next, addr != end); 2579d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier} 2589d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2599d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngierstatic void stage2_flush_memslot(struct kvm *kvm, 2609d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier struct kvm_memory_slot *memslot) 2619d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier{ 2629d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; 2639d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t end = addr + PAGE_SIZE * memslot->npages; 2649d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier phys_addr_t next; 2659d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pgd_t *pgd; 2669d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2679d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier pgd = kvm->arch.pgd + pgd_index(addr); 2689d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier do { 2699d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier next = kvm_pgd_addr_end(addr, end); 2709d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier stage2_flush_puds(kvm, pgd, addr, next); 2719d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier } while (pgd++, addr = next, addr != end); 2729d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier} 2739d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2749d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier/** 2759d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier * stage2_flush_vm - Invalidate cache for pages mapped in stage 2 2769d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier * @kvm: The struct kvm pointer 2779d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier * 2789d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier * Go through the stage 2 page tables and invalidate any cache lines 2799d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier * backing memory already mapped to the VM. 2809d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier */ 2819d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngiervoid stage2_flush_vm(struct kvm *kvm) 2829d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier{ 2839d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier struct kvm_memslots *slots; 2849d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier struct kvm_memory_slot *memslot; 2859d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier int idx; 2869d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2879d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier idx = srcu_read_lock(&kvm->srcu); 2889d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier spin_lock(&kvm->mmu_lock); 2899d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2909d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier slots = kvm_memslots(kvm); 2919d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier kvm_for_each_memslot(memslot, slots) 2929d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier stage2_flush_memslot(kvm, memslot); 2939d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 2949d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier spin_unlock(&kvm->mmu_lock); 2959d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier srcu_read_unlock(&kvm->srcu, idx); 2969d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier} 2979d218a1fcf4c6b759d442ef702842fae92e1ea61Marc Zyngier 298342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall/** 299d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier * free_boot_hyp_pgd - free HYP boot page tables 300d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier * 301d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier * Free the HYP boot page tables. The bounce page is also freed. 302d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier */ 303d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngiervoid free_boot_hyp_pgd(void) 304d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier{ 305d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier mutex_lock(&kvm_hyp_pgd_mutex); 306d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier 307d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier if (boot_hyp_pgd) { 308d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier unmap_range(NULL, boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE); 309d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier unmap_range(NULL, boot_hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE); 31038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall free_pages((unsigned long)boot_hyp_pgd, hyp_pgd_order); 311d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier boot_hyp_pgd = NULL; 312d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier } 313d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier 314d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier if (hyp_pgd) 315d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier unmap_range(NULL, hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE); 316d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier 3175d4e08c45a6cf8f1ab3c7fa375007635ac569165Mark Salter free_page((unsigned long)init_bounce_page); 318d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier init_bounce_page = NULL; 319d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier 320d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier mutex_unlock(&kvm_hyp_pgd_mutex); 321d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier} 322d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier 323d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier/** 3244f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier * free_hyp_pgds - free Hyp-mode page tables 325342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * 3265a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * Assumes hyp_pgd is a page table used strictly in Hyp-mode and 3275a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * therefore contains either mappings in the kernel memory area (above 3285a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * PAGE_OFFSET), or device mappings in the vmalloc range (from 3295a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * VMALLOC_START to VMALLOC_END). 3305a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * 3315a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * boot_hyp_pgd should only map two pages for the init code. 332342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall */ 3334f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngiervoid free_hyp_pgds(void) 334342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 335342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall unsigned long addr; 336342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 337d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier free_boot_hyp_pgd(); 3384f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier 339d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier mutex_lock(&kvm_hyp_pgd_mutex); 3405a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 3414f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier if (hyp_pgd) { 3424f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE) 343d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE); 3444f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE) 345d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE); 346d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier 34738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall free_pages((unsigned long)hyp_pgd, hyp_pgd_order); 348d157f4a5155f4fbd0d1da66b3d2f504c13bd194dMarc Zyngier hyp_pgd = NULL; 3494f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier } 3504f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier 351342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall mutex_unlock(&kvm_hyp_pgd_mutex); 352342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 353342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 354342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallstatic void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start, 3556060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long end, unsigned long pfn, 3566060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pgprot_t prot) 357342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 358342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall pte_t *pte; 359342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall unsigned long addr; 360342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 3613562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier addr = start; 3623562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier do { 3636060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pte = pte_offset_kernel(pmd, addr); 3646060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier kvm_set_pte(pte, pfn_pte(pfn, prot)); 3654f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier get_page(virt_to_page(pte)); 3665a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier kvm_flush_dcache_to_poc(pte, sizeof(*pte)); 3676060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pfn++; 3683562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier } while (addr += PAGE_SIZE, addr != end); 369342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 370342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 371342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallstatic int create_hyp_pmd_mappings(pud_t *pud, unsigned long start, 3726060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long end, unsigned long pfn, 3736060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pgprot_t prot) 374342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 375342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall pmd_t *pmd; 376342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall pte_t *pte; 377342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall unsigned long addr, next; 378342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 3793562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier addr = start; 3803562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier do { 3816060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pmd = pmd_offset(pud, addr); 382342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 383342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall BUG_ON(pmd_sect(*pmd)); 384342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 385342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall if (pmd_none(*pmd)) { 3866060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pte = pte_alloc_one_kernel(NULL, addr); 387342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall if (!pte) { 388342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall kvm_err("Cannot allocate Hyp pte\n"); 389342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall return -ENOMEM; 390342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall } 391342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall pmd_populate_kernel(NULL, pmd, pte); 3924f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier get_page(virt_to_page(pmd)); 3935a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier kvm_flush_dcache_to_poc(pmd, sizeof(*pmd)); 394342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall } 395342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 396342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall next = pmd_addr_end(addr, end); 397342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 3986060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier create_hyp_pte_mappings(pmd, addr, next, pfn, prot); 3996060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pfn += (next - addr) >> PAGE_SHIFT; 4003562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier } while (addr = next, addr != end); 401342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 402342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall return 0; 403342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 404342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 40538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dallstatic int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start, 40638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall unsigned long end, unsigned long pfn, 40738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pgprot_t prot) 40838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall{ 40938f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pud_t *pud; 41038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pmd_t *pmd; 41138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall unsigned long addr, next; 41238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall int ret; 41338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 41438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall addr = start; 41538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall do { 41638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pud = pud_offset(pgd, addr); 41738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 41838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (pud_none_or_clear_bad(pud)) { 41938f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pmd = pmd_alloc_one(NULL, addr); 42038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (!pmd) { 42138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall kvm_err("Cannot allocate Hyp pmd\n"); 42238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall return -ENOMEM; 42338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall } 42438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pud_populate(NULL, pud, pmd); 42538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall get_page(virt_to_page(pud)); 42638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall kvm_flush_dcache_to_poc(pud, sizeof(*pud)); 42738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall } 42838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 42938f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall next = pud_addr_end(addr, end); 43038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot); 43138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (ret) 43238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall return ret; 43338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pfn += (next - addr) >> PAGE_SHIFT; 43438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall } while (addr = next, addr != end); 43538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 43638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall return 0; 43738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall} 43838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 4396060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngierstatic int __create_hyp_mappings(pgd_t *pgdp, 4406060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long start, unsigned long end, 4416060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long pfn, pgprot_t prot) 442342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 443342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall pgd_t *pgd; 444342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall pud_t *pud; 445342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall unsigned long addr, next; 446342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall int err = 0; 447342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 448342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall mutex_lock(&kvm_hyp_pgd_mutex); 4493562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier addr = start & PAGE_MASK; 4503562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier end = PAGE_ALIGN(end); 4513562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier do { 4526060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pgd = pgdp + pgd_index(addr); 453342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 45438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (pgd_none(*pgd)) { 45538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pud = pud_alloc_one(NULL, addr); 45638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (!pud) { 45738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall kvm_err("Cannot allocate Hyp pud\n"); 458342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall err = -ENOMEM; 459342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall goto out; 460342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall } 46138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pgd_populate(NULL, pgd, pud); 46238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall get_page(virt_to_page(pgd)); 46338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall kvm_flush_dcache_to_poc(pgd, sizeof(*pgd)); 464342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall } 465342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 466342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall next = pgd_addr_end(addr, end); 46738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot); 468342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall if (err) 469342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall goto out; 4706060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier pfn += (next - addr) >> PAGE_SHIFT; 4713562c76dcb9ce84853c835eec12a911bf3a8e2daMarc Zyngier } while (addr = next, addr != end); 472342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallout: 473342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall mutex_unlock(&kvm_hyp_pgd_mutex); 474342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall return err; 475342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 476342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 47740c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dallstatic phys_addr_t kvm_kaddr_to_phys(void *kaddr) 47840c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall{ 47940c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall if (!is_vmalloc_addr(kaddr)) { 48040c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall BUG_ON(!virt_addr_valid(kaddr)); 48140c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall return __pa(kaddr); 48240c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall } else { 48340c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall return page_to_phys(vmalloc_to_page(kaddr)) + 48440c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall offset_in_page(kaddr); 48540c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall } 48640c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall} 48740c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall 488342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall/** 48906e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode 490342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * @from: The virtual kernel start address of the range 491342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * @to: The virtual kernel end address of the range (exclusive) 492342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall * 49306e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * The same virtual address as the kernel virtual address is also used 49406e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying 49506e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * physical pages. 496342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall */ 497342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallint create_hyp_mappings(void *from, void *to) 498342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 49940c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall phys_addr_t phys_addr; 50040c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall unsigned long virt_addr; 5016060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long start = KERN_TO_HYP((unsigned long)from); 5026060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long end = KERN_TO_HYP((unsigned long)to); 5036060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier 50440c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall start = start & PAGE_MASK; 50540c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall end = PAGE_ALIGN(end); 5066060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier 50740c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) { 50840c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall int err; 5096060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier 51040c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall phys_addr = kvm_kaddr_to_phys(from + virt_addr - start); 51140c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall err = __create_hyp_mappings(hyp_pgd, virt_addr, 51240c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall virt_addr + PAGE_SIZE, 51340c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall __phys_to_pfn(phys_addr), 51440c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall PAGE_HYP); 51540c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall if (err) 51640c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall return err; 51740c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall } 51840c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall 51940c2729bab48e2832b17c1fa8af9db60e776131bChristoffer Dall return 0; 520342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 521342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 522342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall/** 52306e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * create_hyp_io_mappings - duplicate a kernel IO mapping into Hyp mode 52406e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * @from: The kernel start VA of the range 52506e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * @to: The kernel end VA of the range (exclusive) 5266060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier * @phys_addr: The physical start address which gets mapped 52706e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * 52806e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * The resulting HYP VA is the same as the kernel VA, modulo 52906e8c3b0f3210e5e7039fd2b5e3926b68df7f5d7Marc Zyngier * HYP_PAGE_OFFSET. 530342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall */ 5316060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngierint create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr) 532342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 5336060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long start = KERN_TO_HYP((unsigned long)from); 5346060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier unsigned long end = KERN_TO_HYP((unsigned long)to); 5356060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier 5366060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier /* Check for a valid kernel IO mapping */ 5376060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier if (!is_vmalloc_addr(from) || !is_vmalloc_addr(to - 1)) 5386060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier return -EINVAL; 5396060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier 5406060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier return __create_hyp_mappings(hyp_pgd, start, end, 5416060df84cbe0d36b8e1415b68e3f67b77f27052aMarc Zyngier __phys_to_pfn(phys_addr), PAGE_HYP_DEVICE); 542342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 543342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 544d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall/** 545d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation. 546d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @kvm: The KVM struct pointer for the VM. 547d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * 548d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * Allocates the 1st level table only of size defined by S2_PGD_ORDER (can 549d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * support either full 40-bit input addresses or limited to 32-bit input 550d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * addresses). Clears the allocated pages. 551d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * 552d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * Note we don't need locking here as this is only called when the VM is 553d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * created, which can only be done once. 554d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall */ 555d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallint kvm_alloc_stage2_pgd(struct kvm *kvm) 556d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 55738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall int ret; 558d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pgd_t *pgd; 559d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 560d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (kvm->arch.pgd != NULL) { 561d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall kvm_err("kvm_arch already initialized?\n"); 562d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return -EINVAL; 563d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall } 564d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 56538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (KVM_PREALLOC_LEVEL > 0) { 56638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall /* 56738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall * Allocate fake pgd for the page table manipulation macros to 56838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall * work. This is not used by the hardware and we have no 56938f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall * alignment requirement for this allocation. 57038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall */ 57138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pgd = (pgd_t *)kmalloc(PTRS_PER_S2_PGD * sizeof(pgd_t), 57238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall GFP_KERNEL | __GFP_ZERO); 57338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall } else { 57438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall /* 57538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall * Allocate actual first-level Stage-2 page table used by the 57638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall * hardware for Stage-2 page table walks. 57738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall */ 57838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, S2_PGD_ORDER); 57938f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall } 58038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 581d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (!pgd) 582d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return -ENOMEM; 583d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 58438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall ret = kvm_prealloc_hwpgd(kvm, pgd); 58538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (ret) 58638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall goto out_err; 58738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 588c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier kvm_clean_pgd(pgd); 589d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall kvm->arch.pgd = pgd; 590d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 59138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dallout_err: 59238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (KVM_PREALLOC_LEVEL > 0) 59338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall kfree(pgd); 59438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall else 59538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall free_pages((unsigned long)pgd, S2_PGD_ORDER); 59638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall return ret; 597d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 598d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 599d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall/** 600d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * unmap_stage2_range -- Clear stage2 page table entries to unmap a range 601d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @kvm: The VM pointer 602d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @start: The intermediate physical base address of the range to unmap 603d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @size: The size of the area to unmap 604d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * 605d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * Clear a range of stage-2 mappings, lowering the various ref-counts. Must 606d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * be called while holding mmu_lock (unless for freeing the stage2 pgd before 607d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * destroying the VM), otherwise another faulting VCPU may come in and mess 608d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * with things behind our backs. 609d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall */ 610d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallstatic void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size) 611d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 612d4cb9df5d1f79950b34e78ec5d1b1b59d6e9c7b7Marc Zyngier unmap_range(kvm, kvm->arch.pgd, start, size); 613d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 614d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 615d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall/** 616d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * kvm_free_stage2_pgd - free all stage-2 tables 617d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @kvm: The KVM struct pointer for the VM. 618d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * 619d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all 620d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * underlying level-2 and level-3 tables before freeing the actual level-1 table 621d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * and setting the struct pointer to NULL. 622d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * 623d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * Note we don't need locking here as this is only called when the VM is 624d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * destroyed, which can only be done once. 625d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall */ 626d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallvoid kvm_free_stage2_pgd(struct kvm *kvm) 627d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 628d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (kvm->arch.pgd == NULL) 629d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return; 630d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 631d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE); 63238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall kvm_free_hwpgd(kvm); 63338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (KVM_PREALLOC_LEVEL > 0) 63438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall kfree(kvm->arch.pgd); 63538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall else 63638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall free_pages((unsigned long)kvm->arch.pgd, S2_PGD_ORDER); 637d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall kvm->arch.pgd = NULL; 638d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 639d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 64038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dallstatic pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, 641ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall phys_addr_t addr) 642d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 643d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pgd_t *pgd; 644d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pud_t *pud; 645d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 646d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pgd = kvm->arch.pgd + pgd_index(addr); 64738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (WARN_ON(pgd_none(*pgd))) { 64838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall if (!cache) 64938f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall return NULL; 65038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pud = mmu_memory_cache_alloc(cache); 65138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pgd_populate(NULL, pgd, pud); 65238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall get_page(virt_to_page(pgd)); 65338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall } 65438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 65538f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall return pud_offset(pgd, addr); 65638f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall} 65738f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 65838f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dallstatic pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, 65938f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall phys_addr_t addr) 66038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall{ 66138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pud_t *pud; 66238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pmd_t *pmd; 66338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall 66438f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall pud = stage2_get_pud(kvm, cache, addr); 665d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (pud_none(*pud)) { 666d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (!cache) 667ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall return NULL; 668d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pmd = mmu_memory_cache_alloc(cache); 669d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pud_populate(NULL, pud, pmd); 670d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall get_page(virt_to_page(pud)); 671c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier } 672c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier 673ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall return pmd_offset(pud, addr); 674ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall} 675ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 676ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dallstatic int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache 677ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall *cache, phys_addr_t addr, const pmd_t *new_pmd) 678ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall{ 679ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall pmd_t *pmd, old_pmd; 680ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 681ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall pmd = stage2_get_pmd(kvm, cache, addr); 682ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall VM_BUG_ON(!pmd); 683d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 684ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall /* 685ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * Mapping in huge pages should only happen through a fault. If a 686ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * page is merged into a transparent huge page, the individual 687ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * subpages of that huge page should be unmapped through MMU 688ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * notifiers before we get here. 689ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * 690ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * Merging of CompoundPages is not supported; they should become 691ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * splitting first, unmapped, merged, and mapped back in on-demand. 692ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall */ 693ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall VM_BUG_ON(pmd_present(*pmd) && pmd_pfn(*pmd) != pmd_pfn(*new_pmd)); 694ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 695ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall old_pmd = *pmd; 696ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall kvm_set_pmd(pmd, *new_pmd); 697ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall if (pmd_present(old_pmd)) 698ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall kvm_tlb_flush_vmid_ipa(kvm, addr); 699ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall else 700ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall get_page(virt_to_page(pmd)); 701ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall return 0; 702ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall} 703ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 704ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dallstatic int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, 705ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall phys_addr_t addr, const pte_t *new_pte, bool iomap) 706ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall{ 707ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall pmd_t *pmd; 708ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall pte_t *pte, old_pte; 709ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 71038f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall /* Create stage-2 page table mapping - Levels 0 and 1 */ 711ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall pmd = stage2_get_pmd(kvm, cache, addr); 712ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall if (!pmd) { 713ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall /* 714ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * Ignore calls from kvm_set_spte_hva for unallocated 715ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall * address ranges. 716ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall */ 717ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall return 0; 718ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall } 719ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 720ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall /* Create stage-2 page mappings - Level 2 */ 721d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (pmd_none(*pmd)) { 722d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (!cache) 723d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; /* ignore calls from kvm_set_spte_hva */ 724d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pte = mmu_memory_cache_alloc(cache); 725c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier kvm_clean_pte(pte); 726d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pmd_populate_kernel(NULL, pmd, pte); 727d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall get_page(virt_to_page(pmd)); 728c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier } 729c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier 730c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier pte = pte_offset_kernel(pmd, addr); 731d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 732d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (iomap && pte_present(*pte)) 733d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return -EFAULT; 734d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 735d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall /* Create 2nd stage page table mapping - Level 3 */ 736d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall old_pte = *pte; 737d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall kvm_set_pte(pte, *new_pte); 738d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (pte_present(old_pte)) 73948762767e1c150d58c250650f8202b7d4ad65ec4Marc Zyngier kvm_tlb_flush_vmid_ipa(kvm, addr); 740d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall else 741d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall get_page(virt_to_page(pte)); 742d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 743d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 744d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 745d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 746d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall/** 747d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * kvm_phys_addr_ioremap - map a device range to guest IPA 748d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * 749d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @kvm: The KVM pointer 750d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @guest_ipa: The IPA at which to insert the mapping 751d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @pa: The physical address of the device 752d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * @size: The size of the mapping 753d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall */ 754d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallint kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, 755c40f2f8ff833eddc02cb599ef6e5a162223449baArd Biesheuvel phys_addr_t pa, unsigned long size, bool writable) 756d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 757d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall phys_addr_t addr, end; 758d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall int ret = 0; 759d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unsigned long pfn; 760d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall struct kvm_mmu_memory_cache cache = { 0, }; 761d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 762d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK; 763d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pfn = __phys_to_pfn(pa); 764d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 765d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) { 766c62ee2b22798d7a5a8eb5f799b5183ef993ca7e4Marc Zyngier pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE); 767d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 768c40f2f8ff833eddc02cb599ef6e5a162223449baArd Biesheuvel if (writable) 769c40f2f8ff833eddc02cb599ef6e5a162223449baArd Biesheuvel kvm_set_s2pte_writable(&pte); 770c40f2f8ff833eddc02cb599ef6e5a162223449baArd Biesheuvel 77138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall ret = mmu_topup_memory_cache(&cache, KVM_MMU_CACHE_MIN_PAGES, 77238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall KVM_NR_MEM_OBJS); 773d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (ret) 774d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall goto out; 775d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall spin_lock(&kvm->mmu_lock); 776d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall ret = stage2_set_pte(kvm, &cache, addr, &pte, true); 777d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall spin_unlock(&kvm->mmu_lock); 778d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (ret) 779d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall goto out; 780d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 781d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pfn++; 782d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall } 783d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 784d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallout: 785d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall mmu_free_memory_cache(&cache); 786d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return ret; 787d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 788d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 7899b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dallstatic bool transparent_hugepage_adjust(pfn_t *pfnp, phys_addr_t *ipap) 7909b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall{ 7919b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall pfn_t pfn = *pfnp; 7929b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall gfn_t gfn = *ipap >> PAGE_SHIFT; 7939b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall 7949b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall if (PageTransCompound(pfn_to_page(pfn))) { 7959b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall unsigned long mask; 7969b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall /* 7979b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * The address we faulted on is backed by a transparent huge 7989b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * page. However, because we map the compound huge page and 7999b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * not the individual tail page, we need to transfer the 8009b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * refcount to the head page. We have to be careful that the 8019b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * THP doesn't start to split while we are adjusting the 8029b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * refcounts. 8039b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * 8049b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * We are sure this doesn't happen, because mmu_notifier_retry 8059b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * was successful and we are holding the mmu_lock, so if this 8069b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * THP is trying to split, it will be blocked in the mmu 8079b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * notifier before touching any of the pages, specifically 8089b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * before being able to call __split_huge_page_refcount(). 8099b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * 8109b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * We can therefore safely transfer the refcount from PG_tail 8119b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * to PG_head and switch the pfn from a tail page to the head 8129b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall * page accordingly. 8139b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall */ 8149b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall mask = PTRS_PER_PMD - 1; 8159b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall VM_BUG_ON((gfn & mask) != (pfn & mask)); 8169b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall if (pfn & mask) { 8179b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall *ipap &= PMD_MASK; 8189b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall kvm_release_pfn_clean(pfn); 8199b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall pfn &= ~mask; 8209b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall kvm_get_pfn(pfn); 8219b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall *pfnp = pfn; 8229b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall } 8239b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall 8249b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall return true; 8259b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall } 8269b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall 8279b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall return false; 8289b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall} 8299b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall 830a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvelstatic bool kvm_is_write_fault(struct kvm_vcpu *vcpu) 831a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel{ 832a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel if (kvm_vcpu_trap_is_iabt(vcpu)) 833a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel return false; 834a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel 835a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel return kvm_vcpu_dabt_iswrite(vcpu); 836a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel} 837a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel 838bb55e9b131d70ab9e30d73ab1342ad4907f9e0deArd Biesheuvelstatic bool kvm_is_device_pfn(unsigned long pfn) 839bb55e9b131d70ab9e30d73ab1342ad4907f9e0deArd Biesheuvel{ 840bb55e9b131d70ab9e30d73ab1342ad4907f9e0deArd Biesheuvel return !pfn_valid(pfn); 841bb55e9b131d70ab9e30d73ab1342ad4907f9e0deArd Biesheuvel} 842bb55e9b131d70ab9e30d73ab1342ad4907f9e0deArd Biesheuvel 84394f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dallstatic int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, 84498047888bb9fd57734028c44ec17413ddd623958Christoffer Dall struct kvm_memory_slot *memslot, unsigned long hva, 84594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall unsigned long fault_status) 84694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall{ 84794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall int ret; 8489b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall bool write_fault, writable, hugetlb = false, force_pte = false; 84994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall unsigned long mmu_seq; 850ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall gfn_t gfn = fault_ipa >> PAGE_SHIFT; 851ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall struct kvm *kvm = vcpu->kvm; 85294f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; 853ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall struct vm_area_struct *vma; 854ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall pfn_t pfn; 855b88657674d39fc2127d62d0de9ca142e166443c8Kim Phillips pgprot_t mem_type = PAGE_S2; 85694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 857a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel write_fault = kvm_is_write_fault(vcpu); 85894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall if (fault_status == FSC_PERM && !write_fault) { 85994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall kvm_err("Unexpected L2 read permission error\n"); 86094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall return -EFAULT; 86194f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall } 86294f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 863ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall /* Let's check if we will get back a huge page backed by hugetlbfs */ 864ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall down_read(¤t->mm->mmap_sem); 865ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall vma = find_vma_intersection(current->mm, hva, hva + 1); 86637b544087ef3f65ca68465ba39291a07195dac26Ard Biesheuvel if (unlikely(!vma)) { 86737b544087ef3f65ca68465ba39291a07195dac26Ard Biesheuvel kvm_err("Failed to find VMA for hva 0x%lx\n", hva); 86837b544087ef3f65ca68465ba39291a07195dac26Ard Biesheuvel up_read(¤t->mm->mmap_sem); 86937b544087ef3f65ca68465ba39291a07195dac26Ard Biesheuvel return -EFAULT; 87037b544087ef3f65ca68465ba39291a07195dac26Ard Biesheuvel } 87137b544087ef3f65ca68465ba39291a07195dac26Ard Biesheuvel 872ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall if (is_vm_hugetlb_page(vma)) { 873ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall hugetlb = true; 874ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT; 8759b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall } else { 8769b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall /* 877136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier * Pages belonging to memslots that don't have the same 878136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier * alignment for userspace and IPA cannot be mapped using 879136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier * block descriptors even if the pages belong to a THP for 880136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier * the process, because the stage-2 block descriptor will 881136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier * cover more than a single THP and we loose atomicity for 882136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier * unmapping, updates, and splits of the THP or other pages 883136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier * in the stage-2 block range. 8849b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall */ 885136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier if ((memslot->userspace_addr & ~PMD_MASK) != 886136d737fd20102f1be9b02356590fd55e3a40d0eMarc Zyngier ((memslot->base_gfn << PAGE_SHIFT) & ~PMD_MASK)) 8879b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall force_pte = true; 888ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall } 889ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall up_read(¤t->mm->mmap_sem); 890ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 89194f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall /* We need minimum second+third level pages */ 89238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, 89338f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall KVM_NR_MEM_OBJS); 89494f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall if (ret) 89594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall return ret; 89694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 89794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall mmu_seq = vcpu->kvm->mmu_notifier_seq; 89894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall /* 89994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * Ensure the read of mmu_notifier_seq happens before we call 90094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk 90194f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * the page we just got a reference to gets unmapped before we have a 90294f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * chance to grab the mmu_lock, which ensure that if the page gets 90394f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * unmapped afterwards, the call to kvm_unmap_hva will take it away 90494f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * from us again properly. This smp_rmb() interacts with the smp_wmb() 90594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * in kvm_mmu_notifier_invalidate_<page|range_end>. 90694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall */ 90794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall smp_rmb(); 90894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 909ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); 91094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall if (is_error_pfn(pfn)) 91194f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall return -EFAULT; 91294f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 913bb55e9b131d70ab9e30d73ab1342ad4907f9e0deArd Biesheuvel if (kvm_is_device_pfn(pfn)) 914b88657674d39fc2127d62d0de9ca142e166443c8Kim Phillips mem_type = PAGE_S2_DEVICE; 915b88657674d39fc2127d62d0de9ca142e166443c8Kim Phillips 916ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall spin_lock(&kvm->mmu_lock); 917ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall if (mmu_notifier_retry(kvm, mmu_seq)) 91894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall goto out_unlock; 9199b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall if (!hugetlb && !force_pte) 9209b5fdb9781f74fb15827e465bfb5aa63211953c8Christoffer Dall hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa); 921ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 922ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall if (hugetlb) { 923b88657674d39fc2127d62d0de9ca142e166443c8Kim Phillips pmd_t new_pmd = pfn_pmd(pfn, mem_type); 924ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall new_pmd = pmd_mkhuge(new_pmd); 925ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall if (writable) { 926ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall kvm_set_s2pmd_writable(&new_pmd); 927ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall kvm_set_pfn_dirty(pfn); 928ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall } 9292d58b733c87689d3d5144e4ac94ea861cc729145Marc Zyngier coherent_cache_guest_page(vcpu, hva & PMD_MASK, PMD_SIZE); 930ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd); 931ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall } else { 932b88657674d39fc2127d62d0de9ca142e166443c8Kim Phillips pte_t new_pte = pfn_pte(pfn, mem_type); 933ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall if (writable) { 934ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall kvm_set_s2pte_writable(&new_pte); 935ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall kvm_set_pfn_dirty(pfn); 936ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall } 9372d58b733c87689d3d5144e4ac94ea861cc729145Marc Zyngier coherent_cache_guest_page(vcpu, hva, PAGE_SIZE); 938b88657674d39fc2127d62d0de9ca142e166443c8Kim Phillips ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, 9393d08c629244257473450a8ba17cb8184b91e68f8Steve Capper pgprot_val(mem_type) == pgprot_val(PAGE_S2_DEVICE)); 94094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall } 941ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall 94294f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 94394f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dallout_unlock: 944ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall spin_unlock(&kvm->mmu_lock); 94594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall kvm_release_pfn_clean(pfn); 946ad361f093c1e31d0b43946210a32ab4ff5c49850Christoffer Dall return ret; 94794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall} 94894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 94994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall/** 95094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * kvm_handle_guest_abort - handles all 2nd stage aborts 95194f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * @vcpu: the VCPU pointer 95294f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * @run: the kvm_run structure 95394f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * 95494f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * Any abort that gets to the host is almost guaranteed to be caused by a 95594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * missing second stage translation table entry, which can mean that either the 95694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * guest simply needs more memory and we must allocate an appropriate page or it 95794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * can mean that the guest tried to access I/O memory, which is emulated by user 95894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * space. The distinction is based on the IPA causing the fault and whether this 95994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall * memory region has been registered as standard RAM by user space. 96094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall */ 961342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallint kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) 962342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 96394f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall unsigned long fault_status; 96494f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall phys_addr_t fault_ipa; 96594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall struct kvm_memory_slot *memslot; 96698047888bb9fd57734028c44ec17413ddd623958Christoffer Dall unsigned long hva; 96798047888bb9fd57734028c44ec17413ddd623958Christoffer Dall bool is_iabt, write_fault, writable; 96894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall gfn_t gfn; 96994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall int ret, idx; 97094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 97152d1dba933f601d8d9e6373377377b12d6bcfac0Marc Zyngier is_iabt = kvm_vcpu_trap_is_iabt(vcpu); 9727393b599177d301d4c9ca2c7f69a6849aba793c7Marc Zyngier fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); 97394f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 9747393b599177d301d4c9ca2c7f69a6849aba793c7Marc Zyngier trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu), 9757393b599177d301d4c9ca2c7f69a6849aba793c7Marc Zyngier kvm_vcpu_get_hfar(vcpu), fault_ipa); 97694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 97794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall /* Check the stage-2 fault is trans. fault or write fault */ 9780496daa5cf99741ce8db82686b4c7446a37feabbChristoffer Dall fault_status = kvm_vcpu_trap_get_fault_type(vcpu); 97994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall if (fault_status != FSC_FAULT && fault_status != FSC_PERM) { 9800496daa5cf99741ce8db82686b4c7446a37feabbChristoffer Dall kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", 9810496daa5cf99741ce8db82686b4c7446a37feabbChristoffer Dall kvm_vcpu_trap_get_class(vcpu), 9820496daa5cf99741ce8db82686b4c7446a37feabbChristoffer Dall (unsigned long)kvm_vcpu_trap_get_fault(vcpu), 9830496daa5cf99741ce8db82686b4c7446a37feabbChristoffer Dall (unsigned long)kvm_vcpu_get_hsr(vcpu)); 98494f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall return -EFAULT; 98594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall } 98694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 98794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall idx = srcu_read_lock(&vcpu->kvm->srcu); 98894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 98994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall gfn = fault_ipa >> PAGE_SHIFT; 99098047888bb9fd57734028c44ec17413ddd623958Christoffer Dall memslot = gfn_to_memslot(vcpu->kvm, gfn); 99198047888bb9fd57734028c44ec17413ddd623958Christoffer Dall hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); 992a7d079cea2dffb112e26da2566dd84c0ef1fce97Ard Biesheuvel write_fault = kvm_is_write_fault(vcpu); 99398047888bb9fd57734028c44ec17413ddd623958Christoffer Dall if (kvm_is_error_hva(hva) || (write_fault && !writable)) { 99494f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall if (is_iabt) { 99594f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall /* Prefetch Abort on I/O address */ 9967393b599177d301d4c9ca2c7f69a6849aba793c7Marc Zyngier kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu)); 99794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall ret = 1; 99894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall goto out_unlock; 99994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall } 100094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 1001cfe3950c2a19c1e1ad85b9dd2622617e309d2845Marc Zyngier /* 1002cfe3950c2a19c1e1ad85b9dd2622617e309d2845Marc Zyngier * The IPA is reported as [MAX:12], so we need to 1003cfe3950c2a19c1e1ad85b9dd2622617e309d2845Marc Zyngier * complement it with the bottom 12 bits from the 1004cfe3950c2a19c1e1ad85b9dd2622617e309d2845Marc Zyngier * faulting VA. This is always 12 bits, irrespective 1005cfe3950c2a19c1e1ad85b9dd2622617e309d2845Marc Zyngier * of the page size. 1006cfe3950c2a19c1e1ad85b9dd2622617e309d2845Marc Zyngier */ 1007cfe3950c2a19c1e1ad85b9dd2622617e309d2845Marc Zyngier fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1); 100845e96ea6b369539a37040a8df9c59a39f073d9d6Christoffer Dall ret = io_mem_abort(vcpu, run, fault_ipa); 100994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall goto out_unlock; 101094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall } 101194f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall 1012c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall /* Userspace should not be able to register out-of-bounds IPAs */ 1013c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE); 1014c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall 101598047888bb9fd57734028c44ec17413ddd623958Christoffer Dall ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status); 101694f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall if (ret == 0) 101794f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall ret = 1; 101894f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dallout_unlock: 101994f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall srcu_read_unlock(&vcpu->kvm->srcu, idx); 102094f8e6418d3915dbefbb5d66b63146f1df12b0c0Christoffer Dall return ret; 1021342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 1022342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 1023d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallstatic void handle_hva_to_gpa(struct kvm *kvm, 1024d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unsigned long start, 1025d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unsigned long end, 1026d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall void (*handler)(struct kvm *kvm, 1027d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall gpa_t gpa, void *data), 1028d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall void *data) 1029d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 1030d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall struct kvm_memslots *slots; 1031d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall struct kvm_memory_slot *memslot; 1032d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1033d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall slots = kvm_memslots(kvm); 1034d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1035d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall /* we only care about the pages that the guest sees */ 1036d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall kvm_for_each_memslot(memslot, slots) { 1037d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unsigned long hva_start, hva_end; 1038d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall gfn_t gfn, gfn_end; 1039d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1040d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall hva_start = max(start, memslot->userspace_addr); 1041d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall hva_end = min(end, memslot->userspace_addr + 1042d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall (memslot->npages << PAGE_SHIFT)); 1043d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (hva_start >= hva_end) 1044d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall continue; 1045d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1046d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall /* 1047d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * {gfn(page) | page intersects with [hva_start, hva_end)} = 1048d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall * {gfn_start, gfn_start+1, ..., gfn_end-1}. 1049d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall */ 1050d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall gfn = hva_to_gfn_memslot(hva_start, memslot); 1051d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); 1052d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1053d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall for (; gfn < gfn_end; ++gfn) { 1054d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall gpa_t gpa = gfn << PAGE_SHIFT; 1055d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall handler(kvm, gpa, data); 1056d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall } 1057d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall } 1058d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 1059d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1060d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallstatic void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data) 1061d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 1062d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unmap_stage2_range(kvm, gpa, PAGE_SIZE); 1063d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 1064d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1065d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallint kvm_unmap_hva(struct kvm *kvm, unsigned long hva) 1066d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 1067d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unsigned long end = hva + PAGE_SIZE; 1068d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1069d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (!kvm->arch.pgd) 1070d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 1071d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1072d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall trace_kvm_unmap_hva(hva); 1073d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL); 1074d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 1075d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 1076d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1077d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallint kvm_unmap_hva_range(struct kvm *kvm, 1078d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unsigned long start, unsigned long end) 1079d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 1080d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (!kvm->arch.pgd) 1081d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 1082d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1083d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall trace_kvm_unmap_hva_range(start, end); 1084d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); 1085d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 1086d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 1087d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1088d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallstatic void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data) 1089d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 1090d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pte_t *pte = (pte_t *)data; 1091d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1092d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall stage2_set_pte(kvm, NULL, gpa, pte, false); 1093d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 1094d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1095d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1096d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallvoid kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) 1097d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 1098d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall unsigned long end = hva + PAGE_SIZE; 1099d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall pte_t stage2_pte; 1100d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1101d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall if (!kvm->arch.pgd) 1102d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return; 1103d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1104d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall trace_kvm_set_spte_hva(hva); 1105d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2); 1106d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte); 1107d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 1108d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1109d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dallvoid kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) 1110d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall{ 1111d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 1112d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall} 1113d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 1114342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallphys_addr_t kvm_mmu_get_httbr(void) 1115342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 1116342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall return virt_to_phys(hyp_pgd); 1117342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 1118342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall 11195a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngierphys_addr_t kvm_mmu_get_boot_httbr(void) 11205a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier{ 11215a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier return virt_to_phys(boot_hyp_pgd); 11225a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier} 11235a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 11245a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngierphys_addr_t kvm_get_idmap_vector(void) 11255a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier{ 11265a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier return hyp_idmap_vector; 11275a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier} 11285a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 1129342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dallint kvm_mmu_init(void) 1130342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall{ 11312fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier int err; 11322fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier 11334fda342cc7f577599c53fd27b99c953c7b1da18aSantosh Shilimkar hyp_idmap_start = kvm_virt_to_phys(__hyp_idmap_text_start); 11344fda342cc7f577599c53fd27b99c953c7b1da18aSantosh Shilimkar hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end); 11354fda342cc7f577599c53fd27b99c953c7b1da18aSantosh Shilimkar hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init); 11365a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 11375a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier if ((hyp_idmap_start ^ hyp_idmap_end) & PAGE_MASK) { 11385a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier /* 11395a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * Our init code is crossing a page boundary. Allocate 11405a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * a bounce page, copy the code over and use that. 11415a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier */ 11425a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier size_t len = __hyp_idmap_text_end - __hyp_idmap_text_start; 11435a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier phys_addr_t phys_base; 11445a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 11455d4e08c45a6cf8f1ab3c7fa375007635ac569165Mark Salter init_bounce_page = (void *)__get_free_page(GFP_KERNEL); 11465a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier if (!init_bounce_page) { 11475a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier kvm_err("Couldn't allocate HYP init bounce page\n"); 11485a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier err = -ENOMEM; 11495a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier goto out; 11505a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier } 11515a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 11525a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier memcpy(init_bounce_page, __hyp_idmap_text_start, len); 11535a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier /* 11545a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * Warning: the code we just copied to the bounce page 11555a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * must be flushed to the point of coherency. 11565a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * Otherwise, the data may be sitting in L2, and HYP 11575a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * mode won't be able to observe it as it runs with 11585a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier * caches off at that point. 11595a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier */ 11605a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier kvm_flush_dcache_to_poc(init_bounce_page, len); 11615a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 11624fda342cc7f577599c53fd27b99c953c7b1da18aSantosh Shilimkar phys_base = kvm_virt_to_phys(init_bounce_page); 11635a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier hyp_idmap_vector += phys_base - hyp_idmap_start; 11645a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier hyp_idmap_start = phys_base; 11655a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier hyp_idmap_end = phys_base + len; 11665a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 11675a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier kvm_info("Using HYP init bounce page @%lx\n", 11685a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier (unsigned long)phys_base); 11695a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier } 11705a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 117138f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); 117238f791a4e499792eeb2a3c0572dd5133511c5bbbChristoffer Dall boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); 11735d4e08c45a6cf8f1ab3c7fa375007635ac569165Mark Salter 11745a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier if (!hyp_pgd || !boot_hyp_pgd) { 1175d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall kvm_err("Hyp mode PGD not allocated\n"); 11762fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier err = -ENOMEM; 11772fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier goto out; 11782fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier } 11792fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier 11802fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier /* Create the idmap in the boot page tables */ 11812fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier err = __create_hyp_mappings(boot_hyp_pgd, 11822fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier hyp_idmap_start, hyp_idmap_end, 11832fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier __phys_to_pfn(hyp_idmap_start), 11842fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier PAGE_HYP); 11852fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier 11862fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier if (err) { 11872fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier kvm_err("Failed to idmap %lx-%lx\n", 11882fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier hyp_idmap_start, hyp_idmap_end); 11892fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier goto out; 1190d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall } 1191d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall 11925a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier /* Map the very same page at the trampoline VA */ 11935a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier err = __create_hyp_mappings(boot_hyp_pgd, 11945a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE, 11955a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier __phys_to_pfn(hyp_idmap_start), 11965a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier PAGE_HYP); 11975a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier if (err) { 11985a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier kvm_err("Failed to map trampoline @%lx into boot HYP pgd\n", 11995a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier TRAMPOLINE_VA); 12005a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier goto out; 12015a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier } 12025a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 12035a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier /* Map the same page again into the runtime page tables */ 12045a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier err = __create_hyp_mappings(hyp_pgd, 12055a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE, 12065a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier __phys_to_pfn(hyp_idmap_start), 12075a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier PAGE_HYP); 12085a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier if (err) { 12095a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier kvm_err("Failed to map trampoline @%lx into runtime HYP pgd\n", 12105a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier TRAMPOLINE_VA); 12115a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier goto out; 12125a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier } 12135a677ce044f18a341ab942e23516e52ad89f7687Marc Zyngier 1214d5d8184d35c990b1324d9b30bcd0e4e8aa08f56dChristoffer Dall return 0; 12152fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngierout: 12164f728276fbf1e043010485d7e9275082a1c3d650Marc Zyngier free_hyp_pgds(); 12172fb410596ca917fe6419be61ee5bc1782b17d947Marc Zyngier return err; 1218342cd0ab0e6ca3fe7c88a78890352748b8e894a9Christoffer Dall} 1219df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger 1220df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Augervoid kvm_arch_commit_memory_region(struct kvm *kvm, 1221df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger struct kvm_userspace_memory_region *mem, 1222df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger const struct kvm_memory_slot *old, 1223df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger enum kvm_mr_change change) 1224df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger{ 1225df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger} 1226df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger 1227df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Augerint kvm_arch_prepare_memory_region(struct kvm *kvm, 1228df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger struct kvm_memory_slot *memslot, 1229df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger struct kvm_userspace_memory_region *mem, 1230df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger enum kvm_mr_change change) 1231df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger{ 12328eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel hva_t hva = mem->userspace_addr; 12338eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel hva_t reg_end = hva + mem->memory_size; 12348eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel bool writable = !(mem->flags & KVM_MEM_READONLY); 12358eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel int ret = 0; 12368eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12378eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel if (change != KVM_MR_CREATE && change != KVM_MR_MOVE) 12388eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel return 0; 12398eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12408eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel /* 1241c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall * Prevent userspace from creating a memory region outside of the IPA 1242c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall * space addressable by the KVM guest IPA space. 1243c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall */ 1244c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall if (memslot->base_gfn + memslot->npages >= 1245c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall (KVM_PHYS_SIZE >> PAGE_SHIFT)) 1246c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall return -EFAULT; 1247c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall 1248c3058d5da2222629bc2223c488a4512b59bb4bafChristoffer Dall /* 12498eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * A memory region could potentially cover multiple VMAs, and any holes 12508eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * between them, so iterate over all of them to find out if we can map 12518eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * any of them right now. 12528eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * 12538eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * +--------------------------------------------+ 12548eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * +---------------+----------------+ +----------------+ 12558eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * | : VMA 1 | VMA 2 | | VMA 3 : | 12568eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * +---------------+----------------+ +----------------+ 12578eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * | memory region | 12588eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * +--------------------------------------------+ 12598eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel */ 12608eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel do { 12618eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel struct vm_area_struct *vma = find_vma(current->mm, hva); 12628eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel hva_t vm_start, vm_end; 12638eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12648eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel if (!vma || vma->vm_start >= reg_end) 12658eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel break; 12668eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12678eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel /* 12688eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * Mapping a read-only VMA is only allowed if the 12698eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * memory region is configured as read-only. 12708eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel */ 12718eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel if (writable && !(vma->vm_flags & VM_WRITE)) { 12728eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel ret = -EPERM; 12738eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel break; 12748eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel } 12758eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12768eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel /* 12778eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel * Take the intersection of this VMA with the memory region 12788eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel */ 12798eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel vm_start = max(hva, vma->vm_start); 12808eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel vm_end = min(reg_end, vma->vm_end); 12818eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12828eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel if (vma->vm_flags & VM_PFNMAP) { 12838eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel gpa_t gpa = mem->guest_phys_addr + 12848eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel (vm_start - mem->userspace_addr); 12858eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel phys_addr_t pa = (vma->vm_pgoff << PAGE_SHIFT) + 12868eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel vm_start - vma->vm_start; 12878eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12888eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel ret = kvm_phys_addr_ioremap(kvm, gpa, pa, 12898eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel vm_end - vm_start, 12908eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel writable); 12918eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel if (ret) 12928eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel break; 12938eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel } 12948eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel hva = vm_end; 12958eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel } while (hva < reg_end); 12968eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 12978eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel if (ret) { 12988eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel spin_lock(&kvm->mmu_lock); 12998eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size); 13008eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel spin_unlock(&kvm->mmu_lock); 13018eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel } 13028eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel return ret; 1303df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger} 1304df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger 1305df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Augervoid kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 1306df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger struct kvm_memory_slot *dont) 1307df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger{ 1308df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger} 1309df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger 1310df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Augerint kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 1311df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger unsigned long npages) 1312df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger{ 1313df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger return 0; 1314df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger} 1315df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger 1316df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Augervoid kvm_arch_memslots_updated(struct kvm *kvm) 1317df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger{ 1318df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger} 1319df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger 1320df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Augervoid kvm_arch_flush_shadow_all(struct kvm *kvm) 1321df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger{ 1322df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger} 1323df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger 1324df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Augervoid kvm_arch_flush_shadow_memslot(struct kvm *kvm, 1325df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger struct kvm_memory_slot *slot) 1326df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger{ 13278eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel gpa_t gpa = slot->base_gfn << PAGE_SHIFT; 13288eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel phys_addr_t size = slot->npages << PAGE_SHIFT; 13298eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel 13308eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel spin_lock(&kvm->mmu_lock); 13318eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel unmap_stage2_range(kvm, gpa, size); 13328eef91239e57d2e932e7470879c9a504d5494ebbArd Biesheuvel spin_unlock(&kvm->mmu_lock); 1333df6ce24f2ee485c4f9a5cb610063a5eb60da8267Eric Auger} 1334