1b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge/* 2b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * Xen leaves the responsibility for maintaining p2m mappings to the 3b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * guests themselves, but it must also access and update the p2m array 4b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * during suspend/resume when all the pages are reallocated. 5b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * 6b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * The p2m table is logically a flat array, but we implement it as a 7b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * three-level tree to allow the address space to be sparse. 8b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * 9b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * Xen 10b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * | 11b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * p2m_top p2m_top_mfn 12b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * / \ / \ 13b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * p2m_mid p2m_mid p2m_mid_mfn p2m_mid_mfn 14b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * / \ / \ / / 15b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * p2m p2m p2m p2m p2m p2m p2m ... 16b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * 17b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * The p2m_mid_mfn pages are mapped by p2m_top_mfn_p. 18b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * 19b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * The p2m_top and p2m_top_mfn levels are limited to 1 page, so the 20b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * maximum representable pseudo-physical address space is: 21b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE pages 22b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * 23b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * P2M_PER_PAGE depends on the architecture, as a mfn is always 24b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to 25a3118beb6a8cbe77ae3342125d920205871b0717Konrad Rzeszutek Wilk * 512 and 1024 entries respectively. 26f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 27f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * In short, these structures contain the Machine Frame Number (MFN) of the PFN. 28f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 29f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * However not all entries are filled with MFNs. Specifically for all other 30f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * leaf entries, or for the top root, or middle one, for which there is a void 31f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * entry, we assume it is "missing". So (for example) 32f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * pfn_to_mfn(0x90909090)=INVALID_P2M_ENTRY. 33f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 34f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * We also have the possibility of setting 1-1 mappings on certain regions, so 35f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * that: 36f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * pfn_to_mfn(0xc0000)=0xc0000 37f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 38f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * The benefit of this is, that we can assume for non-RAM regions (think 393cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * PCI BARs, or ACPI spaces), we can create mappings easily because we 40f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * get the PFN value to match the MFN. 41f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 42f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * For this to work efficiently we have one new page p2m_identity and 43f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * allocate (via reserved_brk) any other pages we need to cover the sides 44f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * (1GB or 4MB boundary violations). All entries in p2m_identity are set to 45f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * INVALID_P2M_ENTRY type (Xen toolstack only recognizes that and MFNs, 46f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * no other fancy value). 47f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 48f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * On lookup we spot that the entry points to p2m_identity and return the 49f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * identity value instead of dereferencing and returning INVALID_P2M_ENTRY. 50f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * If the entry points to an allocated page, we just proceed as before and 51f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * return the PFN. If the PFN has IDENTITY_FRAME_BIT set we unmask that in 52f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * appropriate functions (pfn_to_mfn). 53f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 54f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * The reason for having the IDENTITY_FRAME_BIT instead of just returning the 55f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * PFN is that we could find ourselves where pfn_to_mfn(pfn)==pfn for a 56f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * non-identity pfn. To protect ourselves against we elect to set (and get) the 57f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * IDENTITY_FRAME_BIT on all identity mapped PFNs. 58f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 59f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * This simplistic diagram is used to explain the more subtle piece of code. 60f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * There is also a digram of the P2M at the end that can help. 61f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * Imagine your E820 looking as so: 62f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 633cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * 1GB 2GB 4GB 64f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * /-------------------+---------\/----\ /----------\ /---+-----\ 65f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * | System RAM | Sys RAM ||ACPI| | reserved | | Sys RAM | 66f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * \-------------------+---------/\----/ \----------/ \---+-----/ 67f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * ^- 1029MB ^- 2001MB 68f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 69f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * [1029MB = 263424 (0x40500), 2001MB = 512256 (0x7D100), 70f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 2048MB = 524288 (0x80000)] 71f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 72f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * And dom0_mem=max:3GB,1GB is passed in to the guest, meaning memory past 1GB 73f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * is actually not present (would have to kick the balloon driver to put it in). 74f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 75f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * When we are told to set the PFNs for identity mapping (see patch: "xen/setup: 76f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * Set identity mapping for non-RAM E820 and E820 gaps.") we pass in the start 77f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * of the PFN and the end PFN (263424 and 512256 respectively). The first step 78f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * is to reserve_brk a top leaf page if the p2m[1] is missing. The top leaf page 79f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * covers 512^2 of page estate (1GB) and in case the start or end PFN is not 803cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * aligned on 512^2*PAGE_SIZE (1GB) we reserve_brk new middle and leaf pages as 813cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * required to split any existing p2m_mid_missing middle pages. 82f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 83f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * With the E820 example above, 263424 is not 1GB aligned so we allocate a 84f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * reserve_brk page which will cover the PFNs estate from 0x40000 to 0x80000. 85f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * Each entry in the allocate page is "missing" (points to p2m_missing). 86f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 87f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * Next stage is to determine if we need to do a more granular boundary check 88f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * on the 4MB (or 2MB depending on architecture) off the start and end pfn's. 89f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * We check if the start pfn and end pfn violate that boundary check, and if 903cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * so reserve_brk a (p2m[x][y]) leaf page. This way we have a much finer 91f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * granularity of setting which PFNs are missing and which ones are identity. 92f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * In our example 263424 and 512256 both fail the check so we reserve_brk two 93f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * pages. Populate them with INVALID_P2M_ENTRY (so they both have "missing" 94f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * values) and assign them to p2m[1][2] and p2m[1][488] respectively. 95f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 96f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * At this point we would at minimum reserve_brk one page, but could be up to 97f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * three. Each call to set_phys_range_identity has at maximum a three page 98f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * cost. If we were to query the P2M at this stage, all those entries from 99f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * start PFN through end PFN (so 1029MB -> 2001MB) would return 100f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * INVALID_P2M_ENTRY ("missing"). 101f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 102f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * The next step is to walk from the start pfn to the end pfn setting 103f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * the IDENTITY_FRAME_BIT on each PFN. This is done in set_phys_range_identity. 1043cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * If we find that the middle entry is pointing to p2m_missing we can swap it 1053cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * over to p2m_identity - this way covering 4MB (or 2MB) PFN space (and 1063cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * similarly swapping p2m_mid_missing for p2m_mid_identity for larger regions). 1073cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * At this point we do not need to worry about boundary aligment (so no need to 108f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * reserve_brk a middle page, figure out which PFNs are "missing" and which 109f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * ones are identity), as that has been done earlier. If we find that the 110f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * middle leaf is not occupied by p2m_identity or p2m_missing, we dereference 111f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * that page (which covers 512 PFNs) and set the appropriate PFN with 112f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * IDENTITY_FRAME_BIT. In our example 263424 and 512256 end up there, and we 113f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * set from p2m[1][2][256->511] and p2m[1][488][0->256] with 114f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * IDENTITY_FRAME_BIT set. 115f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 116f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * All other regions that are void (or not filled) either point to p2m_missing 117f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * (considered missing) or have the default value of INVALID_P2M_ENTRY (also 118f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * considered missing). In our case, p2m[1][2][0->255] and p2m[1][488][257->511] 119f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * contain the INVALID_P2M_ENTRY value and are considered "missing." 120f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 1213cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * Finally, the region beyond the end of of the E820 (4 GB in this example) 1223cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * is set to be identity (in case there are MMIO regions placed here). 1233cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * 124f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * This is what the p2m ends up looking (for the E820 above) with this 125f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * fabulous drawing: 126f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 127f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * p2m /--------------\ 128f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * /-----\ | &mfn_list[0],| /-----------------\ 129f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * | 0 |------>| &mfn_list[1],| /---------------\ | ~0, ~0, .. | 130f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * |-----| | ..., ~0, ~0 | | ~0, ~0, [x]---+----->| IDENTITY [@256] | 131f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * | 1 |---\ \--------------/ | [p2m_identity]+\ | IDENTITY [@257] | 132f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * |-----| \ | [p2m_identity]+\\ | .... | 133f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * | 2 |--\ \-------------------->| ... | \\ \----------------/ 134f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * |-----| \ \---------------/ \\ 1353cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | 3 |-\ \ \\ p2m_identity [1] 1363cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * |-----| \ \-------------------->/---------------\ /-----------------\ 1373cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | .. |\ | | [p2m_identity]+-->| ~0, ~0, ~0, ... | 1383cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * \-----/ | | | [p2m_identity]+-->| ..., ~0 | 1393cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | | | .... | \-----------------/ 1403cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | | +-[x], ~0, ~0.. +\ 1413cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | | \---------------/ \ 1423cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | | \-> /---------------\ 1433cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | V p2m_mid_missing p2m_missing | IDENTITY[@0] | 1443cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | /-----------------\ /------------\ | IDENTITY[@256]| 1453cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | | [p2m_missing] +---->| ~0, ~0, ...| | ~0, ~0, .... | 1463cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | | [p2m_missing] +---->| ..., ~0 | \---------------/ 1473cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | | ... | \------------/ 1483cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | \-----------------/ 1493cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | 1503cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | p2m_mid_identity 1513cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | /-----------------\ 1523cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * \-->| [p2m_identity] +---->[1] 1533cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | [p2m_identity] +---->[1] 1543cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * | ... | 1553cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * \-----------------/ 156f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * 157f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * where ~0 is INVALID_P2M_ENTRY. IDENTITY is (PFN | IDENTITY_BIT) 158b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge */ 159b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 160b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include <linux/init.h> 161b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include <linux/module.h> 162448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge#include <linux/list.h> 163448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge#include <linux/hash.h> 16487f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge#include <linux/sched.h> 1652222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#include <linux/seq_file.h> 1662c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross#include <linux/bootmem.h> 167b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 168b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include <asm/cache.h> 169b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include <asm/setup.h> 170b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 171b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include <asm/xen/page.h> 172b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include <asm/xen/hypercall.h> 173b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include <asm/xen/hypervisor.h> 174ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini#include <xen/balloon.h> 1750930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini#include <xen/grant_table.h> 176b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 1774fbb67e3c87b806ad54445a1b4a9c6bde2359c98Matt Rushton#include "p2m.h" 1780930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini#include "multicalls.h" 179b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge#include "xen-ops.h" 180b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 181448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardingestatic void __init m2p_override_init(void); 182448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 183b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingeunsigned long xen_max_p2m_pfn __read_mostly; 184b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 1852c185687ab016954557aac80074f5d7f7f5d275cJuergen Grossstatic unsigned long *p2m_mid_missing_mfn; 1862c185687ab016954557aac80074f5d7f7f5d275cJuergen Grossstatic unsigned long *p2m_top_mfn; 1872c185687ab016954557aac80074f5d7f7f5d275cJuergen Grossstatic unsigned long **p2m_top_mfn_p; 1882c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross 189b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge/* Placeholders for holes in the address space */ 190b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE); 191b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE); 192b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 193b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic RESERVE_BRK_ARRAY(unsigned long **, p2m_top, P2M_TOP_PER_PAGE); 194b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 195f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilkstatic RESERVE_BRK_ARRAY(unsigned long, p2m_identity, P2M_PER_PAGE); 1963cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabelstatic RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_identity, P2M_MID_PER_PAGE); 197f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 198b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy FitzhardingeRESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE))); 199b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 2004fbb67e3c87b806ad54445a1b4a9c6bde2359c98Matt Rushton/* For each I/O range remapped we may lose up to two leaf pages for the boundary 2014fbb67e3c87b806ad54445a1b4a9c6bde2359c98Matt Rushton * violations and three mid pages to cover up to 3GB. With 2024fbb67e3c87b806ad54445a1b4a9c6bde2359c98Matt Rushton * early_can_reuse_p2m_middle() most of the leaf pages will be reused by the 2034fbb67e3c87b806ad54445a1b4a9c6bde2359c98Matt Rushton * remapped region. 2044fbb67e3c87b806ad54445a1b4a9c6bde2359c98Matt Rushton */ 2054fbb67e3c87b806ad54445a1b4a9c6bde2359c98Matt RushtonRESERVE_BRK(p2m_identity_remap, PAGE_SIZE * 2 * 3 * MAX_REMAP_RANGES); 206250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 207b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic inline unsigned p2m_top_index(unsigned long pfn) 208b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 209b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge BUG_ON(pfn >= MAX_P2M_PFN); 210b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE); 211b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 212b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 213b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic inline unsigned p2m_mid_index(unsigned long pfn) 214b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 215b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE; 216b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 217b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 218b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic inline unsigned p2m_index(unsigned long pfn) 219b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 220b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return pfn % P2M_PER_PAGE; 221b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 222b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 223b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic void p2m_top_init(unsigned long ***top) 224b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 225b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned i; 226b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 227b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (i = 0; i < P2M_TOP_PER_PAGE; i++) 228b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge top[i] = p2m_mid_missing; 229b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 230b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 231b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic void p2m_top_mfn_init(unsigned long *top) 232b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 233b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned i; 234b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 235b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (i = 0; i < P2M_TOP_PER_PAGE; i++) 236b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge top[i] = virt_to_mfn(p2m_mid_missing_mfn); 237b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 238b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 239b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic void p2m_top_mfn_p_init(unsigned long **top) 240b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 241b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned i; 242b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 243b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (i = 0; i < P2M_TOP_PER_PAGE; i++) 244b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge top[i] = p2m_mid_missing_mfn; 245b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 246b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 2473cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabelstatic void p2m_mid_init(unsigned long **mid, unsigned long *leaf) 248b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 249b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned i; 250b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 251b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (i = 0; i < P2M_MID_PER_PAGE; i++) 2523cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel mid[i] = leaf; 253b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 254b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 2553cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabelstatic void p2m_mid_mfn_init(unsigned long *mid, unsigned long *leaf) 256b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 257b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned i; 258b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 259b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (i = 0; i < P2M_MID_PER_PAGE; i++) 2603cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel mid[i] = virt_to_mfn(leaf); 261b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 262b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 263b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic void p2m_init(unsigned long *p2m) 264b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 265b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned i; 266b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 267b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (i = 0; i < P2M_MID_PER_PAGE; i++) 268b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m[i] = INVALID_P2M_ENTRY; 269b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 270b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 271b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge/* 272b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * Build the parallel p2m_top_mfn and p2m_mid_mfn structures 273b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * 274b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * This is called both at boot time, and after resuming from suspend: 2752c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross * - At boot time we're called rather early, and must use alloc_bootmem*() 276b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * to allocate memory. 277b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * 278b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * - After resume we're called from within stop_machine, but the mfn 2792c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross * tree should already be completely allocated. 280b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge */ 28144b46c3ef805793ab3a7730dc71c72d0f258ea8eIan Campbellvoid __ref xen_build_mfn_list_list(void) 282b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 283b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long pfn; 284b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 285696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk if (xen_feature(XENFEAT_auto_translated_physmap)) 286696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk return; 287696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk 288b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* Pre-initialize p2m_top_mfn to be completely missing */ 289b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (p2m_top_mfn == NULL) { 2902c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross p2m_mid_missing_mfn = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); 2913cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_mfn_init(p2m_mid_missing_mfn, p2m_missing); 292b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 2932c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross p2m_top_mfn_p = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); 294b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top_mfn_p_init(p2m_top_mfn_p); 295b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 2962c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross p2m_top_mfn = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); 297b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top_mfn_init(p2m_top_mfn); 298b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } else { 299b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* Reinitialise, mfn's all change after migration */ 3003cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_mfn_init(p2m_mid_missing_mfn, p2m_missing); 301b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 302b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 303b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += P2M_PER_PAGE) { 304b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned topidx = p2m_top_index(pfn); 305b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned mididx = p2m_mid_index(pfn); 306b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long **mid; 307b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long *mid_mfn_p; 308b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 309b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mid = p2m_top[topidx]; 310b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mid_mfn_p = p2m_top_mfn_p[topidx]; 311b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 312b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* Don't bother allocating any mfn mid levels if 313b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * they're just missing, just update the stored mfn, 314b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * since all could have changed over a migrate. 315b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge */ 316b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (mid == p2m_mid_missing) { 317b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge BUG_ON(mididx); 318b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge BUG_ON(mid_mfn_p != p2m_mid_missing_mfn); 319b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top_mfn[topidx] = virt_to_mfn(p2m_mid_missing_mfn); 320b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE; 321b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge continue; 322b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 323b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 324b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (mid_mfn_p == p2m_mid_missing_mfn) { 325b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* 326b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * XXX boot-time only! We should never find 327b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * missing parts of the mfn tree after 3282c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross * runtime. 329b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge */ 3302c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross mid_mfn_p = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); 3313cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_mfn_init(mid_mfn_p, p2m_missing); 332b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 333b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top_mfn_p[topidx] = mid_mfn_p; 334b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 335b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 336b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p); 337b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mid_mfn_p[mididx] = virt_to_mfn(mid[mididx]); 338b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 339b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 340b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 341b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingevoid xen_setup_mfn_list_list(void) 342b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 3434dd322bc3b25be40dfa91e8ac483b846f3e8dffcMukesh Rathor if (xen_feature(XENFEAT_auto_translated_physmap)) 3444dd322bc3b25be40dfa91e8ac483b846f3e8dffcMukesh Rathor return; 3454dd322bc3b25be40dfa91e8ac483b846f3e8dffcMukesh Rathor 346b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info); 347b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 348b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = 349b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge virt_to_mfn(p2m_top_mfn); 350b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn; 351b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 352b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 353b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge/* Set up p2m_top to point to the domain-builder provided p2m pages */ 354b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingevoid __init xen_build_dynamic_phys_to_machine(void) 355b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 356696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk unsigned long *mfn_list; 357696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk unsigned long max_pfn; 358b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long pfn; 359b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 360696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk if (xen_feature(XENFEAT_auto_translated_physmap)) 361696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk return; 362696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk 363696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk mfn_list = (unsigned long *)xen_start_info->mfn_list; 364696fd7c5b2ecb31b339019ced4fe15a3f9e7419aKonrad Rzeszutek Wilk max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages); 365b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge xen_max_p2m_pfn = max_pfn; 366b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 367b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_missing = extend_brk(PAGE_SIZE, PAGE_SIZE); 368b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_init(p2m_missing); 3693cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_identity = extend_brk(PAGE_SIZE, PAGE_SIZE); 3703cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_init(p2m_identity); 371b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 372b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_mid_missing = extend_brk(PAGE_SIZE, PAGE_SIZE); 3733cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_init(p2m_mid_missing, p2m_missing); 3743cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_identity = extend_brk(PAGE_SIZE, PAGE_SIZE); 3753cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_init(p2m_mid_identity, p2m_identity); 376b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 377b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top = extend_brk(PAGE_SIZE, PAGE_SIZE); 378b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top_init(p2m_top); 379b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 380b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* 381b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * The domain builder gives us a pre-constructed p2m array in 382b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * mfn_list for all the pages initially given to us, so we just 383b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * need to graft that into our tree structure. 384b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge */ 385b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge for (pfn = 0; pfn < max_pfn; pfn += P2M_PER_PAGE) { 386b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned topidx = p2m_top_index(pfn); 387b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned mididx = p2m_mid_index(pfn); 388b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 389b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (p2m_top[topidx] == p2m_mid_missing) { 390b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long **mid = extend_brk(PAGE_SIZE, PAGE_SIZE); 3913cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_init(mid, p2m_missing); 392b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 393b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top[topidx] = mid; 394b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 395b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 3968e1b4cf2108488ccfb9a3e7ed7cd85a435e01d4bStefan Bader /* 3978e1b4cf2108488ccfb9a3e7ed7cd85a435e01d4bStefan Bader * As long as the mfn_list has enough entries to completely 3988e1b4cf2108488ccfb9a3e7ed7cd85a435e01d4bStefan Bader * fill a p2m page, pointing into the array is ok. But if 3998e1b4cf2108488ccfb9a3e7ed7cd85a435e01d4bStefan Bader * not the entries beyond the last pfn will be undefined. 4008e1b4cf2108488ccfb9a3e7ed7cd85a435e01d4bStefan Bader */ 4018e1b4cf2108488ccfb9a3e7ed7cd85a435e01d4bStefan Bader if (unlikely(pfn + P2M_PER_PAGE > max_pfn)) { 4028e1b4cf2108488ccfb9a3e7ed7cd85a435e01d4bStefan Bader unsigned long p2midx; 403cf04d120d9413de581437cf9a29f138ec1178f65Stefan Bader 404cf04d120d9413de581437cf9a29f138ec1178f65Stefan Bader p2midx = max_pfn % P2M_PER_PAGE; 405cf04d120d9413de581437cf9a29f138ec1178f65Stefan Bader for ( ; p2midx < P2M_PER_PAGE; p2midx++) 406cf04d120d9413de581437cf9a29f138ec1178f65Stefan Bader mfn_list[pfn + p2midx] = INVALID_P2M_ENTRY; 407cf04d120d9413de581437cf9a29f138ec1178f65Stefan Bader } 408cf04d120d9413de581437cf9a29f138ec1178f65Stefan Bader p2m_top[topidx][mididx] = &mfn_list[pfn]; 409b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 410448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 411448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge m2p_override_init(); 412b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 413357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk#ifdef CONFIG_X86_64 414357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilkunsigned long __init xen_revector_p2m_tree(void) 415357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk{ 416357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned long va_start; 417357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned long va_end; 418357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned long pfn; 4193fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk unsigned long pfn_free = 0; 420357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned long *mfn_list = NULL; 421357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned long size; 422357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 423357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk va_start = xen_start_info->mfn_list; 424357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk /*We copy in increments of P2M_PER_PAGE * sizeof(unsigned long), 425357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk * so make sure it is rounded up to that */ 426357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); 427357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk va_end = va_start + size; 428357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 429357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk /* If we were revectored already, don't do it again. */ 430357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if (va_start <= __START_KERNEL_map && va_start >= __PAGE_OFFSET) 431357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk return 0; 432357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 433357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk mfn_list = alloc_bootmem_align(size, PAGE_SIZE); 434357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if (!mfn_list) { 435357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk pr_warn("Could not allocate space for a new P2M tree!\n"); 436357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk return xen_start_info->mfn_list; 437357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk } 438357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk /* Fill it out with INVALID_P2M_ENTRY value */ 439357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk memset(mfn_list, 0xFF, size); 440357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 441357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk for (pfn = 0; pfn < ALIGN(MAX_DOMAIN_PAGES, P2M_PER_PAGE); pfn += P2M_PER_PAGE) { 442357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned topidx = p2m_top_index(pfn); 443357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned mididx; 444357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned long *mid_p; 445357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 446357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if (!p2m_top[topidx]) 447357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk continue; 448b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 449357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if (p2m_top[topidx] == p2m_mid_missing) 450357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk continue; 451357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 452357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk mididx = p2m_mid_index(pfn); 453357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk mid_p = p2m_top[topidx][mididx]; 454357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if (!mid_p) 455357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk continue; 456357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if ((mid_p == p2m_missing) || (mid_p == p2m_identity)) 457357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk continue; 458b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 459357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if ((unsigned long)mid_p == INVALID_P2M_ENTRY) 460357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk continue; 461357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 462357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk /* The old va. Rebase it on mfn_list */ 463357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk if (mid_p >= (unsigned long *)va_start && mid_p <= (unsigned long *)va_end) { 464357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk unsigned long *new; 465357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 4663fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk if (pfn_free > (size / sizeof(unsigned long))) { 4673fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk WARN(1, "Only allocated for %ld pages, but we want %ld!\n", 4683fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk size / sizeof(unsigned long), pfn_free); 4693fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk return 0; 4703fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk } 4713fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk new = &mfn_list[pfn_free]; 472357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 473357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk copy_page(new, mid_p); 4743fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk p2m_top[topidx][mididx] = &mfn_list[pfn_free]; 4753fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk 4763fc509fc0c590900568ef516a37101d88f3476f5Konrad Rzeszutek Wilk pfn_free += P2M_PER_PAGE; 477357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 478357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk } 479357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk /* This should be the leafs allocated for identity from _brk. */ 480357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk } 481357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk return (unsigned long)mfn_list; 482357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk 483357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk} 484357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk#else 485357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilkunsigned long __init xen_revector_p2m_tree(void) 486357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk{ 487357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk return 0; 488357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk} 489357a3cfb147ee8e97c6f9cdc51e9a33aa56f7d99Konrad Rzeszutek Wilk#endif 490b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingeunsigned long get_phys_to_machine(unsigned long pfn) 491b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 492b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned topidx, mididx, idx; 493b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 494b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (unlikely(pfn >= MAX_P2M_PFN)) 49525b884a83d487fd62c3de7ac1ab5549979188482David Vrabel return IDENTITY_FRAME(pfn); 496b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 497b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge topidx = p2m_top_index(pfn); 498b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mididx = p2m_mid_index(pfn); 499b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge idx = p2m_index(pfn); 500b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 501f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk /* 502f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * The INVALID_P2M_ENTRY is filled in both p2m_*identity 503f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * and in p2m_*missing, so returning the INVALID_P2M_ENTRY 504f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * would be wrong. 505f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk */ 506f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (p2m_top[topidx][mididx] == p2m_identity) 507f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return IDENTITY_FRAME(pfn); 508f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 509b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return p2m_top[topidx][mididx][idx]; 510b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 511b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy FitzhardingeEXPORT_SYMBOL_GPL(get_phys_to_machine); 512b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 513b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic void *alloc_p2m_page(void) 514b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 515b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT); 516b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 517b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 518b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic void free_p2m_page(void *p) 519b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 520b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge free_page((unsigned long)p); 521b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 522b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 523a3118beb6a8cbe77ae3342125d920205871b0717Konrad Rzeszutek Wilk/* 524b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * Fully allocate the p2m structure for a given pfn. We need to check 525b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * that both the top and mid levels are allocated, and make sure the 526b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * parallel mfn tree is kept in sync. We may race with other cpus, so 527b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * the new pages are installed with cmpxchg; if we lose the race then 528b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge * simply free the page we allocated and use the one that's there. 529b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge */ 530b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingestatic bool alloc_p2m(unsigned long pfn) 531b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 532b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned topidx, mididx; 533b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long ***top_p, **mid; 534b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long *top_mfn_p, *mid_mfn; 5353a0e94f8ead4a58b9719db0f78e13d02d059604fJuergen Gross unsigned long *p2m_orig; 536b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 537b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge topidx = p2m_top_index(pfn); 538b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mididx = p2m_mid_index(pfn); 539b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 540b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge top_p = &p2m_top[topidx]; 5413a0e94f8ead4a58b9719db0f78e13d02d059604fJuergen Gross mid = ACCESS_ONCE(*top_p); 542b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 543b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (mid == p2m_mid_missing) { 544b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* Mid level is missing, allocate a new one */ 545b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mid = alloc_p2m_page(); 546b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (!mid) 547b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return false; 548b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 5493cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_init(mid, p2m_missing); 550b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 551b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing) 552b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge free_p2m_page(mid); 553b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 554b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 555b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge top_mfn_p = &p2m_top_mfn[topidx]; 5563a0e94f8ead4a58b9719db0f78e13d02d059604fJuergen Gross mid_mfn = ACCESS_ONCE(p2m_top_mfn_p[topidx]); 557b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 558b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p); 559b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 560b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (mid_mfn == p2m_mid_missing_mfn) { 561b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* Separately check the mid mfn level */ 562b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long missing_mfn; 563b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long mid_mfn_mfn; 564239af7c7132a617f9dcd05da1dc92b96bc6d0645Juergen Gross unsigned long old_mfn; 565b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 566b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mid_mfn = alloc_p2m_page(); 567b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (!mid_mfn) 568b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return false; 569b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 5703cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_mfn_init(mid_mfn, p2m_missing); 571b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 572b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge missing_mfn = virt_to_mfn(p2m_mid_missing_mfn); 573b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mid_mfn_mfn = virt_to_mfn(mid_mfn); 574239af7c7132a617f9dcd05da1dc92b96bc6d0645Juergen Gross old_mfn = cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn); 575239af7c7132a617f9dcd05da1dc92b96bc6d0645Juergen Gross if (old_mfn != missing_mfn) { 576b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge free_p2m_page(mid_mfn); 577239af7c7132a617f9dcd05da1dc92b96bc6d0645Juergen Gross mid_mfn = mfn_to_virt(old_mfn); 578239af7c7132a617f9dcd05da1dc92b96bc6d0645Juergen Gross } else { 579b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top_mfn_p[topidx] = mid_mfn; 580239af7c7132a617f9dcd05da1dc92b96bc6d0645Juergen Gross } 581b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 582b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 5833a0e94f8ead4a58b9719db0f78e13d02d059604fJuergen Gross p2m_orig = ACCESS_ONCE(p2m_top[topidx][mididx]); 5843a0e94f8ead4a58b9719db0f78e13d02d059604fJuergen Gross if (p2m_orig == p2m_identity || p2m_orig == p2m_missing) { 585b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge /* p2m leaf page is missing */ 586b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned long *p2m; 587b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 588b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m = alloc_p2m_page(); 589b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (!p2m) 590b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return false; 591b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 592b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_init(p2m); 593b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 594f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (cmpxchg(&mid[mididx], p2m_orig, p2m) != p2m_orig) 595b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge free_p2m_page(p2m); 596b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge else 597b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mid_mfn[mididx] = virt_to_mfn(p2m); 598b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 599b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 600b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return true; 601b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 602b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 603fcca2e3119f3771dcfd0b03b3718b3c51b5f21c3David Vrabelstatic bool __init early_alloc_p2m(unsigned long pfn, bool check_boundary) 604f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk{ 605f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk unsigned topidx, mididx, idx; 606d5096850b47424fb0f1c6a75b8f7184f7169319aKonrad Rzeszutek Wilk unsigned long *p2m; 607f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 608f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk topidx = p2m_top_index(pfn); 609f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk mididx = p2m_mid_index(pfn); 610f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk idx = p2m_index(pfn); 611f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 612f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk /* Pfff.. No boundary cross-over, lets get out. */ 613cef4cca551d652b7f69c9d76337c5fae24e069dcKonrad Rzeszutek Wilk if (!idx && check_boundary) 614f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return false; 615f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 616f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk WARN(p2m_top[topidx][mididx] == p2m_identity, 617f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk "P2M[%d][%d] == IDENTITY, should be MISSING (or alloced)!\n", 618f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk topidx, mididx); 619f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 620f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk /* 621f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * Could be done by xen_build_dynamic_phys_to_machine.. 622f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk */ 623f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (p2m_top[topidx][mididx] != p2m_missing) 624f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return false; 625f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 626f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk /* Boundary cross-over for the edges: */ 627d5096850b47424fb0f1c6a75b8f7184f7169319aKonrad Rzeszutek Wilk p2m = extend_brk(PAGE_SIZE, PAGE_SIZE); 628f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 629d5096850b47424fb0f1c6a75b8f7184f7169319aKonrad Rzeszutek Wilk p2m_init(p2m); 630f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 631d5096850b47424fb0f1c6a75b8f7184f7169319aKonrad Rzeszutek Wilk p2m_top[topidx][mididx] = p2m; 632f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 633d5096850b47424fb0f1c6a75b8f7184f7169319aKonrad Rzeszutek Wilk return true; 634f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk} 6353f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk 636fcca2e3119f3771dcfd0b03b3718b3c51b5f21c3David Vrabelstatic bool __init early_alloc_p2m_middle(unsigned long pfn) 6373f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk{ 6383f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk unsigned topidx = p2m_top_index(pfn); 6393f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk unsigned long **mid; 6403f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk 6413f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk mid = p2m_top[topidx]; 6423f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk if (mid == p2m_mid_missing) { 6433f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk mid = extend_brk(PAGE_SIZE, PAGE_SIZE); 6443f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk 6453cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_init(mid, p2m_missing); 6463f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk 6473f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk p2m_top[topidx] = mid; 6483f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk } 6493f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk return true; 6503f3aaea29ff7ee2d43b430338427f30ba7f60ff9Konrad Rzeszutek Wilk} 651250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 652250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk/* 653250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk * Skim over the P2M tree looking at pages that are either filled with 654250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk * INVALID_P2M_ENTRY or with 1:1 PFNs. If found, re-use that page and 655250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk * replace the P2M leaf with a p2m_missing or p2m_identity. 656250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk * Stick the old page in the new P2M tree location. 657250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk */ 6582c185687ab016954557aac80074f5d7f7f5d275cJuergen Grossstatic bool __init early_can_reuse_p2m_middle(unsigned long set_pfn) 659250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk{ 660250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk unsigned topidx; 661250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk unsigned mididx; 662250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk unsigned ident_pfns; 663250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk unsigned inv_pfns; 664250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk unsigned long *p2m; 665250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk unsigned idx; 666250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk unsigned long pfn; 667250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 668250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk /* We only look when this entails a P2M middle layer */ 669250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if (p2m_index(set_pfn)) 670250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk return false; 671250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 67250e900417b8096939d12a46848f965e27a905e36Konrad Rzeszutek Wilk for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_PER_PAGE) { 673250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk topidx = p2m_top_index(pfn); 674250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 675250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if (!p2m_top[topidx]) 676250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk continue; 677250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 678250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if (p2m_top[topidx] == p2m_mid_missing) 679250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk continue; 680250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 681250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk mididx = p2m_mid_index(pfn); 682250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk p2m = p2m_top[topidx][mididx]; 683250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if (!p2m) 684250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk continue; 685250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 686250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if ((p2m == p2m_missing) || (p2m == p2m_identity)) 687250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk continue; 688250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 689250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if ((unsigned long)p2m == INVALID_P2M_ENTRY) 690250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk continue; 691250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 692250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk ident_pfns = 0; 693250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk inv_pfns = 0; 694250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk for (idx = 0; idx < P2M_PER_PAGE; idx++) { 695250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk /* IDENTITY_PFNs are 1:1 */ 696250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if (p2m[idx] == IDENTITY_FRAME(pfn + idx)) 697250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk ident_pfns++; 698250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk else if (p2m[idx] == INVALID_P2M_ENTRY) 699250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk inv_pfns++; 700250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk else 701250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk break; 702250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk } 703250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if ((ident_pfns == P2M_PER_PAGE) || (inv_pfns == P2M_PER_PAGE)) 704250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk goto found; 705250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk } 706250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk return false; 707250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilkfound: 708250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk /* Found one, replace old with p2m_identity or p2m_missing */ 709250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk p2m_top[topidx][mididx] = (ident_pfns ? p2m_identity : p2m_missing); 710250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 711250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk /* Reset where we want to stick the old page in. */ 712250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk topidx = p2m_top_index(set_pfn); 713250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk mididx = p2m_mid_index(set_pfn); 714250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 715250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk /* This shouldn't happen */ 716250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if (WARN_ON(p2m_top[topidx] == p2m_mid_missing)) 717fcca2e3119f3771dcfd0b03b3718b3c51b5f21c3David Vrabel early_alloc_p2m_middle(set_pfn); 718250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 719250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk if (WARN_ON(p2m_top[topidx][mididx] != p2m_missing)) 720250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk return false; 721250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 722250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk p2m_init(p2m); 723250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk p2m_top[topidx][mididx] = p2m; 724250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 725250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk return true; 726250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk} 727940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilkbool __init early_set_phys_to_machine(unsigned long pfn, unsigned long mfn) 728940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk{ 729940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk if (unlikely(!__set_phys_to_machine(pfn, mfn))) { 730fcca2e3119f3771dcfd0b03b3718b3c51b5f21c3David Vrabel if (!early_alloc_p2m_middle(pfn)) 731940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk return false; 732940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk 7332c185687ab016954557aac80074f5d7f7f5d275cJuergen Gross if (early_can_reuse_p2m_middle(pfn)) 734250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk return __set_phys_to_machine(pfn, mfn); 735250a41e0ecc433cdd553a364d0fc74c766425209Konrad Rzeszutek Wilk 736fcca2e3119f3771dcfd0b03b3718b3c51b5f21c3David Vrabel if (!early_alloc_p2m(pfn, false /* boundary crossover OK!*/)) 737940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk return false; 738940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk 739940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk if (!__set_phys_to_machine(pfn, mfn)) 740940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk return false; 741940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk } 742940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk 743940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk return true; 744940713bb2ce3033f468a220094a07250a2f69bddKonrad Rzeszutek Wilk} 7453cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel 7463cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabelstatic void __init early_split_p2m(unsigned long pfn) 7473cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel{ 7483cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel unsigned long mididx, idx; 7493cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel 7503cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel mididx = p2m_mid_index(pfn); 7513cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel idx = p2m_index(pfn); 7523cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel 7533cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel /* 7543cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * Allocate new middle and leaf pages if this pfn lies in the 7553cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * middle of one. 7563cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel */ 7573cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel if (mididx || idx) 7583cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel early_alloc_p2m_middle(pfn); 7593cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel if (idx) 7603cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel early_alloc_p2m(pfn, false); 7613cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel} 7623cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel 763b83c6e55ac482f08984504d61382ecf05f0afe32Randy Dunlapunsigned long __init set_phys_range_identity(unsigned long pfn_s, 764f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk unsigned long pfn_e) 765f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk{ 766f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk unsigned long pfn; 767f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 768a9b5bff66b2a63f7d0f42434f5da9024b442159cDavid Vrabel if (unlikely(pfn_s >= MAX_P2M_PFN)) 769f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return 0; 770f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 771f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) 772f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return pfn_e - pfn_s; 773f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 774f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (pfn_s > pfn_e) 775f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return 0; 776f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 777a9b5bff66b2a63f7d0f42434f5da9024b442159cDavid Vrabel if (pfn_e > MAX_P2M_PFN) 778a9b5bff66b2a63f7d0f42434f5da9024b442159cDavid Vrabel pfn_e = MAX_P2M_PFN; 779a9b5bff66b2a63f7d0f42434f5da9024b442159cDavid Vrabel 7803cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel early_split_p2m(pfn_s); 7813cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel early_split_p2m(pfn_e); 782f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 7833cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel for (pfn = pfn_s; pfn < pfn_e;) { 7843cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel unsigned topidx = p2m_top_index(pfn); 7853cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel unsigned mididx = p2m_mid_index(pfn); 786f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 787f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (!__set_phys_to_machine(pfn, IDENTITY_FRAME(pfn))) 788f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk break; 7893cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel pfn++; 7903cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel 7913cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel /* 7923cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * If the PFN was set to a middle or leaf identity 7933cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * page the remainder must also be identity, so skip 7943cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * ahead to the next middle or leaf entry. 7953cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel */ 7963cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel if (p2m_top[topidx] == p2m_mid_identity) 7973cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel pfn = ALIGN(pfn, P2M_MID_PER_PAGE * P2M_PER_PAGE); 7983cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel else if (p2m_top[topidx][mididx] == p2m_identity) 7993cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel pfn = ALIGN(pfn, P2M_PER_PAGE); 8003cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel } 801f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 8023fbdf631b765571b18ee51e073d8f46958e98cebMatt Rushton WARN((pfn - pfn_s) != (pfn_e - pfn_s), 803f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk "Identity mapping failed. We are %ld short of 1-1 mappings!\n", 8043fbdf631b765571b18ee51e073d8f46958e98cebMatt Rushton (pfn_e - pfn_s) - (pfn - pfn_s)); 805f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 806f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return pfn - pfn_s; 807f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk} 808f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 809b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge/* Try to install p2m mapping; fail if intermediate bits missing */ 810b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingebool __set_phys_to_machine(unsigned long pfn, unsigned long mfn) 811b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 812b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge unsigned topidx, mididx, idx; 813b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 8142f558d40911c1b8f929b8a382833ae1da5df3293Stefano Stabellini /* don't track P2M changes in autotranslate guests */ 8152f558d40911c1b8f929b8a382833ae1da5df3293Stefano Stabellini if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) 8166eaa412f2753d98566b777836a98c6e7f672a3bbKonrad Rzeszutek Wilk return true; 8172f558d40911c1b8f929b8a382833ae1da5df3293Stefano Stabellini 818b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (unlikely(pfn >= MAX_P2M_PFN)) { 819b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge BUG_ON(mfn != INVALID_P2M_ENTRY); 820b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return true; 821b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 822b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 823b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge topidx = p2m_top_index(pfn); 824b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge mididx = p2m_mid_index(pfn); 825b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge idx = p2m_index(pfn); 826b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 827f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk /* For sparse holes were the p2m leaf has real PFN along with 828f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk * PCI holes, stick in the PFN as the MFN value. 8293cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * 8303cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * set_phys_range_identity() will have allocated new middle 8313cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * and leaf pages as required so an existing p2m_mid_missing 8323cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * or p2m_missing mean that whole range will be identity so 8333cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel * these can be switched to p2m_mid_identity or p2m_identity. 834f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk */ 835f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (mfn != INVALID_P2M_ENTRY && (mfn & IDENTITY_FRAME_BIT)) { 8363cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel if (p2m_top[topidx] == p2m_mid_identity) 8373cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel return true; 8383cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel 8393cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel if (p2m_top[topidx] == p2m_mid_missing) { 8403cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel WARN_ON(cmpxchg(&p2m_top[topidx], p2m_mid_missing, 8413cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel p2m_mid_identity) != p2m_mid_missing); 8423cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel return true; 8433cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel } 8443cb83e46d032505016ab2565f067e24c8cba9a9dDavid Vrabel 845f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (p2m_top[topidx][mididx] == p2m_identity) 846f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return true; 847f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 848f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk /* Swap over from MISSING to IDENTITY if needed. */ 849f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk if (p2m_top[topidx][mididx] == p2m_missing) { 850c7617798771ad588d585986d896197c04b737621Konrad Rzeszutek Wilk WARN_ON(cmpxchg(&p2m_top[topidx][mididx], p2m_missing, 851c7617798771ad588d585986d896197c04b737621Konrad Rzeszutek Wilk p2m_identity) != p2m_missing); 852f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk return true; 853f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk } 854f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk } 855f4cec35b0d4b90d96e3770a3d1e68ea882e7a7c8Konrad Rzeszutek Wilk 856b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (p2m_top[topidx][mididx] == p2m_missing) 857b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return mfn == INVALID_P2M_ENTRY; 858b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 859b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge p2m_top[topidx][mididx][idx] = mfn; 860b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 861b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return true; 862b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 863b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 864b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardingebool set_phys_to_machine(unsigned long pfn, unsigned long mfn) 865b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge{ 866b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (unlikely(!__set_phys_to_machine(pfn, mfn))) { 867b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (!alloc_p2m(pfn)) 868b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return false; 869b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 870b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge if (!__set_phys_to_machine(pfn, mfn)) 871b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return false; 872b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge } 873b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge 874b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge return true; 875b5eafe924bb054d7c56e6ebd18106352e8a3f916Jeremy Fitzhardinge} 876448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 877448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge#define M2P_OVERRIDE_HASH_SHIFT 10 878448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge#define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT) 879448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 880448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardingestatic RESERVE_BRK_ARRAY(struct list_head, m2p_overrides, M2P_OVERRIDE_HASH); 881448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardingestatic DEFINE_SPINLOCK(m2p_override_lock); 882448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 883448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardingestatic void __init m2p_override_init(void) 884448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge{ 885448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge unsigned i; 886448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 887448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge m2p_overrides = extend_brk(sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH, 888448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge sizeof(unsigned long)); 889448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 890448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge for (i = 0; i < M2P_OVERRIDE_HASH; i++) 891448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge INIT_LIST_HEAD(&m2p_overrides[i]); 892448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge} 893448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 894448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardingestatic unsigned long mfn_hash(unsigned long mfn) 895448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge{ 896448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT); 897448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge} 898448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 8991429d46df4c538d28460d0b493997006a62a1093Zoltan Kissint set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, 9001429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss struct gnttab_map_grant_ref *kmap_ops, 9011429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss struct page **pages, unsigned int count) 9021429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss{ 9031429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss int i, ret = 0; 9041429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss bool lazy = false; 9051429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss pte_t *pte; 9061429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9071429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (xen_feature(XENFEAT_auto_translated_physmap)) 9081429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss return 0; 9091429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9101429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (kmap_ops && 9111429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss !in_interrupt() && 9121429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { 9131429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss arch_enter_lazy_mmu_mode(); 9141429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss lazy = true; 9151429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 9161429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9171429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss for (i = 0; i < count; i++) { 9181429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss unsigned long mfn, pfn; 9191429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9201429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss /* Do not add to override if the map failed. */ 9211429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (map_ops[i].status) 9221429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss continue; 9231429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9241429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (map_ops[i].flags & GNTMAP_contains_pte) { 9251429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) + 9261429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss (map_ops[i].host_addr & ~PAGE_MASK)); 9271429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss mfn = pte_mfn(*pte); 9281429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } else { 9291429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss mfn = PFN_DOWN(map_ops[i].dev_bus_addr); 9301429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 9311429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss pfn = page_to_pfn(pages[i]); 9321429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9331429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss WARN_ON(PagePrivate(pages[i])); 9341429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss SetPagePrivate(pages[i]); 9351429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss set_page_private(pages[i], mfn); 9361429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss pages[i]->index = pfn_to_mfn(pfn); 9371429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9381429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) { 9391429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss ret = -ENOMEM; 9401429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss goto out; 9411429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 9421429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9431429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (kmap_ops) { 9441429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]); 9451429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (ret) 9461429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss goto out; 9471429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 9481429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 9491429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9501429d46df4c538d28460d0b493997006a62a1093Zoltan Kissout: 9511429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (lazy) 9521429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss arch_leave_lazy_mmu_mode(); 9531429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 9541429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss return ret; 9551429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss} 9561429d46df4c538d28460d0b493997006a62a1093Zoltan KissEXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); 9571429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 958448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge/* Add an MFN override for a particular page */ 9590930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabelliniint m2p_add_override(unsigned long mfn, struct page *page, 9600930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini struct gnttab_map_grant_ref *kmap_op) 961448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge{ 962448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge unsigned long flags; 96387f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge unsigned long pfn; 9646b08cfebd3bd346d8a2fd68a2265fc7736849802Ian Campbell unsigned long uninitialized_var(address); 96587f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge unsigned level; 96687f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge pte_t *ptep = NULL; 96787f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge 96887f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge pfn = page_to_pfn(page); 96987f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge if (!PageHighMem(page)) { 97087f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge address = (unsigned long)__va(pfn << PAGE_SHIFT); 97187f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge ptep = lookup_address(address, &level); 97287f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge if (WARN(ptep == NULL || level != PG_LEVEL_4K, 97387f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge "m2p_add_override: pfn %lx not mapped", pfn)) 97487f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge return -EINVAL; 97587f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge } 976b254244d2682fe975630f176c25a4444cc4e088dDaniel De Graaf 9770930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini if (kmap_op != NULL) { 9780930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini if (!PageHighMem(page)) { 9790930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini struct multicall_space mcs = 9800930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini xen_mc_entry(sizeof(*kmap_op)); 9810930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini 9820930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini MULTI_grant_table_op(mcs.mc, 9830930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini GNTTABOP_map_grant_ref, kmap_op, 1); 9840930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini 9850930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini xen_mc_issue(PARAVIRT_LAZY_MMU); 9860930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini } 9870930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini } 988448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge spin_lock_irqsave(&m2p_override_lock, flags); 989448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]); 990448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge spin_unlock_irqrestore(&m2p_override_lock, flags); 99187f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge 992b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini /* p2m(m2p(mfn)) == mfn: the mfn is already present somewhere in 993b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * this domain. Set the FOREIGN_FRAME_BIT in the p2m for the other 994b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * pfn so that the following mfn_to_pfn(mfn) calls will return the 995b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * pfn from the m2p_override (the backend pfn) instead. 996b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * We need to do this because the pages shared by the frontend 997b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * (xen-blkfront) can be already locked (lock_page, called by 998b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * do_read_cache_page); when the userspace backend tries to use them 999b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * with direct_IO, mfn_to_pfn returns the pfn of the frontend, so 1000b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * do_blockdev_direct_IO is going to try to lock the same pages 1001b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * again resulting in a deadlock. 1002b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * As a side effect get_user_pages_fast might not be safe on the 1003b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * frontend pages while they are being shared with the backend, 1004b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * because mfn_to_pfn (that ends up being called by GUPF) will 1005b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * return the backend pfn rather than the frontend pfn. */ 10060160676bba69523e8b0ac83f306cce7d342ed7c8David Vrabel pfn = mfn_to_pfn_no_overrides(mfn); 10070160676bba69523e8b0ac83f306cce7d342ed7c8David Vrabel if (get_phys_to_machine(pfn) == mfn) 1008b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)); 1009b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini 101087f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge return 0; 1011448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge} 10128a91707d0a1a49193e23cb2d243632f2289feb24Konrad Rzeszutek WilkEXPORT_SYMBOL_GPL(m2p_add_override); 10131429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10141429d46df4c538d28460d0b493997006a62a1093Zoltan Kissint clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, 10151429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss struct gnttab_map_grant_ref *kmap_ops, 10161429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss struct page **pages, unsigned int count) 10171429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss{ 10181429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss int i, ret = 0; 10191429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss bool lazy = false; 10201429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10211429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (xen_feature(XENFEAT_auto_translated_physmap)) 10221429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss return 0; 10231429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10241429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (kmap_ops && 10251429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss !in_interrupt() && 10261429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { 10271429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss arch_enter_lazy_mmu_mode(); 10281429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss lazy = true; 10291429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 10301429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10311429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss for (i = 0; i < count; i++) { 10321429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss unsigned long mfn = get_phys_to_machine(page_to_pfn(pages[i])); 10331429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss unsigned long pfn = page_to_pfn(pages[i]); 10341429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10351429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT)) { 10361429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss ret = -EINVAL; 10371429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss goto out; 10381429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 10391429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10401429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss set_page_private(pages[i], INVALID_P2M_ENTRY); 10411429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss WARN_ON(!PagePrivate(pages[i])); 10421429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss ClearPagePrivate(pages[i]); 10431429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss set_phys_to_machine(pfn, pages[i]->index); 10441429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10451429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (kmap_ops) 10461429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss ret = m2p_remove_override(pages[i], &kmap_ops[i], mfn); 10471429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (ret) 10481429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss goto out; 10491429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss } 10501429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10511429d46df4c538d28460d0b493997006a62a1093Zoltan Kissout: 10521429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss if (lazy) 10531429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss arch_leave_lazy_mmu_mode(); 10541429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss return ret; 10551429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss} 10561429d46df4c538d28460d0b493997006a62a1093Zoltan KissEXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping); 10571429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss 10582fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabelliniint m2p_remove_override(struct page *page, 10591429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss struct gnttab_map_grant_ref *kmap_op, 10601429d46df4c538d28460d0b493997006a62a1093Zoltan Kiss unsigned long mfn) 1061448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge{ 1062448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge unsigned long flags; 10639b705f0e98c489b18ba22a6eab9d694b546c8552Stefano Stabellini unsigned long pfn; 10646b08cfebd3bd346d8a2fd68a2265fc7736849802Ian Campbell unsigned long uninitialized_var(address); 106587f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge unsigned level; 106687f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge pte_t *ptep = NULL; 10679b705f0e98c489b18ba22a6eab9d694b546c8552Stefano Stabellini 10689b705f0e98c489b18ba22a6eab9d694b546c8552Stefano Stabellini pfn = page_to_pfn(page); 106987f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge 107087f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge if (!PageHighMem(page)) { 107187f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge address = (unsigned long)__va(pfn << PAGE_SHIFT); 107287f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge ptep = lookup_address(address, &level); 107387f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge 107487f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge if (WARN(ptep == NULL || level != PG_LEVEL_4K, 107587f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge "m2p_remove_override: pfn %lx not mapped", pfn)) 107687f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge return -EINVAL; 107787f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge } 10789b705f0e98c489b18ba22a6eab9d694b546c8552Stefano Stabellini 1079448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge spin_lock_irqsave(&m2p_override_lock, flags); 1080448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge list_del(&page->lru); 1081448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge spin_unlock_irqrestore(&m2p_override_lock, flags); 108287f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge 10832fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini if (kmap_op != NULL) { 10840930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini if (!PageHighMem(page)) { 10850930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini struct multicall_space mcs; 1086ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini struct gnttab_unmap_and_replace *unmap_op; 1087ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini struct page *scratch_page = get_balloon_scratch_page(); 1088ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini unsigned long scratch_page_address = (unsigned long) 1089ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini __va(page_to_pfn(scratch_page) << PAGE_SHIFT); 10900930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini 10910930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini /* 10920930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini * It might be that we queued all the m2p grant table 10930930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini * hypercalls in a multicall, then m2p_remove_override 10940930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini * get called before the multicall has actually been 10950930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini * issued. In this case handle is going to -1 because 10960930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini * it hasn't been modified yet. 10970930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini */ 10982fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini if (kmap_op->handle == -1) 10990930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini xen_mc_flush(); 11000930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini /* 11012fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini * Now if kmap_op->handle is negative it means that the 11020930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini * hypercall actually returned an error. 11030930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini */ 11042fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini if (kmap_op->handle == GNTST_general_error) { 11050930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini printk(KERN_WARNING "m2p_remove_override: " 11060930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini "pfn %lx mfn %lx, failed to modify kernel mappings", 11070930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini pfn, mfn); 1108d7f8f48d1eb3186b1b80b2ed9a7adab191f753e9Boris Ostrovsky put_balloon_scratch_page(); 11090930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini return -1; 11100930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini } 11110930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini 1112d7f8f48d1eb3186b1b80b2ed9a7adab191f753e9Boris Ostrovsky xen_mc_batch(); 1113d7f8f48d1eb3186b1b80b2ed9a7adab191f753e9Boris Ostrovsky 1114d7f8f48d1eb3186b1b80b2ed9a7adab191f753e9Boris Ostrovsky mcs = __xen_mc_entry( 1115ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini sizeof(struct gnttab_unmap_and_replace)); 11160930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini unmap_op = mcs.args; 11172fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini unmap_op->host_addr = kmap_op->host_addr; 1118ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini unmap_op->new_addr = scratch_page_address; 11192fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini unmap_op->handle = kmap_op->handle; 11200930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini 11210930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini MULTI_grant_table_op(mcs.mc, 1122ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini GNTTABOP_unmap_and_replace, unmap_op, 1); 11230930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini 1124ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini mcs = __xen_mc_entry(0); 1125ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini MULTI_update_va_mapping(mcs.mc, scratch_page_address, 1126d7f8f48d1eb3186b1b80b2ed9a7adab191f753e9Boris Ostrovsky pfn_pte(page_to_pfn(scratch_page), 1127ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini PAGE_KERNEL_RO), 0); 1128d7f8f48d1eb3186b1b80b2ed9a7adab191f753e9Boris Ostrovsky 1129ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini xen_mc_issue(PARAVIRT_LAZY_MMU); 1130ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini 11312fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini kmap_op->host_addr = 0; 1132ee0726407feaf504dff304fb603652fb2d778b42Stefano Stabellini put_balloon_scratch_page(); 11330930bba674e248b921ea659b036ff02564e5a5f4Stefano Stabellini } 11342fc136eecd0c647a6b13fcd00d0c41a1a28f35a5Stefano Stabellini } 113587f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge 1136b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present 1137b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * somewhere in this domain, even before being added to the 1138b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * m2p_override (see comment above in m2p_add_override). 1139b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * If there are no other entries in the m2p_override corresponding 1140b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for 1141b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * the original pfn (the one shared by the frontend): the backend 1142b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * cannot do any IO on this page anymore because it has been 1143b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of 1144b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * the original pfn causes mfn_to_pfn(mfn) to return the frontend 1145b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini * pfn again. */ 1146b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini mfn &= ~FOREIGN_FRAME_BIT; 11470160676bba69523e8b0ac83f306cce7d342ed7c8David Vrabel pfn = mfn_to_pfn_no_overrides(mfn); 11480160676bba69523e8b0ac83f306cce7d342ed7c8David Vrabel if (get_phys_to_machine(pfn) == FOREIGN_FRAME(mfn) && 1149b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini m2p_find_override(mfn) == NULL) 1150b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini set_phys_to_machine(pfn, mfn); 1151b9e0d95c041ca2d7ad297ee37c2e9cfab67a188fStefano Stabellini 115287f1d40a706bdebdc8f959b9ac291d0d8fdfcc7eJeremy Fitzhardinge return 0; 1153448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge} 11548a91707d0a1a49193e23cb2d243632f2289feb24Konrad Rzeszutek WilkEXPORT_SYMBOL_GPL(m2p_remove_override); 1155448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1156448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardingestruct page *m2p_find_override(unsigned long mfn) 1157448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge{ 1158448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge unsigned long flags; 1159448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge struct list_head *bucket = &m2p_overrides[mfn_hash(mfn)]; 1160448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge struct page *p, *ret; 1161448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1162448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge ret = NULL; 1163448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1164448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge spin_lock_irqsave(&m2p_override_lock, flags); 1165448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1166448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge list_for_each_entry(p, bucket, lru) { 11670f4b49eaf25e661fbe63a5370b7781166b34d616Konrad Rzeszutek Wilk if (page_private(p) == mfn) { 1168448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge ret = p; 1169448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge break; 1170448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge } 1171448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge } 1172448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1173448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge spin_unlock_irqrestore(&m2p_override_lock, flags); 1174448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1175448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge return ret; 1176448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge} 1177448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1178448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardingeunsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn) 1179448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge{ 1180448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge struct page *p = m2p_find_override(mfn); 1181448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge unsigned long ret = pfn; 1182448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1183448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge if (p) 1184448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge ret = page_to_pfn(p); 1185448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge 1186448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge return ret; 1187448f2831934381e9d3c4d93e700ba7bbe14612dcJeremy Fitzhardinge} 1188e1b478e4ec4477520767d1a920433626263a2a6bKonrad Rzeszutek WilkEXPORT_SYMBOL_GPL(m2p_find_override_pfn); 11892222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk 11902222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#ifdef CONFIG_XEN_DEBUG_FS 1191a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk#include <linux/debugfs.h> 1192a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk#include "debugfs.h" 1193a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilkstatic int p2m_dump_show(struct seq_file *m, void *v) 11942222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk{ 11952222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk static const char * const level_name[] = { "top", "middle", 11968404877ee1cfdbc872e153fd89022f9e47f6f5a3Konrad Rzeszutek Wilk "entry", "abnormal", "error"}; 11972222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#define TYPE_IDENTITY 0 11982222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#define TYPE_MISSING 1 11992222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#define TYPE_PFN 2 12002222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#define TYPE_UNKNOWN 3 1201a491dbef56f2aba42fb292067d4652d246627738Konrad Rzeszutek Wilk static const char * const type_name[] = { 1202a491dbef56f2aba42fb292067d4652d246627738Konrad Rzeszutek Wilk [TYPE_IDENTITY] = "identity", 1203a491dbef56f2aba42fb292067d4652d246627738Konrad Rzeszutek Wilk [TYPE_MISSING] = "missing", 1204a491dbef56f2aba42fb292067d4652d246627738Konrad Rzeszutek Wilk [TYPE_PFN] = "pfn", 1205a491dbef56f2aba42fb292067d4652d246627738Konrad Rzeszutek Wilk [TYPE_UNKNOWN] = "abnormal"}; 12062222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk unsigned long pfn, prev_pfn_type = 0, prev_pfn_level = 0; 12072222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk unsigned int uninitialized_var(prev_level); 12082222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk unsigned int uninitialized_var(prev_type); 12092222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk 12102222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk if (!p2m_top) 12112222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk return 0; 12122222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk 12132222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn++) { 12142222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk unsigned topidx = p2m_top_index(pfn); 12152222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk unsigned mididx = p2m_mid_index(pfn); 12162222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk unsigned idx = p2m_index(pfn); 12172222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk unsigned lvl, type; 12182222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk 12192222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 4; 12202222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk type = TYPE_UNKNOWN; 12212222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk if (p2m_top[topidx] == p2m_mid_missing) { 12222222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 0; type = TYPE_MISSING; 12232222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx] == NULL) { 12242222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 0; type = TYPE_UNKNOWN; 12252222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx] == NULL) { 12262222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 1; type = TYPE_UNKNOWN; 12272222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx] == p2m_identity) { 12282222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 1; type = TYPE_IDENTITY; 12292222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx] == p2m_missing) { 12302222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 1; type = TYPE_MISSING; 12312222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx][idx] == 0) { 12322222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 2; type = TYPE_UNKNOWN; 12332222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx][idx] == IDENTITY_FRAME(pfn)) { 12342222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 2; type = TYPE_IDENTITY; 12352222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx][idx] == INVALID_P2M_ENTRY) { 12362222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 2; type = TYPE_MISSING; 12372222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx][idx] == pfn) { 12382222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 2; type = TYPE_PFN; 12392222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } else if (p2m_top[topidx][mididx][idx] != pfn) { 12402222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 2; type = TYPE_PFN; 12412222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } 12422222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk if (pfn == 0) { 12432222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_level = lvl; 12442222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_type = type; 12452222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } 12462222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk if (pfn == MAX_DOMAIN_PAGES-1) { 12472222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk lvl = 3; 12482222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk type = TYPE_UNKNOWN; 12492222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } 12502222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk if (prev_type != type) { 12512222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk seq_printf(m, " [0x%lx->0x%lx] %s\n", 12522222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_pfn_type, pfn, type_name[prev_type]); 12532222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_pfn_type = pfn; 12542222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_type = type; 12552222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } 12562222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk if (prev_level != lvl) { 12572222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk seq_printf(m, " [0x%lx->0x%lx] level %s\n", 12582222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_pfn_level, pfn, level_name[prev_level]); 12592222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_pfn_level = pfn; 12602222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk prev_level = lvl; 12612222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } 12622222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk } 12632222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk return 0; 12642222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#undef TYPE_IDENTITY 12652222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#undef TYPE_MISSING 12662222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#undef TYPE_PFN 12672222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk#undef TYPE_UNKNOWN 12682222e71bd6eff7b2ad026d4ee663b6327c5a49f5Konrad Rzeszutek Wilk} 1269a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk 1270a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilkstatic int p2m_dump_open(struct inode *inode, struct file *filp) 1271a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk{ 1272a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk return single_open(filp, p2m_dump_show, NULL); 1273a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk} 1274a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk 1275a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilkstatic const struct file_operations p2m_dump_fops = { 1276a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk .open = p2m_dump_open, 1277a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk .read = seq_read, 1278a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk .llseek = seq_lseek, 1279a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk .release = single_release, 1280a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk}; 1281a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk 1282a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilkstatic struct dentry *d_mmu_debug; 1283a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk 1284a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilkstatic int __init xen_p2m_debugfs(void) 1285a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk{ 1286a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk struct dentry *d_xen = xen_init_debugfs(); 1287a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk 1288a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk if (d_xen == NULL) 1289a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk return -ENOMEM; 1290a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk 1291a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk d_mmu_debug = debugfs_create_dir("mmu", d_xen); 1292a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk 1293a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk debugfs_create_file("p2m", 0600, d_mmu_debug, NULL, &p2m_dump_fops); 1294a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk return 0; 1295a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk} 1296a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilkfs_initcall(xen_p2m_debugfs); 1297a867db10e89e12a3d97dedafdd411aa1527a6540Konrad Rzeszutek Wilk#endif /* CONFIG_XEN_DEBUG_FS */ 1298