cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

radix_pgtable.c (27343B)


      1// SPDX-License-Identifier: GPL-2.0-or-later
      2/*
      3 * Page table handling routines for radix page table.
      4 *
      5 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
      6 */
      7
      8#define pr_fmt(fmt) "radix-mmu: " fmt
      9
     10#include <linux/io.h>
     11#include <linux/kernel.h>
     12#include <linux/sched/mm.h>
     13#include <linux/memblock.h>
     14#include <linux/of.h>
     15#include <linux/of_fdt.h>
     16#include <linux/mm.h>
     17#include <linux/hugetlb.h>
     18#include <linux/string_helpers.h>
     19#include <linux/memory.h>
     20
     21#include <asm/pgalloc.h>
     22#include <asm/mmu_context.h>
     23#include <asm/dma.h>
     24#include <asm/machdep.h>
     25#include <asm/mmu.h>
     26#include <asm/firmware.h>
     27#include <asm/powernv.h>
     28#include <asm/sections.h>
     29#include <asm/smp.h>
     30#include <asm/trace.h>
     31#include <asm/uaccess.h>
     32#include <asm/ultravisor.h>
     33
     34#include <trace/events/thp.h>
     35
     36unsigned int mmu_base_pid;
     37unsigned long radix_mem_block_size __ro_after_init;
     38
     39static __ref void *early_alloc_pgtable(unsigned long size, int nid,
     40			unsigned long region_start, unsigned long region_end)
     41{
     42	phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
     43	phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
     44	void *ptr;
     45
     46	if (region_start)
     47		min_addr = region_start;
     48	if (region_end)
     49		max_addr = region_end;
     50
     51	ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
     52
     53	if (!ptr)
     54		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
     55		      __func__, size, size, nid, &min_addr, &max_addr);
     56
     57	return ptr;
     58}
     59
     60/*
     61 * When allocating pud or pmd pointers, we allocate a complete page
     62 * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
     63 * is to ensure that the page obtained from the memblock allocator
     64 * can be completely used as page table page and can be freed
     65 * correctly when the page table entries are removed.
     66 */
     67static int early_map_kernel_page(unsigned long ea, unsigned long pa,
     68			  pgprot_t flags,
     69			  unsigned int map_page_size,
     70			  int nid,
     71			  unsigned long region_start, unsigned long region_end)
     72{
     73	unsigned long pfn = pa >> PAGE_SHIFT;
     74	pgd_t *pgdp;
     75	p4d_t *p4dp;
     76	pud_t *pudp;
     77	pmd_t *pmdp;
     78	pte_t *ptep;
     79
     80	pgdp = pgd_offset_k(ea);
     81	p4dp = p4d_offset(pgdp, ea);
     82	if (p4d_none(*p4dp)) {
     83		pudp = early_alloc_pgtable(PAGE_SIZE, nid,
     84					   region_start, region_end);
     85		p4d_populate(&init_mm, p4dp, pudp);
     86	}
     87	pudp = pud_offset(p4dp, ea);
     88	if (map_page_size == PUD_SIZE) {
     89		ptep = (pte_t *)pudp;
     90		goto set_the_pte;
     91	}
     92	if (pud_none(*pudp)) {
     93		pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
     94					   region_end);
     95		pud_populate(&init_mm, pudp, pmdp);
     96	}
     97	pmdp = pmd_offset(pudp, ea);
     98	if (map_page_size == PMD_SIZE) {
     99		ptep = pmdp_ptep(pmdp);
    100		goto set_the_pte;
    101	}
    102	if (!pmd_present(*pmdp)) {
    103		ptep = early_alloc_pgtable(PAGE_SIZE, nid,
    104						region_start, region_end);
    105		pmd_populate_kernel(&init_mm, pmdp, ptep);
    106	}
    107	ptep = pte_offset_kernel(pmdp, ea);
    108
    109set_the_pte:
    110	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
    111	asm volatile("ptesync": : :"memory");
    112	return 0;
    113}
    114
    115/*
    116 * nid, region_start, and region_end are hints to try to place the page
    117 * table memory in the same node or region.
    118 */
    119static int __map_kernel_page(unsigned long ea, unsigned long pa,
    120			  pgprot_t flags,
    121			  unsigned int map_page_size,
    122			  int nid,
    123			  unsigned long region_start, unsigned long region_end)
    124{
    125	unsigned long pfn = pa >> PAGE_SHIFT;
    126	pgd_t *pgdp;
    127	p4d_t *p4dp;
    128	pud_t *pudp;
    129	pmd_t *pmdp;
    130	pte_t *ptep;
    131	/*
    132	 * Make sure task size is correct as per the max adddr
    133	 */
    134	BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
    135
    136#ifdef CONFIG_PPC_64K_PAGES
    137	BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
    138#endif
    139
    140	if (unlikely(!slab_is_available()))
    141		return early_map_kernel_page(ea, pa, flags, map_page_size,
    142						nid, region_start, region_end);
    143
    144	/*
    145	 * Should make page table allocation functions be able to take a
    146	 * node, so we can place kernel page tables on the right nodes after
    147	 * boot.
    148	 */
    149	pgdp = pgd_offset_k(ea);
    150	p4dp = p4d_offset(pgdp, ea);
    151	pudp = pud_alloc(&init_mm, p4dp, ea);
    152	if (!pudp)
    153		return -ENOMEM;
    154	if (map_page_size == PUD_SIZE) {
    155		ptep = (pte_t *)pudp;
    156		goto set_the_pte;
    157	}
    158	pmdp = pmd_alloc(&init_mm, pudp, ea);
    159	if (!pmdp)
    160		return -ENOMEM;
    161	if (map_page_size == PMD_SIZE) {
    162		ptep = pmdp_ptep(pmdp);
    163		goto set_the_pte;
    164	}
    165	ptep = pte_alloc_kernel(pmdp, ea);
    166	if (!ptep)
    167		return -ENOMEM;
    168
    169set_the_pte:
    170	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
    171	asm volatile("ptesync": : :"memory");
    172	return 0;
    173}
    174
    175int radix__map_kernel_page(unsigned long ea, unsigned long pa,
    176			  pgprot_t flags,
    177			  unsigned int map_page_size)
    178{
    179	return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
    180}
    181
    182#ifdef CONFIG_STRICT_KERNEL_RWX
    183static void radix__change_memory_range(unsigned long start, unsigned long end,
    184				       unsigned long clear)
    185{
    186	unsigned long idx;
    187	pgd_t *pgdp;
    188	p4d_t *p4dp;
    189	pud_t *pudp;
    190	pmd_t *pmdp;
    191	pte_t *ptep;
    192
    193	start = ALIGN_DOWN(start, PAGE_SIZE);
    194	end = PAGE_ALIGN(end); // aligns up
    195
    196	pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
    197		 start, end, clear);
    198
    199	for (idx = start; idx < end; idx += PAGE_SIZE) {
    200		pgdp = pgd_offset_k(idx);
    201		p4dp = p4d_offset(pgdp, idx);
    202		pudp = pud_alloc(&init_mm, p4dp, idx);
    203		if (!pudp)
    204			continue;
    205		if (pud_is_leaf(*pudp)) {
    206			ptep = (pte_t *)pudp;
    207			goto update_the_pte;
    208		}
    209		pmdp = pmd_alloc(&init_mm, pudp, idx);
    210		if (!pmdp)
    211			continue;
    212		if (pmd_is_leaf(*pmdp)) {
    213			ptep = pmdp_ptep(pmdp);
    214			goto update_the_pte;
    215		}
    216		ptep = pte_alloc_kernel(pmdp, idx);
    217		if (!ptep)
    218			continue;
    219update_the_pte:
    220		radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
    221	}
    222
    223	radix__flush_tlb_kernel_range(start, end);
    224}
    225
    226void radix__mark_rodata_ro(void)
    227{
    228	unsigned long start, end;
    229
    230	start = (unsigned long)_stext;
    231	end = (unsigned long)__init_begin;
    232
    233	radix__change_memory_range(start, end, _PAGE_WRITE);
    234}
    235
    236void radix__mark_initmem_nx(void)
    237{
    238	unsigned long start = (unsigned long)__init_begin;
    239	unsigned long end = (unsigned long)__init_end;
    240
    241	radix__change_memory_range(start, end, _PAGE_EXEC);
    242}
    243#endif /* CONFIG_STRICT_KERNEL_RWX */
    244
    245static inline void __meminit
    246print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
    247{
    248	char buf[10];
    249
    250	if (end <= start)
    251		return;
    252
    253	string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
    254
    255	pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
    256		exec ? " (exec)" : "");
    257}
    258
    259static unsigned long next_boundary(unsigned long addr, unsigned long end)
    260{
    261#ifdef CONFIG_STRICT_KERNEL_RWX
    262	if (addr < __pa_symbol(__init_begin))
    263		return __pa_symbol(__init_begin);
    264#endif
    265	return end;
    266}
    267
    268static int __meminit create_physical_mapping(unsigned long start,
    269					     unsigned long end,
    270					     unsigned long max_mapping_size,
    271					     int nid, pgprot_t _prot)
    272{
    273	unsigned long vaddr, addr, mapping_size = 0;
    274	bool prev_exec, exec = false;
    275	pgprot_t prot;
    276	int psize;
    277
    278	start = ALIGN(start, PAGE_SIZE);
    279	end   = ALIGN_DOWN(end, PAGE_SIZE);
    280	for (addr = start; addr < end; addr += mapping_size) {
    281		unsigned long gap, previous_size;
    282		int rc;
    283
    284		gap = next_boundary(addr, end) - addr;
    285		if (gap > max_mapping_size)
    286			gap = max_mapping_size;
    287		previous_size = mapping_size;
    288		prev_exec = exec;
    289
    290		if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
    291		    mmu_psize_defs[MMU_PAGE_1G].shift) {
    292			mapping_size = PUD_SIZE;
    293			psize = MMU_PAGE_1G;
    294		} else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
    295			   mmu_psize_defs[MMU_PAGE_2M].shift) {
    296			mapping_size = PMD_SIZE;
    297			psize = MMU_PAGE_2M;
    298		} else {
    299			mapping_size = PAGE_SIZE;
    300			psize = mmu_virtual_psize;
    301		}
    302
    303		vaddr = (unsigned long)__va(addr);
    304
    305		if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
    306		    overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
    307			prot = PAGE_KERNEL_X;
    308			exec = true;
    309		} else {
    310			prot = _prot;
    311			exec = false;
    312		}
    313
    314		if (mapping_size != previous_size || exec != prev_exec) {
    315			print_mapping(start, addr, previous_size, prev_exec);
    316			start = addr;
    317		}
    318
    319		rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
    320		if (rc)
    321			return rc;
    322
    323		update_page_count(psize, 1);
    324	}
    325
    326	print_mapping(start, addr, mapping_size, exec);
    327	return 0;
    328}
    329
    330static void __init radix_init_pgtable(void)
    331{
    332	unsigned long rts_field;
    333	phys_addr_t start, end;
    334	u64 i;
    335
    336	/* We don't support slb for radix */
    337	slb_set_size(0);
    338
    339	/*
    340	 * Create the linear mapping
    341	 */
    342	for_each_mem_range(i, &start, &end) {
    343		/*
    344		 * The memblock allocator  is up at this point, so the
    345		 * page tables will be allocated within the range. No
    346		 * need or a node (which we don't have yet).
    347		 */
    348
    349		if (end >= RADIX_VMALLOC_START) {
    350			pr_warn("Outside the supported range\n");
    351			continue;
    352		}
    353
    354		WARN_ON(create_physical_mapping(start, end,
    355						radix_mem_block_size,
    356						-1, PAGE_KERNEL));
    357	}
    358
    359	if (!cpu_has_feature(CPU_FTR_HVMODE) &&
    360			cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
    361		/*
    362		 * Older versions of KVM on these machines prefer if the
    363		 * guest only uses the low 19 PID bits.
    364		 */
    365		mmu_pid_bits = 19;
    366	}
    367	mmu_base_pid = 1;
    368
    369	/*
    370	 * Allocate Partition table and process table for the
    371	 * host.
    372	 */
    373	BUG_ON(PRTB_SIZE_SHIFT > 36);
    374	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
    375	/*
    376	 * Fill in the process table.
    377	 */
    378	rts_field = radix__get_tree_size();
    379	process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
    380
    381	/*
    382	 * The init_mm context is given the first available (non-zero) PID,
    383	 * which is the "guard PID" and contains no page table. PIDR should
    384	 * never be set to zero because that duplicates the kernel address
    385	 * space at the 0x0... offset (quadrant 0)!
    386	 *
    387	 * An arbitrary PID that may later be allocated by the PID allocator
    388	 * for userspace processes must not be used either, because that
    389	 * would cause stale user mappings for that PID on CPUs outside of
    390	 * the TLB invalidation scheme (because it won't be in mm_cpumask).
    391	 *
    392	 * So permanently carve out one PID for the purpose of a guard PID.
    393	 */
    394	init_mm.context.id = mmu_base_pid;
    395	mmu_base_pid++;
    396}
    397
    398static void __init radix_init_partition_table(void)
    399{
    400	unsigned long rts_field, dw0, dw1;
    401
    402	mmu_partition_table_init();
    403	rts_field = radix__get_tree_size();
    404	dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
    405	dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
    406	mmu_partition_table_set_entry(0, dw0, dw1, false);
    407
    408	pr_info("Initializing Radix MMU\n");
    409}
    410
    411static int __init get_idx_from_shift(unsigned int shift)
    412{
    413	int idx = -1;
    414
    415	switch (shift) {
    416	case 0xc:
    417		idx = MMU_PAGE_4K;
    418		break;
    419	case 0x10:
    420		idx = MMU_PAGE_64K;
    421		break;
    422	case 0x15:
    423		idx = MMU_PAGE_2M;
    424		break;
    425	case 0x1e:
    426		idx = MMU_PAGE_1G;
    427		break;
    428	}
    429	return idx;
    430}
    431
    432static int __init radix_dt_scan_page_sizes(unsigned long node,
    433					   const char *uname, int depth,
    434					   void *data)
    435{
    436	int size = 0;
    437	int shift, idx;
    438	unsigned int ap;
    439	const __be32 *prop;
    440	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
    441
    442	/* We are scanning "cpu" nodes only */
    443	if (type == NULL || strcmp(type, "cpu") != 0)
    444		return 0;
    445
    446	/* Grab page size encodings */
    447	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
    448	if (!prop)
    449		return 0;
    450
    451	pr_info("Page sizes from device-tree:\n");
    452	for (; size >= 4; size -= 4, ++prop) {
    453
    454		struct mmu_psize_def *def;
    455
    456		/* top 3 bit is AP encoding */
    457		shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
    458		ap = be32_to_cpu(prop[0]) >> 29;
    459		pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
    460
    461		idx = get_idx_from_shift(shift);
    462		if (idx < 0)
    463			continue;
    464
    465		def = &mmu_psize_defs[idx];
    466		def->shift = shift;
    467		def->ap  = ap;
    468		def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
    469	}
    470
    471	/* needed ? */
    472	cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
    473	return 1;
    474}
    475
    476#ifdef CONFIG_MEMORY_HOTPLUG
    477static int __init probe_memory_block_size(unsigned long node, const char *uname, int
    478					  depth, void *data)
    479{
    480	unsigned long *mem_block_size = (unsigned long *)data;
    481	const __be32 *prop;
    482	int len;
    483
    484	if (depth != 1)
    485		return 0;
    486
    487	if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
    488		return 0;
    489
    490	prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
    491
    492	if (!prop || len < dt_root_size_cells * sizeof(__be32))
    493		/*
    494		 * Nothing in the device tree
    495		 */
    496		*mem_block_size = MIN_MEMORY_BLOCK_SIZE;
    497	else
    498		*mem_block_size = of_read_number(prop, dt_root_size_cells);
    499	return 1;
    500}
    501
    502static unsigned long __init radix_memory_block_size(void)
    503{
    504	unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
    505
    506	/*
    507	 * OPAL firmware feature is set by now. Hence we are ok
    508	 * to test OPAL feature.
    509	 */
    510	if (firmware_has_feature(FW_FEATURE_OPAL))
    511		mem_block_size = 1UL * 1024 * 1024 * 1024;
    512	else
    513		of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
    514
    515	return mem_block_size;
    516}
    517
    518#else   /* CONFIG_MEMORY_HOTPLUG */
    519
    520static unsigned long __init radix_memory_block_size(void)
    521{
    522	return 1UL * 1024 * 1024 * 1024;
    523}
    524
    525#endif /* CONFIG_MEMORY_HOTPLUG */
    526
    527
    528void __init radix__early_init_devtree(void)
    529{
    530	int rc;
    531
    532	/*
    533	 * Try to find the available page sizes in the device-tree
    534	 */
    535	rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
    536	if (!rc) {
    537		/*
    538		 * No page size details found in device tree.
    539		 * Let's assume we have page 4k and 64k support
    540		 */
    541		mmu_psize_defs[MMU_PAGE_4K].shift = 12;
    542		mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
    543		mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
    544			psize_to_rpti_pgsize(MMU_PAGE_4K);
    545
    546		mmu_psize_defs[MMU_PAGE_64K].shift = 16;
    547		mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
    548		mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
    549			psize_to_rpti_pgsize(MMU_PAGE_64K);
    550	}
    551
    552	/*
    553	 * Max mapping size used when mapping pages. We don't use
    554	 * ppc_md.memory_block_size() here because this get called
    555	 * early and we don't have machine probe called yet. Also
    556	 * the pseries implementation only check for ibm,lmb-size.
    557	 * All hypervisor supporting radix do expose that device
    558	 * tree node.
    559	 */
    560	radix_mem_block_size = radix_memory_block_size();
    561	return;
    562}
    563
    564void __init radix__early_init_mmu(void)
    565{
    566	unsigned long lpcr;
    567
    568#ifdef CONFIG_PPC_64S_HASH_MMU
    569#ifdef CONFIG_PPC_64K_PAGES
    570	/* PAGE_SIZE mappings */
    571	mmu_virtual_psize = MMU_PAGE_64K;
    572#else
    573	mmu_virtual_psize = MMU_PAGE_4K;
    574#endif
    575
    576#ifdef CONFIG_SPARSEMEM_VMEMMAP
    577	/* vmemmap mapping */
    578	if (mmu_psize_defs[MMU_PAGE_2M].shift) {
    579		/*
    580		 * map vmemmap using 2M if available
    581		 */
    582		mmu_vmemmap_psize = MMU_PAGE_2M;
    583	} else
    584		mmu_vmemmap_psize = mmu_virtual_psize;
    585#endif
    586#endif
    587	/*
    588	 * initialize page table size
    589	 */
    590	__pte_index_size = RADIX_PTE_INDEX_SIZE;
    591	__pmd_index_size = RADIX_PMD_INDEX_SIZE;
    592	__pud_index_size = RADIX_PUD_INDEX_SIZE;
    593	__pgd_index_size = RADIX_PGD_INDEX_SIZE;
    594	__pud_cache_index = RADIX_PUD_INDEX_SIZE;
    595	__pte_table_size = RADIX_PTE_TABLE_SIZE;
    596	__pmd_table_size = RADIX_PMD_TABLE_SIZE;
    597	__pud_table_size = RADIX_PUD_TABLE_SIZE;
    598	__pgd_table_size = RADIX_PGD_TABLE_SIZE;
    599
    600	__pmd_val_bits = RADIX_PMD_VAL_BITS;
    601	__pud_val_bits = RADIX_PUD_VAL_BITS;
    602	__pgd_val_bits = RADIX_PGD_VAL_BITS;
    603
    604	__kernel_virt_start = RADIX_KERN_VIRT_START;
    605	__vmalloc_start = RADIX_VMALLOC_START;
    606	__vmalloc_end = RADIX_VMALLOC_END;
    607	__kernel_io_start = RADIX_KERN_IO_START;
    608	__kernel_io_end = RADIX_KERN_IO_END;
    609	vmemmap = (struct page *)RADIX_VMEMMAP_START;
    610	ioremap_bot = IOREMAP_BASE;
    611
    612#ifdef CONFIG_PCI
    613	pci_io_base = ISA_IO_BASE;
    614#endif
    615	__pte_frag_nr = RADIX_PTE_FRAG_NR;
    616	__pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
    617	__pmd_frag_nr = RADIX_PMD_FRAG_NR;
    618	__pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
    619
    620	radix_init_pgtable();
    621
    622	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
    623		lpcr = mfspr(SPRN_LPCR);
    624		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
    625		radix_init_partition_table();
    626	} else {
    627		radix_init_pseries();
    628	}
    629
    630	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
    631
    632	/* Switch to the guard PID before turning on MMU */
    633	radix__switch_mmu_context(NULL, &init_mm);
    634	tlbiel_all();
    635}
    636
    637void radix__early_init_mmu_secondary(void)
    638{
    639	unsigned long lpcr;
    640	/*
    641	 * update partition table control register and UPRT
    642	 */
    643	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
    644		lpcr = mfspr(SPRN_LPCR);
    645		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
    646
    647		set_ptcr_when_no_uv(__pa(partition_tb) |
    648				    (PATB_SIZE_SHIFT - 12));
    649	}
    650
    651	radix__switch_mmu_context(NULL, &init_mm);
    652	tlbiel_all();
    653
    654	/* Make sure userspace can't change the AMR */
    655	mtspr(SPRN_UAMOR, 0);
    656}
    657
    658/* Called during kexec sequence with MMU off */
    659notrace void radix__mmu_cleanup_all(void)
    660{
    661	unsigned long lpcr;
    662
    663	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
    664		lpcr = mfspr(SPRN_LPCR);
    665		mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
    666		set_ptcr_when_no_uv(0);
    667		powernv_set_nmmu_ptcr(0);
    668		radix__flush_tlb_all();
    669	}
    670}
    671
    672#ifdef CONFIG_MEMORY_HOTPLUG
    673static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
    674{
    675	pte_t *pte;
    676	int i;
    677
    678	for (i = 0; i < PTRS_PER_PTE; i++) {
    679		pte = pte_start + i;
    680		if (!pte_none(*pte))
    681			return;
    682	}
    683
    684	pte_free_kernel(&init_mm, pte_start);
    685	pmd_clear(pmd);
    686}
    687
    688static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
    689{
    690	pmd_t *pmd;
    691	int i;
    692
    693	for (i = 0; i < PTRS_PER_PMD; i++) {
    694		pmd = pmd_start + i;
    695		if (!pmd_none(*pmd))
    696			return;
    697	}
    698
    699	pmd_free(&init_mm, pmd_start);
    700	pud_clear(pud);
    701}
    702
    703static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
    704{
    705	pud_t *pud;
    706	int i;
    707
    708	for (i = 0; i < PTRS_PER_PUD; i++) {
    709		pud = pud_start + i;
    710		if (!pud_none(*pud))
    711			return;
    712	}
    713
    714	pud_free(&init_mm, pud_start);
    715	p4d_clear(p4d);
    716}
    717
    718static void remove_pte_table(pte_t *pte_start, unsigned long addr,
    719			     unsigned long end)
    720{
    721	unsigned long next;
    722	pte_t *pte;
    723
    724	pte = pte_start + pte_index(addr);
    725	for (; addr < end; addr = next, pte++) {
    726		next = (addr + PAGE_SIZE) & PAGE_MASK;
    727		if (next > end)
    728			next = end;
    729
    730		if (!pte_present(*pte))
    731			continue;
    732
    733		if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
    734			/*
    735			 * The vmemmap_free() and remove_section_mapping()
    736			 * codepaths call us with aligned addresses.
    737			 */
    738			WARN_ONCE(1, "%s: unaligned range\n", __func__);
    739			continue;
    740		}
    741
    742		pte_clear(&init_mm, addr, pte);
    743	}
    744}
    745
    746static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
    747			     unsigned long end)
    748{
    749	unsigned long next;
    750	pte_t *pte_base;
    751	pmd_t *pmd;
    752
    753	pmd = pmd_start + pmd_index(addr);
    754	for (; addr < end; addr = next, pmd++) {
    755		next = pmd_addr_end(addr, end);
    756
    757		if (!pmd_present(*pmd))
    758			continue;
    759
    760		if (pmd_is_leaf(*pmd)) {
    761			if (!IS_ALIGNED(addr, PMD_SIZE) ||
    762			    !IS_ALIGNED(next, PMD_SIZE)) {
    763				WARN_ONCE(1, "%s: unaligned range\n", __func__);
    764				continue;
    765			}
    766			pte_clear(&init_mm, addr, (pte_t *)pmd);
    767			continue;
    768		}
    769
    770		pte_base = (pte_t *)pmd_page_vaddr(*pmd);
    771		remove_pte_table(pte_base, addr, next);
    772		free_pte_table(pte_base, pmd);
    773	}
    774}
    775
    776static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
    777			     unsigned long end)
    778{
    779	unsigned long next;
    780	pmd_t *pmd_base;
    781	pud_t *pud;
    782
    783	pud = pud_start + pud_index(addr);
    784	for (; addr < end; addr = next, pud++) {
    785		next = pud_addr_end(addr, end);
    786
    787		if (!pud_present(*pud))
    788			continue;
    789
    790		if (pud_is_leaf(*pud)) {
    791			if (!IS_ALIGNED(addr, PUD_SIZE) ||
    792			    !IS_ALIGNED(next, PUD_SIZE)) {
    793				WARN_ONCE(1, "%s: unaligned range\n", __func__);
    794				continue;
    795			}
    796			pte_clear(&init_mm, addr, (pte_t *)pud);
    797			continue;
    798		}
    799
    800		pmd_base = pud_pgtable(*pud);
    801		remove_pmd_table(pmd_base, addr, next);
    802		free_pmd_table(pmd_base, pud);
    803	}
    804}
    805
    806static void __meminit remove_pagetable(unsigned long start, unsigned long end)
    807{
    808	unsigned long addr, next;
    809	pud_t *pud_base;
    810	pgd_t *pgd;
    811	p4d_t *p4d;
    812
    813	spin_lock(&init_mm.page_table_lock);
    814
    815	for (addr = start; addr < end; addr = next) {
    816		next = pgd_addr_end(addr, end);
    817
    818		pgd = pgd_offset_k(addr);
    819		p4d = p4d_offset(pgd, addr);
    820		if (!p4d_present(*p4d))
    821			continue;
    822
    823		if (p4d_is_leaf(*p4d)) {
    824			if (!IS_ALIGNED(addr, P4D_SIZE) ||
    825			    !IS_ALIGNED(next, P4D_SIZE)) {
    826				WARN_ONCE(1, "%s: unaligned range\n", __func__);
    827				continue;
    828			}
    829
    830			pte_clear(&init_mm, addr, (pte_t *)pgd);
    831			continue;
    832		}
    833
    834		pud_base = p4d_pgtable(*p4d);
    835		remove_pud_table(pud_base, addr, next);
    836		free_pud_table(pud_base, p4d);
    837	}
    838
    839	spin_unlock(&init_mm.page_table_lock);
    840	radix__flush_tlb_kernel_range(start, end);
    841}
    842
    843int __meminit radix__create_section_mapping(unsigned long start,
    844					    unsigned long end, int nid,
    845					    pgprot_t prot)
    846{
    847	if (end >= RADIX_VMALLOC_START) {
    848		pr_warn("Outside the supported range\n");
    849		return -1;
    850	}
    851
    852	return create_physical_mapping(__pa(start), __pa(end),
    853				       radix_mem_block_size, nid, prot);
    854}
    855
    856int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
    857{
    858	remove_pagetable(start, end);
    859	return 0;
    860}
    861#endif /* CONFIG_MEMORY_HOTPLUG */
    862
    863#ifdef CONFIG_SPARSEMEM_VMEMMAP
    864static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
    865				 pgprot_t flags, unsigned int map_page_size,
    866				 int nid)
    867{
    868	return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
    869}
    870
    871int __meminit radix__vmemmap_create_mapping(unsigned long start,
    872				      unsigned long page_size,
    873				      unsigned long phys)
    874{
    875	/* Create a PTE encoding */
    876	unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
    877	int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
    878	int ret;
    879
    880	if ((start + page_size) >= RADIX_VMEMMAP_END) {
    881		pr_warn("Outside the supported range\n");
    882		return -1;
    883	}
    884
    885	ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
    886	BUG_ON(ret);
    887
    888	return 0;
    889}
    890
    891#ifdef CONFIG_MEMORY_HOTPLUG
    892void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
    893{
    894	remove_pagetable(start, start + page_size);
    895}
    896#endif
    897#endif
    898
    899#ifdef CONFIG_DEBUG_PAGEALLOC
    900void radix__kernel_map_pages(struct page *page, int numpages, int enable)
    901{
    902	pr_warn_once("DEBUG_PAGEALLOC not supported in radix mode\n");
    903}
    904#endif
    905
    906#ifdef CONFIG_TRANSPARENT_HUGEPAGE
    907
    908unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
    909				  pmd_t *pmdp, unsigned long clr,
    910				  unsigned long set)
    911{
    912	unsigned long old;
    913
    914#ifdef CONFIG_DEBUG_VM
    915	WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
    916	assert_spin_locked(pmd_lockptr(mm, pmdp));
    917#endif
    918
    919	old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
    920	trace_hugepage_update(addr, old, clr, set);
    921
    922	return old;
    923}
    924
    925pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
    926			pmd_t *pmdp)
    927
    928{
    929	pmd_t pmd;
    930
    931	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
    932	VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
    933	VM_BUG_ON(pmd_devmap(*pmdp));
    934	/*
    935	 * khugepaged calls this for normal pmd
    936	 */
    937	pmd = *pmdp;
    938	pmd_clear(pmdp);
    939
    940	/*
    941	 * pmdp collapse_flush need to ensure that there are no parallel gup
    942	 * walk after this call. This is needed so that we can have stable
    943	 * page ref count when collapsing a page. We don't allow a collapse page
    944	 * if we have gup taken on the page. We can ensure that by sending IPI
    945	 * because gup walk happens with IRQ disabled.
    946	 */
    947	serialize_against_pte_lookup(vma->vm_mm);
    948
    949	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
    950
    951	return pmd;
    952}
    953
    954/*
    955 * For us pgtable_t is pte_t *. Inorder to save the deposisted
    956 * page table, we consider the allocated page table as a list
    957 * head. On withdraw we need to make sure we zero out the used
    958 * list_head memory area.
    959 */
    960void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
    961				 pgtable_t pgtable)
    962{
    963	struct list_head *lh = (struct list_head *) pgtable;
    964
    965	assert_spin_locked(pmd_lockptr(mm, pmdp));
    966
    967	/* FIFO */
    968	if (!pmd_huge_pte(mm, pmdp))
    969		INIT_LIST_HEAD(lh);
    970	else
    971		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
    972	pmd_huge_pte(mm, pmdp) = pgtable;
    973}
    974
    975pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
    976{
    977	pte_t *ptep;
    978	pgtable_t pgtable;
    979	struct list_head *lh;
    980
    981	assert_spin_locked(pmd_lockptr(mm, pmdp));
    982
    983	/* FIFO */
    984	pgtable = pmd_huge_pte(mm, pmdp);
    985	lh = (struct list_head *) pgtable;
    986	if (list_empty(lh))
    987		pmd_huge_pte(mm, pmdp) = NULL;
    988	else {
    989		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
    990		list_del(lh);
    991	}
    992	ptep = (pte_t *) pgtable;
    993	*ptep = __pte(0);
    994	ptep++;
    995	*ptep = __pte(0);
    996	return pgtable;
    997}
    998
    999pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
   1000				     unsigned long addr, pmd_t *pmdp)
   1001{
   1002	pmd_t old_pmd;
   1003	unsigned long old;
   1004
   1005	old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
   1006	old_pmd = __pmd(old);
   1007	return old_pmd;
   1008}
   1009
   1010#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
   1011
   1012void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
   1013				  pte_t entry, unsigned long address, int psize)
   1014{
   1015	struct mm_struct *mm = vma->vm_mm;
   1016	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
   1017					      _PAGE_RW | _PAGE_EXEC);
   1018
   1019	unsigned long change = pte_val(entry) ^ pte_val(*ptep);
   1020	/*
   1021	 * To avoid NMMU hang while relaxing access, we need mark
   1022	 * the pte invalid in between.
   1023	 */
   1024	if ((change & _PAGE_RW) && atomic_read(&mm->context.copros) > 0) {
   1025		unsigned long old_pte, new_pte;
   1026
   1027		old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
   1028		/*
   1029		 * new value of pte
   1030		 */
   1031		new_pte = old_pte | set;
   1032		radix__flush_tlb_page_psize(mm, address, psize);
   1033		__radix_pte_update(ptep, _PAGE_INVALID, new_pte);
   1034	} else {
   1035		__radix_pte_update(ptep, 0, set);
   1036		/*
   1037		 * Book3S does not require a TLB flush when relaxing access
   1038		 * restrictions when the address space is not attached to a
   1039		 * NMMU, because the core MMU will reload the pte after taking
   1040		 * an access fault, which is defined by the architecture.
   1041		 */
   1042	}
   1043	/* See ptesync comment in radix__set_pte_at */
   1044}
   1045
   1046void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
   1047				    unsigned long addr, pte_t *ptep,
   1048				    pte_t old_pte, pte_t pte)
   1049{
   1050	struct mm_struct *mm = vma->vm_mm;
   1051
   1052	/*
   1053	 * To avoid NMMU hang while relaxing access we need to flush the tlb before
   1054	 * we set the new value. We need to do this only for radix, because hash
   1055	 * translation does flush when updating the linux pte.
   1056	 */
   1057	if (is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
   1058	    (atomic_read(&mm->context.copros) > 0))
   1059		radix__flush_tlb_page(vma, addr);
   1060
   1061	set_pte_at(mm, addr, ptep, pte);
   1062}
   1063
   1064int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
   1065{
   1066	pte_t *ptep = (pte_t *)pud;
   1067	pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
   1068
   1069	if (!radix_enabled())
   1070		return 0;
   1071
   1072	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
   1073
   1074	return 1;
   1075}
   1076
   1077int pud_clear_huge(pud_t *pud)
   1078{
   1079	if (pud_is_leaf(*pud)) {
   1080		pud_clear(pud);
   1081		return 1;
   1082	}
   1083
   1084	return 0;
   1085}
   1086
   1087int pud_free_pmd_page(pud_t *pud, unsigned long addr)
   1088{
   1089	pmd_t *pmd;
   1090	int i;
   1091
   1092	pmd = pud_pgtable(*pud);
   1093	pud_clear(pud);
   1094
   1095	flush_tlb_kernel_range(addr, addr + PUD_SIZE);
   1096
   1097	for (i = 0; i < PTRS_PER_PMD; i++) {
   1098		if (!pmd_none(pmd[i])) {
   1099			pte_t *pte;
   1100			pte = (pte_t *)pmd_page_vaddr(pmd[i]);
   1101
   1102			pte_free_kernel(&init_mm, pte);
   1103		}
   1104	}
   1105
   1106	pmd_free(&init_mm, pmd);
   1107
   1108	return 1;
   1109}
   1110
   1111int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
   1112{
   1113	pte_t *ptep = (pte_t *)pmd;
   1114	pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
   1115
   1116	if (!radix_enabled())
   1117		return 0;
   1118
   1119	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
   1120
   1121	return 1;
   1122}
   1123
   1124int pmd_clear_huge(pmd_t *pmd)
   1125{
   1126	if (pmd_is_leaf(*pmd)) {
   1127		pmd_clear(pmd);
   1128		return 1;
   1129	}
   1130
   1131	return 0;
   1132}
   1133
   1134int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
   1135{
   1136	pte_t *pte;
   1137
   1138	pte = (pte_t *)pmd_page_vaddr(*pmd);
   1139	pmd_clear(pmd);
   1140
   1141	flush_tlb_kernel_range(addr, addr + PMD_SIZE);
   1142
   1143	pte_free_kernel(&init_mm, pte);
   1144
   1145	return 1;
   1146}