cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

hugetlbpage.c (4653B)

      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
      4 *
      5 * Copyright (C) 2003 David Gibson, IBM Corporation.
      6 *
      7 * Based on the IA-32 version:
      8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
      9 */
     10
     11#include <linux/mm.h>
     12#include <linux/hugetlb.h>
     13#include <asm/cacheflush.h>
     14#include <asm/machdep.h>
     15
     16unsigned int hpage_shift;
     17EXPORT_SYMBOL(hpage_shift);
     18
     19#ifdef CONFIG_PPC_64S_HASH_MMU
     20int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
     21		     pte_t *ptep, unsigned long trap, unsigned long flags,
     22		     int ssize, unsigned int shift, unsigned int mmu_psize)
     23{
     24	real_pte_t rpte;
     25	unsigned long vpn;
     26	unsigned long old_pte, new_pte;
     27	unsigned long rflags, pa;
     28	long slot, offset;
     29
     30	BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
     31
     32	/* Search the Linux page table for a match with va */
     33	vpn = hpt_vpn(ea, vsid, ssize);
     34
     35	/*
     36	 * At this point, we have a pte (old_pte) which can be used to build
     37	 * or update an HPTE. There are 2 cases:
     38	 *
     39	 * 1. There is a valid (present) pte with no associated HPTE (this is
     40	 *	the most common case)
     41	 * 2. There is a valid (present) pte with an associated HPTE. The
     42	 *	current values of the pp bits in the HPTE prevent access
     43	 *	because we are doing software DIRTY bit management and the
     44	 *	page is currently not DIRTY.
     45	 */
     46
     47
     48	do {
     49		old_pte = pte_val(*ptep);
     50		/* If PTE busy, retry the access */
     51		if (unlikely(old_pte & H_PAGE_BUSY))
     52			return 0;
     53		/* If PTE permissions don't match, take page fault */
     54		if (unlikely(!check_pte_access(access, old_pte)))
     55			return 1;
     56
     57		/*
     58		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
     59		 * a write access
     60		 */
     61		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
     62		if (access & _PAGE_WRITE)
     63			new_pte |= _PAGE_DIRTY;
     64	} while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
     65
     66	/* Make sure this is a hugetlb entry */
     67	if (old_pte & (H_PAGE_THP_HUGE | _PAGE_DEVMAP))
     68		return 0;
     69
     70	rflags = htab_convert_pte_flags(new_pte, flags);
     71	if (unlikely(mmu_psize == MMU_PAGE_16G))
     72		offset = PTRS_PER_PUD;
     73	else
     74		offset = PTRS_PER_PMD;
     75	rpte = __real_pte(__pte(old_pte), ptep, offset);
     76
     77	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
     78		/*
     79		 * No CPU has hugepages but lacks no execute, so we
     80		 * don't need to worry about that case
     81		 */
     82		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
     83
     84	/* Check if pte already has an hpte (case 2) */
     85	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
     86		/* There MIGHT be an HPTE for this pte */
     87		unsigned long gslot;
     88
     89		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
     90		if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, mmu_psize,
     91					       mmu_psize, ssize, flags) == -1)
     92			old_pte &= ~_PAGE_HPTEFLAGS;
     93	}
     94
     95	if (likely(!(old_pte & H_PAGE_HASHPTE))) {
     96		unsigned long hash = hpt_hash(vpn, shift, ssize);
     97
     98		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
     99
    100		/* clear HPTE slot informations in new PTE */
    101		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
    102
    103		slot = hpte_insert_repeating(hash, vpn, pa, rflags, 0,
    104					     mmu_psize, ssize);
    105
    106		/*
    107		 * Hypervisor failure. Restore old pte and return -1
    108		 * similar to __hash_page_*
    109		 */
    110		if (unlikely(slot == -2)) {
    111			*ptep = __pte(old_pte);
    112			hash_failure_debug(ea, access, vsid, trap, ssize,
    113					   mmu_psize, mmu_psize, old_pte);
    114			return -1;
    115		}
    116
    117		new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
    118	}
    119
    120	/*
    121	 * No need to use ldarx/stdcx here
    122	 */
    123	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
    124	return 0;
    125}
    126#endif
    127
    128pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
    129				  unsigned long addr, pte_t *ptep)
    130{
    131	unsigned long pte_val;
    132	/*
    133	 * Clear the _PAGE_PRESENT so that no hardware parallel update is
    134	 * possible. Also keep the pte_present true so that we don't take
    135	 * wrong fault.
    136	 */
    137	pte_val = pte_update(vma->vm_mm, addr, ptep,
    138			     _PAGE_PRESENT, _PAGE_INVALID, 1);
    139
    140	return __pte(pte_val);
    141}
    142
    143void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
    144				  pte_t *ptep, pte_t old_pte, pte_t pte)
    145{
    146
    147	if (radix_enabled())
    148		return radix__huge_ptep_modify_prot_commit(vma, addr, ptep,
    149							   old_pte, pte);
    150	set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
    151}
    152
    153void __init hugetlbpage_init_defaultsize(void)
    154{
    155	/* Set default large page size. Currently, we pick 16M or 1M
    156	 * depending on what is available
    157	 */
    158	if (mmu_psize_defs[MMU_PAGE_16M].shift)
    159		hpage_shift = mmu_psize_defs[MMU_PAGE_16M].shift;
    160	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
    161		hpage_shift = mmu_psize_defs[MMU_PAGE_1M].shift;
    162	else if (mmu_psize_defs[MMU_PAGE_2M].shift)
    163		hpage_shift = mmu_psize_defs[MMU_PAGE_2M].shift;
    164}