cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

pgalloc.h (4699B)

      1/* SPDX-License-Identifier: GPL-2.0-or-later */
      2#ifndef _ASM_POWERPC_BOOK3S_64_PGALLOC_H
      3#define _ASM_POWERPC_BOOK3S_64_PGALLOC_H
      4/*
      5 */
      6
      7#include <linux/slab.h>
      8#include <linux/cpumask.h>
      9#include <linux/kmemleak.h>
     10#include <linux/percpu.h>
     11
     12struct vmemmap_backing {
     13	struct vmemmap_backing *list;
     14	unsigned long phys;
     15	unsigned long virt_addr;
     16};
     17extern struct vmemmap_backing *vmemmap_list;
     18
     19extern pmd_t *pmd_fragment_alloc(struct mm_struct *, unsigned long);
     20extern void pmd_fragment_free(unsigned long *);
     21extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
     22extern void __tlb_remove_table(void *_table);
     23void pte_frag_destroy(void *pte_frag);
     24
     25static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm)
     26{
     27#ifdef CONFIG_PPC_64K_PAGES
     28	return (pgd_t *)__get_free_page(pgtable_gfp_flags(mm, PGALLOC_GFP));
     29#else
     30	struct page *page;
     31	page = alloc_pages(pgtable_gfp_flags(mm, PGALLOC_GFP | __GFP_RETRY_MAYFAIL),
     32				4);
     33	if (!page)
     34		return NULL;
     35	return (pgd_t *) page_address(page);
     36#endif
     37}
     38
     39static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd)
     40{
     41#ifdef CONFIG_PPC_64K_PAGES
     42	free_page((unsigned long)pgd);
     43#else
     44	free_pages((unsigned long)pgd, 4);
     45#endif
     46}
     47
     48static inline pgd_t *pgd_alloc(struct mm_struct *mm)
     49{
     50	pgd_t *pgd;
     51
     52	if (radix_enabled())
     53		return radix__pgd_alloc(mm);
     54
     55	pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
     56			       pgtable_gfp_flags(mm, GFP_KERNEL));
     57	if (unlikely(!pgd))
     58		return pgd;
     59
     60	/*
     61	 * Don't scan the PGD for pointers, it contains references to PUDs but
     62	 * those references are not full pointers and so can't be recognised by
     63	 * kmemleak.
     64	 */
     65	kmemleak_no_scan(pgd);
     66
     67	/*
     68	 * With hugetlb, we don't clear the second half of the page table.
     69	 * If we share the same slab cache with the pmd or pud level table,
     70	 * we need to make sure we zero out the full table on alloc.
     71	 * With 4K we don't store slot in the second half. Hence we don't
     72	 * need to do this for 4k.
     73	 */
     74#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) && \
     75	(H_PGD_INDEX_SIZE == H_PUD_CACHE_INDEX)
     76	memset(pgd, 0, PGD_TABLE_SIZE);
     77#endif
     78	return pgd;
     79}
     80
     81static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
     82{
     83	if (radix_enabled())
     84		return radix__pgd_free(mm, pgd);
     85	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
     86}
     87
     88static inline void p4d_populate(struct mm_struct *mm, p4d_t *pgd, pud_t *pud)
     89{
     90	*pgd =  __p4d(__pgtable_ptr_val(pud) | PGD_VAL_BITS);
     91}
     92
     93static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
     94{
     95	pud_t *pud;
     96
     97	pud = kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX),
     98			       pgtable_gfp_flags(mm, GFP_KERNEL));
     99	/*
    100	 * Tell kmemleak to ignore the PUD, that means don't scan it for
    101	 * pointers and don't consider it a leak. PUDs are typically only
    102	 * referred to by their PGD, but kmemleak is not able to recognise those
    103	 * as pointers, leading to false leak reports.
    104	 */
    105	kmemleak_ignore(pud);
    106
    107	return pud;
    108}
    109
    110static inline void __pud_free(pud_t *pud)
    111{
    112	struct page *page = virt_to_page(pud);
    113
    114	/*
    115	 * Early pud pages allocated via memblock allocator
    116	 * can't be directly freed to slab
    117	 */
    118	if (PageReserved(page))
    119		free_reserved_page(page);
    120	else
    121		kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud);
    122}
    123
    124static inline void pud_free(struct mm_struct *mm, pud_t *pud)
    125{
    126	return __pud_free(pud);
    127}
    128
    129static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
    130{
    131	*pud = __pud(__pgtable_ptr_val(pmd) | PUD_VAL_BITS);
    132}
    133
    134static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
    135				  unsigned long address)
    136{
    137	pgtable_free_tlb(tlb, pud, PUD_INDEX);
    138}
    139
    140static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
    141{
    142	return pmd_fragment_alloc(mm, addr);
    143}
    144
    145static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
    146{
    147	pmd_fragment_free((unsigned long *)pmd);
    148}
    149
    150static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
    151				  unsigned long address)
    152{
    153	return pgtable_free_tlb(tlb, pmd, PMD_INDEX);
    154}
    155
    156static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
    157				       pte_t *pte)
    158{
    159	*pmd = __pmd(__pgtable_ptr_val(pte) | PMD_VAL_BITS);
    160}
    161
    162static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
    163				pgtable_t pte_page)
    164{
    165	*pmd = __pmd(__pgtable_ptr_val(pte_page) | PMD_VAL_BITS);
    166}
    167
    168static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
    169				  unsigned long address)
    170{
    171	pgtable_free_tlb(tlb, table, PTE_INDEX);
    172}
    173
    174extern atomic_long_t direct_pages_count[MMU_PAGE_COUNT];
    175static inline void update_page_count(int psize, long count)
    176{
    177	if (IS_ENABLED(CONFIG_PROC_FS))
    178		atomic_long_add(count, &direct_pages_count[psize]);
    179}
    180
    181#endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */