cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

page.c (4653B)


      1/*
      2 * Copyright (c) 2006 Oracle.  All rights reserved.
      3 *
      4 * This software is available to you under a choice of one of two
      5 * licenses.  You may choose to be licensed under the terms of the GNU
      6 * General Public License (GPL) Version 2, available from the file
      7 * COPYING in the main directory of this source tree, or the
      8 * OpenIB.org BSD license below:
      9 *
     10 *     Redistribution and use in source and binary forms, with or
     11 *     without modification, are permitted provided that the following
     12 *     conditions are met:
     13 *
     14 *      - Redistributions of source code must retain the above
     15 *        copyright notice, this list of conditions and the following
     16 *        disclaimer.
     17 *
     18 *      - Redistributions in binary form must reproduce the above
     19 *        copyright notice, this list of conditions and the following
     20 *        disclaimer in the documentation and/or other materials
     21 *        provided with the distribution.
     22 *
     23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
     24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
     25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
     26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
     27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
     28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
     29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
     30 * SOFTWARE.
     31 *
     32 */
     33#include <linux/highmem.h>
     34#include <linux/gfp.h>
     35#include <linux/cpu.h>
     36#include <linux/export.h>
     37
     38#include "rds.h"
     39
     40struct rds_page_remainder {
     41	struct page	*r_page;
     42	unsigned long	r_offset;
     43};
     44
     45static
     46DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder, rds_page_remainders);
     47
     48/**
     49 * rds_page_remainder_alloc - build up regions of a message.
     50 *
     51 * @scat: Scatter list for message
     52 * @bytes: the number of bytes needed.
     53 * @gfp: the waiting behaviour of the allocation
     54 *
     55 * @gfp is always ored with __GFP_HIGHMEM.  Callers must be prepared to
     56 * kmap the pages, etc.
     57 *
     58 * If @bytes is at least a full page then this just returns a page from
     59 * alloc_page().
     60 *
     61 * If @bytes is a partial page then this stores the unused region of the
     62 * page in a per-cpu structure.  Future partial-page allocations may be
     63 * satisfied from that cached region.  This lets us waste less memory on
     64 * small allocations with minimal complexity.  It works because the transmit
     65 * path passes read-only page regions down to devices.  They hold a page
     66 * reference until they are done with the region.
     67 */
     68int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
     69			     gfp_t gfp)
     70{
     71	struct rds_page_remainder *rem;
     72	unsigned long flags;
     73	struct page *page;
     74	int ret;
     75
     76	gfp |= __GFP_HIGHMEM;
     77
     78	/* jump straight to allocation if we're trying for a huge page */
     79	if (bytes >= PAGE_SIZE) {
     80		page = alloc_page(gfp);
     81		if (!page) {
     82			ret = -ENOMEM;
     83		} else {
     84			sg_set_page(scat, page, PAGE_SIZE, 0);
     85			ret = 0;
     86		}
     87		goto out;
     88	}
     89
     90	rem = &per_cpu(rds_page_remainders, get_cpu());
     91	local_irq_save(flags);
     92
     93	while (1) {
     94		/* avoid a tiny region getting stuck by tossing it */
     95		if (rem->r_page && bytes > (PAGE_SIZE - rem->r_offset)) {
     96			rds_stats_inc(s_page_remainder_miss);
     97			__free_page(rem->r_page);
     98			rem->r_page = NULL;
     99		}
    100
    101		/* hand out a fragment from the cached page */
    102		if (rem->r_page && bytes <= (PAGE_SIZE - rem->r_offset)) {
    103			sg_set_page(scat, rem->r_page, bytes, rem->r_offset);
    104			get_page(sg_page(scat));
    105
    106			if (rem->r_offset != 0)
    107				rds_stats_inc(s_page_remainder_hit);
    108
    109			rem->r_offset += ALIGN(bytes, 8);
    110			if (rem->r_offset >= PAGE_SIZE) {
    111				__free_page(rem->r_page);
    112				rem->r_page = NULL;
    113			}
    114			ret = 0;
    115			break;
    116		}
    117
    118		/* alloc if there is nothing for us to use */
    119		local_irq_restore(flags);
    120		put_cpu();
    121
    122		page = alloc_page(gfp);
    123
    124		rem = &per_cpu(rds_page_remainders, get_cpu());
    125		local_irq_save(flags);
    126
    127		if (!page) {
    128			ret = -ENOMEM;
    129			break;
    130		}
    131
    132		/* did someone race to fill the remainder before us? */
    133		if (rem->r_page) {
    134			__free_page(page);
    135			continue;
    136		}
    137
    138		/* otherwise install our page and loop around to alloc */
    139		rem->r_page = page;
    140		rem->r_offset = 0;
    141	}
    142
    143	local_irq_restore(flags);
    144	put_cpu();
    145out:
    146	rdsdebug("bytes %lu ret %d %p %u %u\n", bytes, ret,
    147		 ret ? NULL : sg_page(scat), ret ? 0 : scat->offset,
    148		 ret ? 0 : scat->length);
    149	return ret;
    150}
    151EXPORT_SYMBOL_GPL(rds_page_remainder_alloc);
    152
    153void rds_page_exit(void)
    154{
    155	unsigned int cpu;
    156
    157	for_each_possible_cpu(cpu) {
    158		struct rds_page_remainder *rem;
    159
    160		rem = &per_cpu(rds_page_remainders, cpu);
    161		rdsdebug("cpu %u\n", cpu);
    162
    163		if (rem->r_page)
    164			__free_page(rem->r_page);
    165		rem->r_page = NULL;
    166	}
    167}