cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

sg_pool.c (4340B)

      1// SPDX-License-Identifier: GPL-2.0-only
      2#include <linux/module.h>
      3#include <linux/scatterlist.h>
      4#include <linux/mempool.h>
      5#include <linux/slab.h>
      6
      7#define SG_MEMPOOL_NR		ARRAY_SIZE(sg_pools)
      8#define SG_MEMPOOL_SIZE		2
      9
     10struct sg_pool {
     11	size_t		size;
     12	char		*name;
     13	struct kmem_cache	*slab;
     14	mempool_t	*pool;
     15};
     16
     17#define SP(x) { .size = x, "sgpool-" __stringify(x) }
     18#if (SG_CHUNK_SIZE < 32)
     19#error SG_CHUNK_SIZE is too small (must be 32 or greater)
     20#endif
     21static struct sg_pool sg_pools[] = {
     22	SP(8),
     23	SP(16),
     24#if (SG_CHUNK_SIZE > 32)
     25	SP(32),
     26#if (SG_CHUNK_SIZE > 64)
     27	SP(64),
     28#if (SG_CHUNK_SIZE > 128)
     29	SP(128),
     30#if (SG_CHUNK_SIZE > 256)
     31#error SG_CHUNK_SIZE is too large (256 MAX)
     32#endif
     33#endif
     34#endif
     35#endif
     36	SP(SG_CHUNK_SIZE)
     37};
     38#undef SP
     39
     40static inline unsigned int sg_pool_index(unsigned short nents)
     41{
     42	unsigned int index;
     43
     44	BUG_ON(nents > SG_CHUNK_SIZE);
     45
     46	if (nents <= 8)
     47		index = 0;
     48	else
     49		index = get_count_order(nents) - 3;
     50
     51	return index;
     52}
     53
     54static void sg_pool_free(struct scatterlist *sgl, unsigned int nents)
     55{
     56	struct sg_pool *sgp;
     57
     58	sgp = sg_pools + sg_pool_index(nents);
     59	mempool_free(sgl, sgp->pool);
     60}
     61
     62static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask)
     63{
     64	struct sg_pool *sgp;
     65
     66	sgp = sg_pools + sg_pool_index(nents);
     67	return mempool_alloc(sgp->pool, gfp_mask);
     68}
     69
     70/**
     71 * sg_free_table_chained - Free a previously mapped sg table
     72 * @table:	The sg table header to use
     73 * @nents_first_chunk: size of the first_chunk SGL passed to
     74 *		sg_alloc_table_chained
     75 *
     76 *  Description:
     77 *    Free an sg table previously allocated and setup with
     78 *    sg_alloc_table_chained().
     79 *
     80 *    @nents_first_chunk has to be same with that same parameter passed
     81 *    to sg_alloc_table_chained().
     82 *
     83 **/
     84void sg_free_table_chained(struct sg_table *table,
     85		unsigned nents_first_chunk)
     86{
     87	if (table->orig_nents <= nents_first_chunk)
     88		return;
     89
     90	if (nents_first_chunk == 1)
     91		nents_first_chunk = 0;
     92
     93	__sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free,
     94			table->orig_nents);
     95}
     96EXPORT_SYMBOL_GPL(sg_free_table_chained);
     97
     98/**
     99 * sg_alloc_table_chained - Allocate and chain SGLs in an sg table
    100 * @table:	The sg table header to use
    101 * @nents:	Number of entries in sg list
    102 * @first_chunk: first SGL
    103 * @nents_first_chunk: number of the SGL of @first_chunk
    104 *
    105 *  Description:
    106 *    Allocate and chain SGLs in an sg table. If @nents@ is larger than
    107 *    @nents_first_chunk a chained sg table will be setup. @first_chunk is
    108 *    ignored if nents_first_chunk <= 1 because user expects the SGL points
    109 *    non-chain SGL.
    110 *
    111 **/
    112int sg_alloc_table_chained(struct sg_table *table, int nents,
    113		struct scatterlist *first_chunk, unsigned nents_first_chunk)
    114{
    115	int ret;
    116
    117	BUG_ON(!nents);
    118
    119	if (first_chunk && nents_first_chunk) {
    120		if (nents <= nents_first_chunk) {
    121			table->nents = table->orig_nents = nents;
    122			sg_init_table(table->sgl, nents);
    123			return 0;
    124		}
    125	}
    126
    127	/* User supposes that the 1st SGL includes real entry */
    128	if (nents_first_chunk <= 1) {
    129		first_chunk = NULL;
    130		nents_first_chunk = 0;
    131	}
    132
    133	ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE,
    134			       first_chunk, nents_first_chunk,
    135			       GFP_ATOMIC, sg_pool_alloc);
    136	if (unlikely(ret))
    137		sg_free_table_chained(table, nents_first_chunk);
    138	return ret;
    139}
    140EXPORT_SYMBOL_GPL(sg_alloc_table_chained);
    141
    142static __init int sg_pool_init(void)
    143{
    144	int i;
    145
    146	for (i = 0; i < SG_MEMPOOL_NR; i++) {
    147		struct sg_pool *sgp = sg_pools + i;
    148		int size = sgp->size * sizeof(struct scatterlist);
    149
    150		sgp->slab = kmem_cache_create(sgp->name, size, 0,
    151				SLAB_HWCACHE_ALIGN, NULL);
    152		if (!sgp->slab) {
    153			printk(KERN_ERR "SG_POOL: can't init sg slab %s\n",
    154					sgp->name);
    155			goto cleanup_sdb;
    156		}
    157
    158		sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
    159						     sgp->slab);
    160		if (!sgp->pool) {
    161			printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n",
    162					sgp->name);
    163			goto cleanup_sdb;
    164		}
    165	}
    166
    167	return 0;
    168
    169cleanup_sdb:
    170	for (i = 0; i < SG_MEMPOOL_NR; i++) {
    171		struct sg_pool *sgp = sg_pools + i;
    172
    173		mempool_destroy(sgp->pool);
    174		kmem_cache_destroy(sgp->slab);
    175	}
    176
    177	return -ENOMEM;
    178}
    179
    180static __exit void sg_pool_exit(void)
    181{
    182	int i;
    183
    184	for (i = 0; i < SG_MEMPOOL_NR; i++) {
    185		struct sg_pool *sgp = sg_pools + i;
    186		mempool_destroy(sgp->pool);
    187		kmem_cache_destroy(sgp->slab);
    188	}
    189}
    190
    191module_init(sg_pool_init);
    192module_exit(sg_pool_exit);