cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

uio.h (9430B)


      1/* SPDX-License-Identifier: GPL-2.0-or-later */
      2/*
      3 *	Berkeley style UIO structures	-	Alan Cox 1994.
      4 */
      5#ifndef __LINUX_UIO_H
      6#define __LINUX_UIO_H
      7
      8#include <linux/kernel.h>
      9#include <linux/thread_info.h>
     10#include <linux/mm_types.h>
     11#include <uapi/linux/uio.h>
     12
     13struct page;
     14struct pipe_inode_info;
     15
     16struct kvec {
     17	void *iov_base; /* and that should *never* hold a userland pointer */
     18	size_t iov_len;
     19};
     20
     21enum iter_type {
     22	/* iter types */
     23	ITER_IOVEC,
     24	ITER_KVEC,
     25	ITER_BVEC,
     26	ITER_PIPE,
     27	ITER_XARRAY,
     28	ITER_DISCARD,
     29};
     30
     31struct iov_iter_state {
     32	size_t iov_offset;
     33	size_t count;
     34	unsigned long nr_segs;
     35};
     36
     37struct iov_iter {
     38	u8 iter_type;
     39	bool nofault;
     40	bool data_source;
     41	size_t iov_offset;
     42	size_t count;
     43	union {
     44		const struct iovec *iov;
     45		const struct kvec *kvec;
     46		const struct bio_vec *bvec;
     47		struct xarray *xarray;
     48		struct pipe_inode_info *pipe;
     49	};
     50	union {
     51		unsigned long nr_segs;
     52		struct {
     53			unsigned int head;
     54			unsigned int start_head;
     55		};
     56		loff_t xarray_start;
     57	};
     58};
     59
     60static inline enum iter_type iov_iter_type(const struct iov_iter *i)
     61{
     62	return i->iter_type;
     63}
     64
     65static inline void iov_iter_save_state(struct iov_iter *iter,
     66				       struct iov_iter_state *state)
     67{
     68	state->iov_offset = iter->iov_offset;
     69	state->count = iter->count;
     70	state->nr_segs = iter->nr_segs;
     71}
     72
     73static inline bool iter_is_iovec(const struct iov_iter *i)
     74{
     75	return iov_iter_type(i) == ITER_IOVEC;
     76}
     77
     78static inline bool iov_iter_is_kvec(const struct iov_iter *i)
     79{
     80	return iov_iter_type(i) == ITER_KVEC;
     81}
     82
     83static inline bool iov_iter_is_bvec(const struct iov_iter *i)
     84{
     85	return iov_iter_type(i) == ITER_BVEC;
     86}
     87
     88static inline bool iov_iter_is_pipe(const struct iov_iter *i)
     89{
     90	return iov_iter_type(i) == ITER_PIPE;
     91}
     92
     93static inline bool iov_iter_is_discard(const struct iov_iter *i)
     94{
     95	return iov_iter_type(i) == ITER_DISCARD;
     96}
     97
     98static inline bool iov_iter_is_xarray(const struct iov_iter *i)
     99{
    100	return iov_iter_type(i) == ITER_XARRAY;
    101}
    102
    103static inline unsigned char iov_iter_rw(const struct iov_iter *i)
    104{
    105	return i->data_source ? WRITE : READ;
    106}
    107
    108/*
    109 * Total number of bytes covered by an iovec.
    110 *
    111 * NOTE that it is not safe to use this function until all the iovec's
    112 * segment lengths have been validated.  Because the individual lengths can
    113 * overflow a size_t when added together.
    114 */
    115static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
    116{
    117	unsigned long seg;
    118	size_t ret = 0;
    119
    120	for (seg = 0; seg < nr_segs; seg++)
    121		ret += iov[seg].iov_len;
    122	return ret;
    123}
    124
    125static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
    126{
    127	return (struct iovec) {
    128		.iov_base = iter->iov->iov_base + iter->iov_offset,
    129		.iov_len = min(iter->count,
    130			       iter->iov->iov_len - iter->iov_offset),
    131	};
    132}
    133
    134size_t copy_page_from_iter_atomic(struct page *page, unsigned offset,
    135				  size_t bytes, struct iov_iter *i);
    136void iov_iter_advance(struct iov_iter *i, size_t bytes);
    137void iov_iter_revert(struct iov_iter *i, size_t bytes);
    138size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes);
    139size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t bytes);
    140size_t iov_iter_single_seg_count(const struct iov_iter *i);
    141size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
    142			 struct iov_iter *i);
    143size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
    144			 struct iov_iter *i);
    145
    146size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
    147size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
    148size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
    149
    150static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset,
    151		size_t bytes, struct iov_iter *i)
    152{
    153	return copy_page_to_iter(&folio->page, offset, bytes, i);
    154}
    155
    156static __always_inline __must_check
    157size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
    158{
    159	if (unlikely(!check_copy_size(addr, bytes, true)))
    160		return 0;
    161	else
    162		return _copy_to_iter(addr, bytes, i);
    163}
    164
    165static __always_inline __must_check
    166size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
    167{
    168	if (unlikely(!check_copy_size(addr, bytes, false)))
    169		return 0;
    170	else
    171		return _copy_from_iter(addr, bytes, i);
    172}
    173
    174static __always_inline __must_check
    175bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
    176{
    177	size_t copied = copy_from_iter(addr, bytes, i);
    178	if (likely(copied == bytes))
    179		return true;
    180	iov_iter_revert(i, copied);
    181	return false;
    182}
    183
    184static __always_inline __must_check
    185size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
    186{
    187	if (unlikely(!check_copy_size(addr, bytes, false)))
    188		return 0;
    189	else
    190		return _copy_from_iter_nocache(addr, bytes, i);
    191}
    192
    193static __always_inline __must_check
    194bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
    195{
    196	size_t copied = copy_from_iter_nocache(addr, bytes, i);
    197	if (likely(copied == bytes))
    198		return true;
    199	iov_iter_revert(i, copied);
    200	return false;
    201}
    202
    203#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
    204/*
    205 * Note, users like pmem that depend on the stricter semantics of
    206 * _copy_from_iter_flushcache() than _copy_from_iter_nocache() must check for
    207 * IS_ENABLED(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) before assuming that the
    208 * destination is flushed from the cache on return.
    209 */
    210size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i);
    211#else
    212#define _copy_from_iter_flushcache _copy_from_iter_nocache
    213#endif
    214
    215#ifdef CONFIG_ARCH_HAS_COPY_MC
    216size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
    217#else
    218#define _copy_mc_to_iter _copy_to_iter
    219#endif
    220
    221size_t iov_iter_zero(size_t bytes, struct iov_iter *);
    222unsigned long iov_iter_alignment(const struct iov_iter *i);
    223unsigned long iov_iter_gap_alignment(const struct iov_iter *i);
    224void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov,
    225			unsigned long nr_segs, size_t count);
    226void iov_iter_kvec(struct iov_iter *i, unsigned int direction, const struct kvec *kvec,
    227			unsigned long nr_segs, size_t count);
    228void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_vec *bvec,
    229			unsigned long nr_segs, size_t count);
    230void iov_iter_pipe(struct iov_iter *i, unsigned int direction, struct pipe_inode_info *pipe,
    231			size_t count);
    232void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count);
    233void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray,
    234		     loff_t start, size_t count);
    235ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages,
    236			size_t maxsize, unsigned maxpages, size_t *start);
    237ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages,
    238			size_t maxsize, size_t *start);
    239int iov_iter_npages(const struct iov_iter *i, int maxpages);
    240void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state);
    241
    242const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags);
    243
    244static inline size_t iov_iter_count(const struct iov_iter *i)
    245{
    246	return i->count;
    247}
    248
    249/*
    250 * Cap the iov_iter by given limit; note that the second argument is
    251 * *not* the new size - it's upper limit for such.  Passing it a value
    252 * greater than the amount of data in iov_iter is fine - it'll just do
    253 * nothing in that case.
    254 */
    255static inline void iov_iter_truncate(struct iov_iter *i, u64 count)
    256{
    257	/*
    258	 * count doesn't have to fit in size_t - comparison extends both
    259	 * operands to u64 here and any value that would be truncated by
    260	 * conversion in assignement is by definition greater than all
    261	 * values of size_t, including old i->count.
    262	 */
    263	if (i->count > count)
    264		i->count = count;
    265}
    266
    267/*
    268 * reexpand a previously truncated iterator; count must be no more than how much
    269 * we had shrunk it.
    270 */
    271static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
    272{
    273	i->count = count;
    274}
    275
    276static inline int
    277iov_iter_npages_cap(struct iov_iter *i, int maxpages, size_t max_bytes)
    278{
    279	size_t shorted = 0;
    280	int npages;
    281
    282	if (iov_iter_count(i) > max_bytes) {
    283		shorted = iov_iter_count(i) - max_bytes;
    284		iov_iter_truncate(i, max_bytes);
    285	}
    286	npages = iov_iter_npages(i, INT_MAX);
    287	if (shorted)
    288		iov_iter_reexpand(i, iov_iter_count(i) + shorted);
    289
    290	return npages;
    291}
    292
    293struct csum_state {
    294	__wsum csum;
    295	size_t off;
    296};
    297
    298size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csstate, struct iov_iter *i);
    299size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
    300
    301static __always_inline __must_check
    302bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
    303				  __wsum *csum, struct iov_iter *i)
    304{
    305	size_t copied = csum_and_copy_from_iter(addr, bytes, csum, i);
    306	if (likely(copied == bytes))
    307		return true;
    308	iov_iter_revert(i, copied);
    309	return false;
    310}
    311size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
    312		struct iov_iter *i);
    313
    314struct iovec *iovec_from_user(const struct iovec __user *uvector,
    315		unsigned long nr_segs, unsigned long fast_segs,
    316		struct iovec *fast_iov, bool compat);
    317ssize_t import_iovec(int type, const struct iovec __user *uvec,
    318		 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
    319		 struct iov_iter *i);
    320ssize_t __import_iovec(int type, const struct iovec __user *uvec,
    321		 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
    322		 struct iov_iter *i, bool compat);
    323int import_single_range(int type, void __user *buf, size_t len,
    324		 struct iovec *iov, struct iov_iter *i);
    325
    326#endif