uio.h (9430B)
1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * Berkeley style UIO structures - Alan Cox 1994. 4 */ 5#ifndef __LINUX_UIO_H 6#define __LINUX_UIO_H 7 8#include <linux/kernel.h> 9#include <linux/thread_info.h> 10#include <linux/mm_types.h> 11#include <uapi/linux/uio.h> 12 13struct page; 14struct pipe_inode_info; 15 16struct kvec { 17 void *iov_base; /* and that should *never* hold a userland pointer */ 18 size_t iov_len; 19}; 20 21enum iter_type { 22 /* iter types */ 23 ITER_IOVEC, 24 ITER_KVEC, 25 ITER_BVEC, 26 ITER_PIPE, 27 ITER_XARRAY, 28 ITER_DISCARD, 29}; 30 31struct iov_iter_state { 32 size_t iov_offset; 33 size_t count; 34 unsigned long nr_segs; 35}; 36 37struct iov_iter { 38 u8 iter_type; 39 bool nofault; 40 bool data_source; 41 size_t iov_offset; 42 size_t count; 43 union { 44 const struct iovec *iov; 45 const struct kvec *kvec; 46 const struct bio_vec *bvec; 47 struct xarray *xarray; 48 struct pipe_inode_info *pipe; 49 }; 50 union { 51 unsigned long nr_segs; 52 struct { 53 unsigned int head; 54 unsigned int start_head; 55 }; 56 loff_t xarray_start; 57 }; 58}; 59 60static inline enum iter_type iov_iter_type(const struct iov_iter *i) 61{ 62 return i->iter_type; 63} 64 65static inline void iov_iter_save_state(struct iov_iter *iter, 66 struct iov_iter_state *state) 67{ 68 state->iov_offset = iter->iov_offset; 69 state->count = iter->count; 70 state->nr_segs = iter->nr_segs; 71} 72 73static inline bool iter_is_iovec(const struct iov_iter *i) 74{ 75 return iov_iter_type(i) == ITER_IOVEC; 76} 77 78static inline bool iov_iter_is_kvec(const struct iov_iter *i) 79{ 80 return iov_iter_type(i) == ITER_KVEC; 81} 82 83static inline bool iov_iter_is_bvec(const struct iov_iter *i) 84{ 85 return iov_iter_type(i) == ITER_BVEC; 86} 87 88static inline bool iov_iter_is_pipe(const struct iov_iter *i) 89{ 90 return iov_iter_type(i) == ITER_PIPE; 91} 92 93static inline bool iov_iter_is_discard(const struct iov_iter *i) 94{ 95 return iov_iter_type(i) == ITER_DISCARD; 96} 97 98static inline bool iov_iter_is_xarray(const struct iov_iter *i) 99{ 100 return iov_iter_type(i) == ITER_XARRAY; 101} 102 103static inline unsigned char iov_iter_rw(const struct iov_iter *i) 104{ 105 return i->data_source ? WRITE : READ; 106} 107 108/* 109 * Total number of bytes covered by an iovec. 110 * 111 * NOTE that it is not safe to use this function until all the iovec's 112 * segment lengths have been validated. Because the individual lengths can 113 * overflow a size_t when added together. 114 */ 115static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs) 116{ 117 unsigned long seg; 118 size_t ret = 0; 119 120 for (seg = 0; seg < nr_segs; seg++) 121 ret += iov[seg].iov_len; 122 return ret; 123} 124 125static inline struct iovec iov_iter_iovec(const struct iov_iter *iter) 126{ 127 return (struct iovec) { 128 .iov_base = iter->iov->iov_base + iter->iov_offset, 129 .iov_len = min(iter->count, 130 iter->iov->iov_len - iter->iov_offset), 131 }; 132} 133 134size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, 135 size_t bytes, struct iov_iter *i); 136void iov_iter_advance(struct iov_iter *i, size_t bytes); 137void iov_iter_revert(struct iov_iter *i, size_t bytes); 138size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes); 139size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t bytes); 140size_t iov_iter_single_seg_count(const struct iov_iter *i); 141size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 142 struct iov_iter *i); 143size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 144 struct iov_iter *i); 145 146size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i); 147size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i); 148size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i); 149 150static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset, 151 size_t bytes, struct iov_iter *i) 152{ 153 return copy_page_to_iter(&folio->page, offset, bytes, i); 154} 155 156static __always_inline __must_check 157size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 158{ 159 if (unlikely(!check_copy_size(addr, bytes, true))) 160 return 0; 161 else 162 return _copy_to_iter(addr, bytes, i); 163} 164 165static __always_inline __must_check 166size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 167{ 168 if (unlikely(!check_copy_size(addr, bytes, false))) 169 return 0; 170 else 171 return _copy_from_iter(addr, bytes, i); 172} 173 174static __always_inline __must_check 175bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) 176{ 177 size_t copied = copy_from_iter(addr, bytes, i); 178 if (likely(copied == bytes)) 179 return true; 180 iov_iter_revert(i, copied); 181 return false; 182} 183 184static __always_inline __must_check 185size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 186{ 187 if (unlikely(!check_copy_size(addr, bytes, false))) 188 return 0; 189 else 190 return _copy_from_iter_nocache(addr, bytes, i); 191} 192 193static __always_inline __must_check 194bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) 195{ 196 size_t copied = copy_from_iter_nocache(addr, bytes, i); 197 if (likely(copied == bytes)) 198 return true; 199 iov_iter_revert(i, copied); 200 return false; 201} 202 203#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 204/* 205 * Note, users like pmem that depend on the stricter semantics of 206 * _copy_from_iter_flushcache() than _copy_from_iter_nocache() must check for 207 * IS_ENABLED(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) before assuming that the 208 * destination is flushed from the cache on return. 209 */ 210size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i); 211#else 212#define _copy_from_iter_flushcache _copy_from_iter_nocache 213#endif 214 215#ifdef CONFIG_ARCH_HAS_COPY_MC 216size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i); 217#else 218#define _copy_mc_to_iter _copy_to_iter 219#endif 220 221size_t iov_iter_zero(size_t bytes, struct iov_iter *); 222unsigned long iov_iter_alignment(const struct iov_iter *i); 223unsigned long iov_iter_gap_alignment(const struct iov_iter *i); 224void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov, 225 unsigned long nr_segs, size_t count); 226void iov_iter_kvec(struct iov_iter *i, unsigned int direction, const struct kvec *kvec, 227 unsigned long nr_segs, size_t count); 228void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_vec *bvec, 229 unsigned long nr_segs, size_t count); 230void iov_iter_pipe(struct iov_iter *i, unsigned int direction, struct pipe_inode_info *pipe, 231 size_t count); 232void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count); 233void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray, 234 loff_t start, size_t count); 235ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages, 236 size_t maxsize, unsigned maxpages, size_t *start); 237ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages, 238 size_t maxsize, size_t *start); 239int iov_iter_npages(const struct iov_iter *i, int maxpages); 240void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state); 241 242const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags); 243 244static inline size_t iov_iter_count(const struct iov_iter *i) 245{ 246 return i->count; 247} 248 249/* 250 * Cap the iov_iter by given limit; note that the second argument is 251 * *not* the new size - it's upper limit for such. Passing it a value 252 * greater than the amount of data in iov_iter is fine - it'll just do 253 * nothing in that case. 254 */ 255static inline void iov_iter_truncate(struct iov_iter *i, u64 count) 256{ 257 /* 258 * count doesn't have to fit in size_t - comparison extends both 259 * operands to u64 here and any value that would be truncated by 260 * conversion in assignement is by definition greater than all 261 * values of size_t, including old i->count. 262 */ 263 if (i->count > count) 264 i->count = count; 265} 266 267/* 268 * reexpand a previously truncated iterator; count must be no more than how much 269 * we had shrunk it. 270 */ 271static inline void iov_iter_reexpand(struct iov_iter *i, size_t count) 272{ 273 i->count = count; 274} 275 276static inline int 277iov_iter_npages_cap(struct iov_iter *i, int maxpages, size_t max_bytes) 278{ 279 size_t shorted = 0; 280 int npages; 281 282 if (iov_iter_count(i) > max_bytes) { 283 shorted = iov_iter_count(i) - max_bytes; 284 iov_iter_truncate(i, max_bytes); 285 } 286 npages = iov_iter_npages(i, INT_MAX); 287 if (shorted) 288 iov_iter_reexpand(i, iov_iter_count(i) + shorted); 289 290 return npages; 291} 292 293struct csum_state { 294 __wsum csum; 295 size_t off; 296}; 297 298size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csstate, struct iov_iter *i); 299size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); 300 301static __always_inline __must_check 302bool csum_and_copy_from_iter_full(void *addr, size_t bytes, 303 __wsum *csum, struct iov_iter *i) 304{ 305 size_t copied = csum_and_copy_from_iter(addr, bytes, csum, i); 306 if (likely(copied == bytes)) 307 return true; 308 iov_iter_revert(i, copied); 309 return false; 310} 311size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, 312 struct iov_iter *i); 313 314struct iovec *iovec_from_user(const struct iovec __user *uvector, 315 unsigned long nr_segs, unsigned long fast_segs, 316 struct iovec *fast_iov, bool compat); 317ssize_t import_iovec(int type, const struct iovec __user *uvec, 318 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, 319 struct iov_iter *i); 320ssize_t __import_iovec(int type, const struct iovec __user *uvec, 321 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, 322 struct iov_iter *i, bool compat); 323int import_single_range(int type, void __user *buf, size_t len, 324 struct iovec *iov, struct iov_iter *i); 325 326#endif