cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

io.c (13201B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * Alpha IO and memory functions.
      4 */
      5
      6#include <linux/kernel.h>
      7#include <linux/types.h>
      8#include <linux/string.h>
      9#include <linux/module.h>
     10#include <asm/io.h>
     11
     12/* Out-of-line versions of the i/o routines that redirect into the 
     13   platform-specific version.  Note that "platform-specific" may mean
     14   "generic", which bumps through the machine vector.  */
     15
     16unsigned int
     17ioread8(const void __iomem *addr)
     18{
     19	unsigned int ret;
     20	mb();
     21	ret = IO_CONCAT(__IO_PREFIX,ioread8)(addr);
     22	mb();
     23	return ret;
     24}
     25
     26unsigned int ioread16(const void __iomem *addr)
     27{
     28	unsigned int ret;
     29	mb();
     30	ret = IO_CONCAT(__IO_PREFIX,ioread16)(addr);
     31	mb();
     32	return ret;
     33}
     34
     35unsigned int ioread32(const void __iomem *addr)
     36{
     37	unsigned int ret;
     38	mb();
     39	ret = IO_CONCAT(__IO_PREFIX,ioread32)(addr);
     40	mb();
     41	return ret;
     42}
     43
     44void iowrite8(u8 b, void __iomem *addr)
     45{
     46	mb();
     47	IO_CONCAT(__IO_PREFIX,iowrite8)(b, addr);
     48}
     49
     50void iowrite16(u16 b, void __iomem *addr)
     51{
     52	mb();
     53	IO_CONCAT(__IO_PREFIX,iowrite16)(b, addr);
     54}
     55
     56void iowrite32(u32 b, void __iomem *addr)
     57{
     58	mb();
     59	IO_CONCAT(__IO_PREFIX,iowrite32)(b, addr);
     60}
     61
     62EXPORT_SYMBOL(ioread8);
     63EXPORT_SYMBOL(ioread16);
     64EXPORT_SYMBOL(ioread32);
     65EXPORT_SYMBOL(iowrite8);
     66EXPORT_SYMBOL(iowrite16);
     67EXPORT_SYMBOL(iowrite32);
     68
     69u8 inb(unsigned long port)
     70{
     71	return ioread8(ioport_map(port, 1));
     72}
     73
     74u16 inw(unsigned long port)
     75{
     76	return ioread16(ioport_map(port, 2));
     77}
     78
     79u32 inl(unsigned long port)
     80{
     81	return ioread32(ioport_map(port, 4));
     82}
     83
     84void outb(u8 b, unsigned long port)
     85{
     86	iowrite8(b, ioport_map(port, 1));
     87}
     88
     89void outw(u16 b, unsigned long port)
     90{
     91	iowrite16(b, ioport_map(port, 2));
     92}
     93
     94void outl(u32 b, unsigned long port)
     95{
     96	iowrite32(b, ioport_map(port, 4));
     97}
     98
     99EXPORT_SYMBOL(inb);
    100EXPORT_SYMBOL(inw);
    101EXPORT_SYMBOL(inl);
    102EXPORT_SYMBOL(outb);
    103EXPORT_SYMBOL(outw);
    104EXPORT_SYMBOL(outl);
    105
    106u8 __raw_readb(const volatile void __iomem *addr)
    107{
    108	return IO_CONCAT(__IO_PREFIX,readb)(addr);
    109}
    110
    111u16 __raw_readw(const volatile void __iomem *addr)
    112{
    113	return IO_CONCAT(__IO_PREFIX,readw)(addr);
    114}
    115
    116u32 __raw_readl(const volatile void __iomem *addr)
    117{
    118	return IO_CONCAT(__IO_PREFIX,readl)(addr);
    119}
    120
    121u64 __raw_readq(const volatile void __iomem *addr)
    122{
    123	return IO_CONCAT(__IO_PREFIX,readq)(addr);
    124}
    125
    126void __raw_writeb(u8 b, volatile void __iomem *addr)
    127{
    128	IO_CONCAT(__IO_PREFIX,writeb)(b, addr);
    129}
    130
    131void __raw_writew(u16 b, volatile void __iomem *addr)
    132{
    133	IO_CONCAT(__IO_PREFIX,writew)(b, addr);
    134}
    135
    136void __raw_writel(u32 b, volatile void __iomem *addr)
    137{
    138	IO_CONCAT(__IO_PREFIX,writel)(b, addr);
    139}
    140
    141void __raw_writeq(u64 b, volatile void __iomem *addr)
    142{
    143	IO_CONCAT(__IO_PREFIX,writeq)(b, addr);
    144}
    145
    146EXPORT_SYMBOL(__raw_readb); 
    147EXPORT_SYMBOL(__raw_readw); 
    148EXPORT_SYMBOL(__raw_readl); 
    149EXPORT_SYMBOL(__raw_readq); 
    150EXPORT_SYMBOL(__raw_writeb); 
    151EXPORT_SYMBOL(__raw_writew); 
    152EXPORT_SYMBOL(__raw_writel); 
    153EXPORT_SYMBOL(__raw_writeq); 
    154
    155u8 readb(const volatile void __iomem *addr)
    156{
    157	u8 ret;
    158	mb();
    159	ret = __raw_readb(addr);
    160	mb();
    161	return ret;
    162}
    163
    164u16 readw(const volatile void __iomem *addr)
    165{
    166	u16 ret;
    167	mb();
    168	ret = __raw_readw(addr);
    169	mb();
    170	return ret;
    171}
    172
    173u32 readl(const volatile void __iomem *addr)
    174{
    175	u32 ret;
    176	mb();
    177	ret = __raw_readl(addr);
    178	mb();
    179	return ret;
    180}
    181
    182u64 readq(const volatile void __iomem *addr)
    183{
    184	u64 ret;
    185	mb();
    186	ret = __raw_readq(addr);
    187	mb();
    188	return ret;
    189}
    190
    191void writeb(u8 b, volatile void __iomem *addr)
    192{
    193	mb();
    194	__raw_writeb(b, addr);
    195}
    196
    197void writew(u16 b, volatile void __iomem *addr)
    198{
    199	mb();
    200	__raw_writew(b, addr);
    201}
    202
    203void writel(u32 b, volatile void __iomem *addr)
    204{
    205	mb();
    206	__raw_writel(b, addr);
    207}
    208
    209void writeq(u64 b, volatile void __iomem *addr)
    210{
    211	mb();
    212	__raw_writeq(b, addr);
    213}
    214
    215EXPORT_SYMBOL(readb);
    216EXPORT_SYMBOL(readw);
    217EXPORT_SYMBOL(readl);
    218EXPORT_SYMBOL(readq);
    219EXPORT_SYMBOL(writeb);
    220EXPORT_SYMBOL(writew);
    221EXPORT_SYMBOL(writel);
    222EXPORT_SYMBOL(writeq);
    223
    224/*
    225 * The _relaxed functions must be ordered w.r.t. each other, but they don't
    226 * have to be ordered w.r.t. other memory accesses.
    227 */
    228u8 readb_relaxed(const volatile void __iomem *addr)
    229{
    230	mb();
    231	return __raw_readb(addr);
    232}
    233
    234u16 readw_relaxed(const volatile void __iomem *addr)
    235{
    236	mb();
    237	return __raw_readw(addr);
    238}
    239
    240u32 readl_relaxed(const volatile void __iomem *addr)
    241{
    242	mb();
    243	return __raw_readl(addr);
    244}
    245
    246u64 readq_relaxed(const volatile void __iomem *addr)
    247{
    248	mb();
    249	return __raw_readq(addr);
    250}
    251
    252EXPORT_SYMBOL(readb_relaxed);
    253EXPORT_SYMBOL(readw_relaxed);
    254EXPORT_SYMBOL(readl_relaxed);
    255EXPORT_SYMBOL(readq_relaxed);
    256
    257/*
    258 * Read COUNT 8-bit bytes from port PORT into memory starting at SRC.
    259 */
    260void ioread8_rep(const void __iomem *port, void *dst, unsigned long count)
    261{
    262	while ((unsigned long)dst & 0x3) {
    263		if (!count)
    264			return;
    265		count--;
    266		*(unsigned char *)dst = ioread8(port);
    267		dst += 1;
    268	}
    269
    270	while (count >= 4) {
    271		unsigned int w;
    272		count -= 4;
    273		w = ioread8(port);
    274		w |= ioread8(port) << 8;
    275		w |= ioread8(port) << 16;
    276		w |= ioread8(port) << 24;
    277		*(unsigned int *)dst = w;
    278		dst += 4;
    279	}
    280
    281	while (count) {
    282		--count;
    283		*(unsigned char *)dst = ioread8(port);
    284		dst += 1;
    285	}
    286}
    287
    288void insb(unsigned long port, void *dst, unsigned long count)
    289{
    290	ioread8_rep(ioport_map(port, 1), dst, count);
    291}
    292
    293EXPORT_SYMBOL(ioread8_rep);
    294EXPORT_SYMBOL(insb);
    295
    296/*
    297 * Read COUNT 16-bit words from port PORT into memory starting at
    298 * SRC.  SRC must be at least short aligned.  This is used by the
    299 * IDE driver to read disk sectors.  Performance is important, but
    300 * the interfaces seems to be slow: just using the inlined version
    301 * of the inw() breaks things.
    302 */
    303void ioread16_rep(const void __iomem *port, void *dst, unsigned long count)
    304{
    305	if (unlikely((unsigned long)dst & 0x3)) {
    306		if (!count)
    307			return;
    308		BUG_ON((unsigned long)dst & 0x1);
    309		count--;
    310		*(unsigned short *)dst = ioread16(port);
    311		dst += 2;
    312	}
    313
    314	while (count >= 2) {
    315		unsigned int w;
    316		count -= 2;
    317		w = ioread16(port);
    318		w |= ioread16(port) << 16;
    319		*(unsigned int *)dst = w;
    320		dst += 4;
    321	}
    322
    323	if (count) {
    324		*(unsigned short*)dst = ioread16(port);
    325	}
    326}
    327
    328void insw(unsigned long port, void *dst, unsigned long count)
    329{
    330	ioread16_rep(ioport_map(port, 2), dst, count);
    331}
    332
    333EXPORT_SYMBOL(ioread16_rep);
    334EXPORT_SYMBOL(insw);
    335
    336
    337/*
    338 * Read COUNT 32-bit words from port PORT into memory starting at
    339 * SRC. Now works with any alignment in SRC. Performance is important,
    340 * but the interfaces seems to be slow: just using the inlined version
    341 * of the inl() breaks things.
    342 */
    343void ioread32_rep(const void __iomem *port, void *dst, unsigned long count)
    344{
    345	if (unlikely((unsigned long)dst & 0x3)) {
    346		while (count--) {
    347			struct S { int x __attribute__((packed)); };
    348			((struct S *)dst)->x = ioread32(port);
    349			dst += 4;
    350		}
    351	} else {
    352		/* Buffer 32-bit aligned.  */
    353		while (count--) {
    354			*(unsigned int *)dst = ioread32(port);
    355			dst += 4;
    356		}
    357	}
    358}
    359
    360void insl(unsigned long port, void *dst, unsigned long count)
    361{
    362	ioread32_rep(ioport_map(port, 4), dst, count);
    363}
    364
    365EXPORT_SYMBOL(ioread32_rep);
    366EXPORT_SYMBOL(insl);
    367
    368
    369/*
    370 * Like insb but in the opposite direction.
    371 * Don't worry as much about doing aligned memory transfers:
    372 * doing byte reads the "slow" way isn't nearly as slow as
    373 * doing byte writes the slow way (no r-m-w cycle).
    374 */
    375void iowrite8_rep(void __iomem *port, const void *xsrc, unsigned long count)
    376{
    377	const unsigned char *src = xsrc;
    378	while (count--)
    379		iowrite8(*src++, port);
    380}
    381
    382void outsb(unsigned long port, const void *src, unsigned long count)
    383{
    384	iowrite8_rep(ioport_map(port, 1), src, count);
    385}
    386
    387EXPORT_SYMBOL(iowrite8_rep);
    388EXPORT_SYMBOL(outsb);
    389
    390
    391/*
    392 * Like insw but in the opposite direction.  This is used by the IDE
    393 * driver to write disk sectors.  Performance is important, but the
    394 * interfaces seems to be slow: just using the inlined version of the
    395 * outw() breaks things.
    396 */
    397void iowrite16_rep(void __iomem *port, const void *src, unsigned long count)
    398{
    399	if (unlikely((unsigned long)src & 0x3)) {
    400		if (!count)
    401			return;
    402		BUG_ON((unsigned long)src & 0x1);
    403		iowrite16(*(unsigned short *)src, port);
    404		src += 2;
    405		--count;
    406	}
    407
    408	while (count >= 2) {
    409		unsigned int w;
    410		count -= 2;
    411		w = *(unsigned int *)src;
    412		src += 4;
    413		iowrite16(w >>  0, port);
    414		iowrite16(w >> 16, port);
    415	}
    416
    417	if (count) {
    418		iowrite16(*(unsigned short *)src, port);
    419	}
    420}
    421
    422void outsw(unsigned long port, const void *src, unsigned long count)
    423{
    424	iowrite16_rep(ioport_map(port, 2), src, count);
    425}
    426
    427EXPORT_SYMBOL(iowrite16_rep);
    428EXPORT_SYMBOL(outsw);
    429
    430
    431/*
    432 * Like insl but in the opposite direction.  This is used by the IDE
    433 * driver to write disk sectors.  Works with any alignment in SRC.
    434 * Performance is important, but the interfaces seems to be slow:
    435 * just using the inlined version of the outl() breaks things.
    436 */
    437void iowrite32_rep(void __iomem *port, const void *src, unsigned long count)
    438{
    439	if (unlikely((unsigned long)src & 0x3)) {
    440		while (count--) {
    441			struct S { int x __attribute__((packed)); };
    442			iowrite32(((struct S *)src)->x, port);
    443			src += 4;
    444		}
    445	} else {
    446		/* Buffer 32-bit aligned.  */
    447		while (count--) {
    448			iowrite32(*(unsigned int *)src, port);
    449			src += 4;
    450		}
    451	}
    452}
    453
    454void outsl(unsigned long port, const void *src, unsigned long count)
    455{
    456	iowrite32_rep(ioport_map(port, 4), src, count);
    457}
    458
    459EXPORT_SYMBOL(iowrite32_rep);
    460EXPORT_SYMBOL(outsl);
    461
    462
    463/*
    464 * Copy data from IO memory space to "real" memory space.
    465 * This needs to be optimized.
    466 */
    467void memcpy_fromio(void *to, const volatile void __iomem *from, long count)
    468{
    469	/* Optimize co-aligned transfers.  Everything else gets handled
    470	   a byte at a time. */
    471
    472	if (count >= 8 && ((u64)to & 7) == ((u64)from & 7)) {
    473		count -= 8;
    474		do {
    475			*(u64 *)to = __raw_readq(from);
    476			count -= 8;
    477			to += 8;
    478			from += 8;
    479		} while (count >= 0);
    480		count += 8;
    481	}
    482
    483	if (count >= 4 && ((u64)to & 3) == ((u64)from & 3)) {
    484		count -= 4;
    485		do {
    486			*(u32 *)to = __raw_readl(from);
    487			count -= 4;
    488			to += 4;
    489			from += 4;
    490		} while (count >= 0);
    491		count += 4;
    492	}
    493
    494	if (count >= 2 && ((u64)to & 1) == ((u64)from & 1)) {
    495		count -= 2;
    496		do {
    497			*(u16 *)to = __raw_readw(from);
    498			count -= 2;
    499			to += 2;
    500			from += 2;
    501		} while (count >= 0);
    502		count += 2;
    503	}
    504
    505	while (count > 0) {
    506		*(u8 *) to = __raw_readb(from);
    507		count--;
    508		to++;
    509		from++;
    510	}
    511	mb();
    512}
    513
    514EXPORT_SYMBOL(memcpy_fromio);
    515
    516
    517/*
    518 * Copy data from "real" memory space to IO memory space.
    519 * This needs to be optimized.
    520 */
    521void memcpy_toio(volatile void __iomem *to, const void *from, long count)
    522{
    523	/* Optimize co-aligned transfers.  Everything else gets handled
    524	   a byte at a time. */
    525	/* FIXME -- align FROM.  */
    526
    527	if (count >= 8 && ((u64)to & 7) == ((u64)from & 7)) {
    528		count -= 8;
    529		do {
    530			__raw_writeq(*(const u64 *)from, to);
    531			count -= 8;
    532			to += 8;
    533			from += 8;
    534		} while (count >= 0);
    535		count += 8;
    536	}
    537
    538	if (count >= 4 && ((u64)to & 3) == ((u64)from & 3)) {
    539		count -= 4;
    540		do {
    541			__raw_writel(*(const u32 *)from, to);
    542			count -= 4;
    543			to += 4;
    544			from += 4;
    545		} while (count >= 0);
    546		count += 4;
    547	}
    548
    549	if (count >= 2 && ((u64)to & 1) == ((u64)from & 1)) {
    550		count -= 2;
    551		do {
    552			__raw_writew(*(const u16 *)from, to);
    553			count -= 2;
    554			to += 2;
    555			from += 2;
    556		} while (count >= 0);
    557		count += 2;
    558	}
    559
    560	while (count > 0) {
    561		__raw_writeb(*(const u8 *) from, to);
    562		count--;
    563		to++;
    564		from++;
    565	}
    566	mb();
    567}
    568
    569EXPORT_SYMBOL(memcpy_toio);
    570
    571
    572/*
    573 * "memset" on IO memory space.
    574 */
    575void _memset_c_io(volatile void __iomem *to, unsigned long c, long count)
    576{
    577	/* Handle any initial odd byte */
    578	if (count > 0 && ((u64)to & 1)) {
    579		__raw_writeb(c, to);
    580		to++;
    581		count--;
    582	}
    583
    584	/* Handle any initial odd halfword */
    585	if (count >= 2 && ((u64)to & 2)) {
    586		__raw_writew(c, to);
    587		to += 2;
    588		count -= 2;
    589	}
    590
    591	/* Handle any initial odd word */
    592	if (count >= 4 && ((u64)to & 4)) {
    593		__raw_writel(c, to);
    594		to += 4;
    595		count -= 4;
    596	}
    597
    598	/* Handle all full-sized quadwords: we're aligned
    599	   (or have a small count) */
    600	count -= 8;
    601	if (count >= 0) {
    602		do {
    603			__raw_writeq(c, to);
    604			to += 8;
    605			count -= 8;
    606		} while (count >= 0);
    607	}
    608	count += 8;
    609
    610	/* The tail is word-aligned if we still have count >= 4 */
    611	if (count >= 4) {
    612		__raw_writel(c, to);
    613		to += 4;
    614		count -= 4;
    615	}
    616
    617	/* The tail is half-word aligned if we have count >= 2 */
    618	if (count >= 2) {
    619		__raw_writew(c, to);
    620		to += 2;
    621		count -= 2;
    622	}
    623
    624	/* And finally, one last byte.. */
    625	if (count) {
    626		__raw_writeb(c, to);
    627	}
    628	mb();
    629}
    630
    631EXPORT_SYMBOL(_memset_c_io);
    632
    633/* A version of memcpy used by the vga console routines to move data around
    634   arbitrarily between screen and main memory.  */
    635
    636void
    637scr_memcpyw(u16 *d, const u16 *s, unsigned int count)
    638{
    639	const u16 __iomem *ios = (const u16 __iomem *) s;
    640	u16 __iomem *iod = (u16 __iomem *) d;
    641	int s_isio = __is_ioaddr(s);
    642	int d_isio = __is_ioaddr(d);
    643
    644	if (s_isio) {
    645		if (d_isio) {
    646			/* FIXME: Should handle unaligned ops and
    647			   operation widening.  */
    648
    649			count /= 2;
    650			while (count--) {
    651				u16 tmp = __raw_readw(ios++);
    652				__raw_writew(tmp, iod++);
    653			}
    654		}
    655		else
    656			memcpy_fromio(d, ios, count);
    657	} else {
    658		if (d_isio)
    659			memcpy_toio(iod, s, count);
    660		else
    661			memcpy(d, s, count);
    662	}
    663}
    664
    665EXPORT_SYMBOL(scr_memcpyw);
    666
    667void __iomem *ioport_map(unsigned long port, unsigned int size)
    668{
    669	return IO_CONCAT(__IO_PREFIX,ioportmap) (port);
    670}
    671
    672void ioport_unmap(void __iomem *addr)
    673{
    674}
    675
    676EXPORT_SYMBOL(ioport_map);
    677EXPORT_SYMBOL(ioport_unmap);