cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

s390vx.uc (3898B)

      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * raid6_vx$#.c
      4 *
      5 * $#-way unrolled RAID6 gen/xor functions for s390
      6 * based on the vector facility
      7 *
      8 * Copyright IBM Corp. 2016
      9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
     10 *
     11 * This file is postprocessed using unroll.awk.
     12 */
     13
     14#include <linux/raid/pq.h>
     15#include <asm/fpu/api.h>
     16
     17asm(".include \"asm/vx-insn.h\"\n");
     18
     19#define NSIZE 16
     20
     21static inline void LOAD_CONST(void)
     22{
     23	asm volatile("VREPIB %v24,7");
     24	asm volatile("VREPIB %v25,0x1d");
     25}
     26
     27/*
     28 * The SHLBYTE() operation shifts each of the 16 bytes in
     29 * vector register y left by 1 bit and stores the result in
     30 * vector register x.
     31 */
     32static inline void SHLBYTE(int x, int y)
     33{
     34	asm volatile ("VAB %0,%1,%1" : : "i" (x), "i" (y));
     35}
     36
     37/*
     38 * For each of the 16 bytes in the vector register y the MASK()
     39 * operation returns 0xFF if the high bit of the byte is 1,
     40 * or 0x00 if the high bit is 0. The result is stored in vector
     41 * register x.
     42 */
     43static inline void MASK(int x, int y)
     44{
     45	asm volatile ("VESRAVB	%0,%1,24" : : "i" (x), "i" (y));
     46}
     47
     48static inline void AND(int x, int y, int z)
     49{
     50	asm volatile ("VN %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
     51}
     52
     53static inline void XOR(int x, int y, int z)
     54{
     55	asm volatile ("VX %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
     56}
     57
     58static inline void LOAD_DATA(int x, u8 *ptr)
     59{
     60	typedef struct { u8 _[16 * $#]; } addrtype;
     61	register addrtype *__ptr asm("1") = (addrtype *) ptr;
     62
     63	asm volatile ("VLM %2,%3,0,%1"
     64		      : : "m" (*__ptr), "a" (__ptr), "i" (x),
     65			  "i" (x + $# - 1));
     66}
     67
     68static inline void STORE_DATA(int x, u8 *ptr)
     69{
     70	typedef struct { u8 _[16 * $#]; } addrtype;
     71	register addrtype *__ptr asm("1") = (addrtype *) ptr;
     72
     73	asm volatile ("VSTM %2,%3,0,1"
     74		      : "=m" (*__ptr) : "a" (__ptr), "i" (x),
     75			"i" (x + $# - 1));
     76}
     77
     78static inline void COPY_VEC(int x, int y)
     79{
     80	asm volatile ("VLR %0,%1" : : "i" (x), "i" (y));
     81}
     82
     83static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
     84{
     85	struct kernel_fpu vxstate;
     86	u8 **dptr, *p, *q;
     87	int d, z, z0;
     88
     89	kernel_fpu_begin(&vxstate, KERNEL_VXR);
     90	LOAD_CONST();
     91
     92	dptr = (u8 **) ptrs;
     93	z0 = disks - 3;		/* Highest data disk */
     94	p = dptr[z0 + 1];	/* XOR parity */
     95	q = dptr[z0 + 2];	/* RS syndrome */
     96
     97	for (d = 0; d < bytes; d += $#*NSIZE) {
     98		LOAD_DATA(0,&dptr[z0][d]);
     99		COPY_VEC(8+$$,0+$$);
    100		for (z = z0 - 1; z >= 0; z--) {
    101			MASK(16+$$,8+$$);
    102			AND(16+$$,16+$$,25);
    103			SHLBYTE(8+$$,8+$$);
    104			XOR(8+$$,8+$$,16+$$);
    105			LOAD_DATA(16,&dptr[z][d]);
    106			XOR(0+$$,0+$$,16+$$);
    107			XOR(8+$$,8+$$,16+$$);
    108		}
    109		STORE_DATA(0,&p[d]);
    110		STORE_DATA(8,&q[d]);
    111	}
    112	kernel_fpu_end(&vxstate, KERNEL_VXR);
    113}
    114
    115static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop,
    116					size_t bytes, void **ptrs)
    117{
    118	struct kernel_fpu vxstate;
    119	u8 **dptr, *p, *q;
    120	int d, z, z0;
    121
    122	dptr = (u8 **) ptrs;
    123	z0 = stop;		/* P/Q right side optimization */
    124	p = dptr[disks - 2];	/* XOR parity */
    125	q = dptr[disks - 1];	/* RS syndrome */
    126
    127	kernel_fpu_begin(&vxstate, KERNEL_VXR);
    128	LOAD_CONST();
    129
    130	for (d = 0; d < bytes; d += $#*NSIZE) {
    131		/* P/Q data pages */
    132		LOAD_DATA(0,&dptr[z0][d]);
    133		COPY_VEC(8+$$,0+$$);
    134		for (z = z0 - 1; z >= start; z--) {
    135			MASK(16+$$,8+$$);
    136			AND(16+$$,16+$$,25);
    137			SHLBYTE(8+$$,8+$$);
    138			XOR(8+$$,8+$$,16+$$);
    139			LOAD_DATA(16,&dptr[z][d]);
    140			XOR(0+$$,0+$$,16+$$);
    141			XOR(8+$$,8+$$,16+$$);
    142		}
    143		/* P/Q left side optimization */
    144		for (z = start - 1; z >= 0; z--) {
    145			MASK(16+$$,8+$$);
    146			AND(16+$$,16+$$,25);
    147			SHLBYTE(8+$$,8+$$);
    148			XOR(8+$$,8+$$,16+$$);
    149		}
    150		LOAD_DATA(16,&p[d]);
    151		XOR(16+$$,16+$$,0+$$);
    152		STORE_DATA(16,&p[d]);
    153		LOAD_DATA(16,&q[d]);
    154		XOR(16+$$,16+$$,8+$$);
    155		STORE_DATA(16,&q[d]);
    156	}
    157	kernel_fpu_end(&vxstate, KERNEL_VXR);
    158}
    159
    160static int raid6_s390vx$#_valid(void)
    161{
    162	return MACHINE_HAS_VX;
    163}
    164
    165const struct raid6_calls raid6_s390vx$# = {
    166	raid6_s390vx$#_gen_syndrome,
    167	raid6_s390vx$#_xor_syndrome,
    168	raid6_s390vx$#_valid,
    169	"vx128x$#",
    170	1
    171};