cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

mmx.c (3637B)

      1// SPDX-License-Identifier: GPL-2.0-or-later
      2/* -*- linux-c -*- ------------------------------------------------------- *
      3 *
      4 *   Copyright 2002 H. Peter Anvin - All Rights Reserved
      5 *
      6 * ----------------------------------------------------------------------- */
      7
      8/*
      9 * raid6/mmx.c
     10 *
     11 * MMX implementation of RAID-6 syndrome functions
     12 */
     13
     14#ifdef CONFIG_X86_32
     15
     16#include <linux/raid/pq.h>
     17#include "x86.h"
     18
     19/* Shared with raid6/sse1.c */
     20const struct raid6_mmx_constants {
     21	u64 x1d;
     22} raid6_mmx_constants = {
     23	0x1d1d1d1d1d1d1d1dULL,
     24};
     25
     26static int raid6_have_mmx(void)
     27{
     28	/* Not really "boot_cpu" but "all_cpus" */
     29	return boot_cpu_has(X86_FEATURE_MMX);
     30}
     31
     32/*
     33 * Plain MMX implementation
     34 */
     35static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
     36{
     37	u8 **dptr = (u8 **)ptrs;
     38	u8 *p, *q;
     39	int d, z, z0;
     40
     41	z0 = disks - 3;		/* Highest data disk */
     42	p = dptr[z0+1];		/* XOR parity */
     43	q = dptr[z0+2];		/* RS syndrome */
     44
     45	kernel_fpu_begin();
     46
     47	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
     48	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
     49
     50	for ( d = 0 ; d < bytes ; d += 8 ) {
     51		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
     52		asm volatile("movq %mm2,%mm4");	/* Q[0] */
     53		for ( z = z0-1 ; z >= 0 ; z-- ) {
     54			asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
     55			asm volatile("pcmpgtb %mm4,%mm5");
     56			asm volatile("paddb %mm4,%mm4");
     57			asm volatile("pand %mm0,%mm5");
     58			asm volatile("pxor %mm5,%mm4");
     59			asm volatile("pxor %mm5,%mm5");
     60			asm volatile("pxor %mm6,%mm2");
     61			asm volatile("pxor %mm6,%mm4");
     62		}
     63		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
     64		asm volatile("pxor %mm2,%mm2");
     65		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
     66		asm volatile("pxor %mm4,%mm4");
     67	}
     68
     69	kernel_fpu_end();
     70}
     71
     72const struct raid6_calls raid6_mmxx1 = {
     73	raid6_mmx1_gen_syndrome,
     74	NULL,			/* XOR not yet implemented */
     75	raid6_have_mmx,
     76	"mmxx1",
     77	0
     78};
     79
     80/*
     81 * Unrolled-by-2 MMX implementation
     82 */
     83static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
     84{
     85	u8 **dptr = (u8 **)ptrs;
     86	u8 *p, *q;
     87	int d, z, z0;
     88
     89	z0 = disks - 3;		/* Highest data disk */
     90	p = dptr[z0+1];		/* XOR parity */
     91	q = dptr[z0+2];		/* RS syndrome */
     92
     93	kernel_fpu_begin();
     94
     95	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
     96	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
     97	asm volatile("pxor %mm7,%mm7"); /* Zero temp */
     98
     99	for ( d = 0 ; d < bytes ; d += 16 ) {
    100		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
    101		asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
    102		asm volatile("movq %mm2,%mm4"); /* Q[0] */
    103		asm volatile("movq %mm3,%mm6"); /* Q[1] */
    104		for ( z = z0-1 ; z >= 0 ; z-- ) {
    105			asm volatile("pcmpgtb %mm4,%mm5");
    106			asm volatile("pcmpgtb %mm6,%mm7");
    107			asm volatile("paddb %mm4,%mm4");
    108			asm volatile("paddb %mm6,%mm6");
    109			asm volatile("pand %mm0,%mm5");
    110			asm volatile("pand %mm0,%mm7");
    111			asm volatile("pxor %mm5,%mm4");
    112			asm volatile("pxor %mm7,%mm6");
    113			asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
    114			asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
    115			asm volatile("pxor %mm5,%mm2");
    116			asm volatile("pxor %mm7,%mm3");
    117			asm volatile("pxor %mm5,%mm4");
    118			asm volatile("pxor %mm7,%mm6");
    119			asm volatile("pxor %mm5,%mm5");
    120			asm volatile("pxor %mm7,%mm7");
    121		}
    122		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
    123		asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
    124		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
    125		asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
    126	}
    127
    128	kernel_fpu_end();
    129}
    130
    131const struct raid6_calls raid6_mmxx2 = {
    132	raid6_mmx2_gen_syndrome,
    133	NULL,			/* XOR not yet implemented */
    134	raid6_have_mmx,
    135	"mmxx2",
    136	0
    137};
    138
    139#endif