cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

init.c (39011B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Intel I/OAT DMA Linux driver
      4 * Copyright(c) 2004 - 2015 Intel Corporation.
      5 */
      6
      7#include <linux/init.h>
      8#include <linux/module.h>
      9#include <linux/slab.h>
     10#include <linux/pci.h>
     11#include <linux/interrupt.h>
     12#include <linux/dmaengine.h>
     13#include <linux/delay.h>
     14#include <linux/dma-mapping.h>
     15#include <linux/workqueue.h>
     16#include <linux/prefetch.h>
     17#include <linux/dca.h>
     18#include <linux/aer.h>
     19#include <linux/sizes.h>
     20#include "dma.h"
     21#include "registers.h"
     22#include "hw.h"
     23
     24#include "../dmaengine.h"
     25
     26MODULE_VERSION(IOAT_DMA_VERSION);
     27MODULE_LICENSE("Dual BSD/GPL");
     28MODULE_AUTHOR("Intel Corporation");
     29
     30static const struct pci_device_id ioat_pci_tbl[] = {
     31	/* I/OAT v3 platforms */
     32	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
     33	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
     34	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
     35	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
     36	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
     37	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
     38	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
     39	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
     40
     41	/* I/OAT v3.2 platforms */
     42	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) },
     43	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) },
     44	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) },
     45	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) },
     46	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) },
     47	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) },
     48	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) },
     49	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) },
     50	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) },
     51	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) },
     52
     53	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) },
     54	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) },
     55	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) },
     56	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) },
     57	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) },
     58	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) },
     59	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) },
     60	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) },
     61	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) },
     62	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) },
     63
     64	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) },
     65	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) },
     66	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) },
     67	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) },
     68	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) },
     69	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) },
     70	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) },
     71	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) },
     72	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) },
     73	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) },
     74
     75	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) },
     76	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) },
     77	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) },
     78	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) },
     79	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) },
     80	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) },
     81	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) },
     82	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) },
     83	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) },
     84	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) },
     85
     86	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0) },
     87	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1) },
     88	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2) },
     89	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3) },
     90	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4) },
     91	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5) },
     92	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6) },
     93	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7) },
     94	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) },
     95	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) },
     96
     97	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SKX) },
     98
     99	/* I/OAT v3.3 platforms */
    100	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) },
    101	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) },
    102	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) },
    103	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) },
    104
    105	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) },
    106	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) },
    107	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) },
    108	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) },
    109
    110	/* I/OAT v3.4 platforms */
    111	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_ICX) },
    112
    113	{ 0, }
    114};
    115MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
    116
    117static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
    118static void ioat_remove(struct pci_dev *pdev);
    119static void
    120ioat_init_channel(struct ioatdma_device *ioat_dma,
    121		  struct ioatdma_chan *ioat_chan, int idx);
    122static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
    123static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
    124static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);
    125
    126static int ioat_dca_enabled = 1;
    127module_param(ioat_dca_enabled, int, 0644);
    128MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
    129int ioat_pending_level = 7;
    130module_param(ioat_pending_level, int, 0644);
    131MODULE_PARM_DESC(ioat_pending_level,
    132		 "high-water mark for pushing ioat descriptors (default: 7)");
    133static char ioat_interrupt_style[32] = "msix";
    134module_param_string(ioat_interrupt_style, ioat_interrupt_style,
    135		    sizeof(ioat_interrupt_style), 0644);
    136MODULE_PARM_DESC(ioat_interrupt_style,
    137		 "set ioat interrupt style: msix (default), msi, intx");
    138
    139struct kmem_cache *ioat_cache;
    140struct kmem_cache *ioat_sed_cache;
    141
    142static bool is_jf_ioat(struct pci_dev *pdev)
    143{
    144	switch (pdev->device) {
    145	case PCI_DEVICE_ID_INTEL_IOAT_JSF0:
    146	case PCI_DEVICE_ID_INTEL_IOAT_JSF1:
    147	case PCI_DEVICE_ID_INTEL_IOAT_JSF2:
    148	case PCI_DEVICE_ID_INTEL_IOAT_JSF3:
    149	case PCI_DEVICE_ID_INTEL_IOAT_JSF4:
    150	case PCI_DEVICE_ID_INTEL_IOAT_JSF5:
    151	case PCI_DEVICE_ID_INTEL_IOAT_JSF6:
    152	case PCI_DEVICE_ID_INTEL_IOAT_JSF7:
    153	case PCI_DEVICE_ID_INTEL_IOAT_JSF8:
    154	case PCI_DEVICE_ID_INTEL_IOAT_JSF9:
    155		return true;
    156	default:
    157		return false;
    158	}
    159}
    160
    161static bool is_snb_ioat(struct pci_dev *pdev)
    162{
    163	switch (pdev->device) {
    164	case PCI_DEVICE_ID_INTEL_IOAT_SNB0:
    165	case PCI_DEVICE_ID_INTEL_IOAT_SNB1:
    166	case PCI_DEVICE_ID_INTEL_IOAT_SNB2:
    167	case PCI_DEVICE_ID_INTEL_IOAT_SNB3:
    168	case PCI_DEVICE_ID_INTEL_IOAT_SNB4:
    169	case PCI_DEVICE_ID_INTEL_IOAT_SNB5:
    170	case PCI_DEVICE_ID_INTEL_IOAT_SNB6:
    171	case PCI_DEVICE_ID_INTEL_IOAT_SNB7:
    172	case PCI_DEVICE_ID_INTEL_IOAT_SNB8:
    173	case PCI_DEVICE_ID_INTEL_IOAT_SNB9:
    174		return true;
    175	default:
    176		return false;
    177	}
    178}
    179
    180static bool is_ivb_ioat(struct pci_dev *pdev)
    181{
    182	switch (pdev->device) {
    183	case PCI_DEVICE_ID_INTEL_IOAT_IVB0:
    184	case PCI_DEVICE_ID_INTEL_IOAT_IVB1:
    185	case PCI_DEVICE_ID_INTEL_IOAT_IVB2:
    186	case PCI_DEVICE_ID_INTEL_IOAT_IVB3:
    187	case PCI_DEVICE_ID_INTEL_IOAT_IVB4:
    188	case PCI_DEVICE_ID_INTEL_IOAT_IVB5:
    189	case PCI_DEVICE_ID_INTEL_IOAT_IVB6:
    190	case PCI_DEVICE_ID_INTEL_IOAT_IVB7:
    191	case PCI_DEVICE_ID_INTEL_IOAT_IVB8:
    192	case PCI_DEVICE_ID_INTEL_IOAT_IVB9:
    193		return true;
    194	default:
    195		return false;
    196	}
    197
    198}
    199
    200static bool is_hsw_ioat(struct pci_dev *pdev)
    201{
    202	switch (pdev->device) {
    203	case PCI_DEVICE_ID_INTEL_IOAT_HSW0:
    204	case PCI_DEVICE_ID_INTEL_IOAT_HSW1:
    205	case PCI_DEVICE_ID_INTEL_IOAT_HSW2:
    206	case PCI_DEVICE_ID_INTEL_IOAT_HSW3:
    207	case PCI_DEVICE_ID_INTEL_IOAT_HSW4:
    208	case PCI_DEVICE_ID_INTEL_IOAT_HSW5:
    209	case PCI_DEVICE_ID_INTEL_IOAT_HSW6:
    210	case PCI_DEVICE_ID_INTEL_IOAT_HSW7:
    211	case PCI_DEVICE_ID_INTEL_IOAT_HSW8:
    212	case PCI_DEVICE_ID_INTEL_IOAT_HSW9:
    213		return true;
    214	default:
    215		return false;
    216	}
    217
    218}
    219
    220static bool is_bdx_ioat(struct pci_dev *pdev)
    221{
    222	switch (pdev->device) {
    223	case PCI_DEVICE_ID_INTEL_IOAT_BDX0:
    224	case PCI_DEVICE_ID_INTEL_IOAT_BDX1:
    225	case PCI_DEVICE_ID_INTEL_IOAT_BDX2:
    226	case PCI_DEVICE_ID_INTEL_IOAT_BDX3:
    227	case PCI_DEVICE_ID_INTEL_IOAT_BDX4:
    228	case PCI_DEVICE_ID_INTEL_IOAT_BDX5:
    229	case PCI_DEVICE_ID_INTEL_IOAT_BDX6:
    230	case PCI_DEVICE_ID_INTEL_IOAT_BDX7:
    231	case PCI_DEVICE_ID_INTEL_IOAT_BDX8:
    232	case PCI_DEVICE_ID_INTEL_IOAT_BDX9:
    233		return true;
    234	default:
    235		return false;
    236	}
    237}
    238
    239static inline bool is_skx_ioat(struct pci_dev *pdev)
    240{
    241	return (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SKX) ? true : false;
    242}
    243
    244static bool is_xeon_cb32(struct pci_dev *pdev)
    245{
    246	return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) ||
    247		is_hsw_ioat(pdev) || is_bdx_ioat(pdev) || is_skx_ioat(pdev);
    248}
    249
    250bool is_bwd_ioat(struct pci_dev *pdev)
    251{
    252	switch (pdev->device) {
    253	case PCI_DEVICE_ID_INTEL_IOAT_BWD0:
    254	case PCI_DEVICE_ID_INTEL_IOAT_BWD1:
    255	case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
    256	case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
    257	/* even though not Atom, BDX-DE has same DMA silicon */
    258	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
    259	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
    260	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
    261	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
    262		return true;
    263	default:
    264		return false;
    265	}
    266}
    267
    268static bool is_bwd_noraid(struct pci_dev *pdev)
    269{
    270	switch (pdev->device) {
    271	case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
    272	case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
    273	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
    274	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
    275	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
    276	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
    277		return true;
    278	default:
    279		return false;
    280	}
    281
    282}
    283
    284/*
    285 * Perform a IOAT transaction to verify the HW works.
    286 */
    287#define IOAT_TEST_SIZE 2000
    288
    289static void ioat_dma_test_callback(void *dma_async_param)
    290{
    291	struct completion *cmp = dma_async_param;
    292
    293	complete(cmp);
    294}
    295
    296/**
    297 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
    298 * @ioat_dma: dma device to be tested
    299 */
    300static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)
    301{
    302	int i;
    303	u8 *src;
    304	u8 *dest;
    305	struct dma_device *dma = &ioat_dma->dma_dev;
    306	struct device *dev = &ioat_dma->pdev->dev;
    307	struct dma_chan *dma_chan;
    308	struct dma_async_tx_descriptor *tx;
    309	dma_addr_t dma_dest, dma_src;
    310	dma_cookie_t cookie;
    311	int err = 0;
    312	struct completion cmp;
    313	unsigned long tmo;
    314	unsigned long flags;
    315
    316	src = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL);
    317	if (!src)
    318		return -ENOMEM;
    319	dest = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL);
    320	if (!dest) {
    321		kfree(src);
    322		return -ENOMEM;
    323	}
    324
    325	/* Fill in src buffer */
    326	for (i = 0; i < IOAT_TEST_SIZE; i++)
    327		src[i] = (u8)i;
    328
    329	/* Start copy, using first DMA channel */
    330	dma_chan = container_of(dma->channels.next, struct dma_chan,
    331				device_node);
    332	if (dma->device_alloc_chan_resources(dma_chan) < 1) {
    333		dev_err(dev, "selftest cannot allocate chan resource\n");
    334		err = -ENODEV;
    335		goto out;
    336	}
    337
    338	dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
    339	if (dma_mapping_error(dev, dma_src)) {
    340		dev_err(dev, "mapping src buffer failed\n");
    341		err = -ENOMEM;
    342		goto free_resources;
    343	}
    344	dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
    345	if (dma_mapping_error(dev, dma_dest)) {
    346		dev_err(dev, "mapping dest buffer failed\n");
    347		err = -ENOMEM;
    348		goto unmap_src;
    349	}
    350	flags = DMA_PREP_INTERRUPT;
    351	tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest,
    352						      dma_src, IOAT_TEST_SIZE,
    353						      flags);
    354	if (!tx) {
    355		dev_err(dev, "Self-test prep failed, disabling\n");
    356		err = -ENODEV;
    357		goto unmap_dma;
    358	}
    359
    360	async_tx_ack(tx);
    361	init_completion(&cmp);
    362	tx->callback = ioat_dma_test_callback;
    363	tx->callback_param = &cmp;
    364	cookie = tx->tx_submit(tx);
    365	if (cookie < 0) {
    366		dev_err(dev, "Self-test setup failed, disabling\n");
    367		err = -ENODEV;
    368		goto unmap_dma;
    369	}
    370	dma->device_issue_pending(dma_chan);
    371
    372	tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
    373
    374	if (tmo == 0 ||
    375	    dma->device_tx_status(dma_chan, cookie, NULL)
    376					!= DMA_COMPLETE) {
    377		dev_err(dev, "Self-test copy timed out, disabling\n");
    378		err = -ENODEV;
    379		goto unmap_dma;
    380	}
    381	if (memcmp(src, dest, IOAT_TEST_SIZE)) {
    382		dev_err(dev, "Self-test copy failed compare, disabling\n");
    383		err = -ENODEV;
    384		goto unmap_dma;
    385	}
    386
    387unmap_dma:
    388	dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
    389unmap_src:
    390	dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
    391free_resources:
    392	dma->device_free_chan_resources(dma_chan);
    393out:
    394	kfree(src);
    395	kfree(dest);
    396	return err;
    397}
    398
    399/**
    400 * ioat_dma_setup_interrupts - setup interrupt handler
    401 * @ioat_dma: ioat dma device
    402 */
    403int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)
    404{
    405	struct ioatdma_chan *ioat_chan;
    406	struct pci_dev *pdev = ioat_dma->pdev;
    407	struct device *dev = &pdev->dev;
    408	struct msix_entry *msix;
    409	int i, j, msixcnt;
    410	int err = -EINVAL;
    411	u8 intrctrl = 0;
    412
    413	if (!strcmp(ioat_interrupt_style, "msix"))
    414		goto msix;
    415	if (!strcmp(ioat_interrupt_style, "msi"))
    416		goto msi;
    417	if (!strcmp(ioat_interrupt_style, "intx"))
    418		goto intx;
    419	dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
    420	goto err_no_irq;
    421
    422msix:
    423	/* The number of MSI-X vectors should equal the number of channels */
    424	msixcnt = ioat_dma->dma_dev.chancnt;
    425	for (i = 0; i < msixcnt; i++)
    426		ioat_dma->msix_entries[i].entry = i;
    427
    428	err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt);
    429	if (err)
    430		goto msi;
    431
    432	for (i = 0; i < msixcnt; i++) {
    433		msix = &ioat_dma->msix_entries[i];
    434		ioat_chan = ioat_chan_by_index(ioat_dma, i);
    435		err = devm_request_irq(dev, msix->vector,
    436				       ioat_dma_do_interrupt_msix, 0,
    437				       "ioat-msix", ioat_chan);
    438		if (err) {
    439			for (j = 0; j < i; j++) {
    440				msix = &ioat_dma->msix_entries[j];
    441				ioat_chan = ioat_chan_by_index(ioat_dma, j);
    442				devm_free_irq(dev, msix->vector, ioat_chan);
    443			}
    444			goto msi;
    445		}
    446	}
    447	intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
    448	ioat_dma->irq_mode = IOAT_MSIX;
    449	goto done;
    450
    451msi:
    452	err = pci_enable_msi(pdev);
    453	if (err)
    454		goto intx;
    455
    456	err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
    457			       "ioat-msi", ioat_dma);
    458	if (err) {
    459		pci_disable_msi(pdev);
    460		goto intx;
    461	}
    462	ioat_dma->irq_mode = IOAT_MSI;
    463	goto done;
    464
    465intx:
    466	err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
    467			       IRQF_SHARED, "ioat-intx", ioat_dma);
    468	if (err)
    469		goto err_no_irq;
    470
    471	ioat_dma->irq_mode = IOAT_INTX;
    472done:
    473	if (is_bwd_ioat(pdev))
    474		ioat_intr_quirk(ioat_dma);
    475	intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
    476	writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
    477	return 0;
    478
    479err_no_irq:
    480	/* Disable all interrupt generation */
    481	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
    482	ioat_dma->irq_mode = IOAT_NOIRQ;
    483	dev_err(dev, "no usable interrupts\n");
    484	return err;
    485}
    486
    487static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)
    488{
    489	/* Disable all interrupt generation */
    490	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
    491}
    492
    493static int ioat_probe(struct ioatdma_device *ioat_dma)
    494{
    495	int err = -ENODEV;
    496	struct dma_device *dma = &ioat_dma->dma_dev;
    497	struct pci_dev *pdev = ioat_dma->pdev;
    498	struct device *dev = &pdev->dev;
    499
    500	ioat_dma->completion_pool = dma_pool_create("completion_pool", dev,
    501						    sizeof(u64),
    502						    SMP_CACHE_BYTES,
    503						    SMP_CACHE_BYTES);
    504
    505	if (!ioat_dma->completion_pool) {
    506		err = -ENOMEM;
    507		goto err_out;
    508	}
    509
    510	ioat_enumerate_channels(ioat_dma);
    511
    512	dma_cap_set(DMA_MEMCPY, dma->cap_mask);
    513	dma->dev = &pdev->dev;
    514
    515	if (!dma->chancnt) {
    516		dev_err(dev, "channel enumeration error\n");
    517		goto err_setup_interrupts;
    518	}
    519
    520	err = ioat_dma_setup_interrupts(ioat_dma);
    521	if (err)
    522		goto err_setup_interrupts;
    523
    524	err = ioat3_dma_self_test(ioat_dma);
    525	if (err)
    526		goto err_self_test;
    527
    528	return 0;
    529
    530err_self_test:
    531	ioat_disable_interrupts(ioat_dma);
    532err_setup_interrupts:
    533	dma_pool_destroy(ioat_dma->completion_pool);
    534err_out:
    535	return err;
    536}
    537
    538static int ioat_register(struct ioatdma_device *ioat_dma)
    539{
    540	int err = dma_async_device_register(&ioat_dma->dma_dev);
    541
    542	if (err) {
    543		ioat_disable_interrupts(ioat_dma);
    544		dma_pool_destroy(ioat_dma->completion_pool);
    545	}
    546
    547	return err;
    548}
    549
    550static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
    551{
    552	struct dma_device *dma = &ioat_dma->dma_dev;
    553
    554	ioat_disable_interrupts(ioat_dma);
    555
    556	ioat_kobject_del(ioat_dma);
    557
    558	dma_async_device_unregister(dma);
    559}
    560
    561/**
    562 * ioat_enumerate_channels - find and initialize the device's channels
    563 * @ioat_dma: the ioat dma device to be enumerated
    564 */
    565static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma)
    566{
    567	struct ioatdma_chan *ioat_chan;
    568	struct device *dev = &ioat_dma->pdev->dev;
    569	struct dma_device *dma = &ioat_dma->dma_dev;
    570	u8 xfercap_log;
    571	int i;
    572
    573	INIT_LIST_HEAD(&dma->channels);
    574	dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);
    575	dma->chancnt &= 0x1f; /* bits [4:0] valid */
    576	if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {
    577		dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
    578			 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));
    579		dma->chancnt = ARRAY_SIZE(ioat_dma->idx);
    580	}
    581	xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);
    582	xfercap_log &= 0x1f; /* bits [4:0] valid */
    583	if (xfercap_log == 0)
    584		return;
    585	dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
    586
    587	for (i = 0; i < dma->chancnt; i++) {
    588		ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
    589		if (!ioat_chan)
    590			break;
    591
    592		ioat_init_channel(ioat_dma, ioat_chan, i);
    593		ioat_chan->xfercap_log = xfercap_log;
    594		spin_lock_init(&ioat_chan->prep_lock);
    595		if (ioat_reset_hw(ioat_chan)) {
    596			i = 0;
    597			break;
    598		}
    599	}
    600	dma->chancnt = i;
    601}
    602
    603/**
    604 * ioat_free_chan_resources - release all the descriptors
    605 * @c: the channel to be cleaned
    606 */
    607static void ioat_free_chan_resources(struct dma_chan *c)
    608{
    609	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
    610	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
    611	struct ioat_ring_ent *desc;
    612	const int total_descs = 1 << ioat_chan->alloc_order;
    613	int descs;
    614	int i;
    615
    616	/* Before freeing channel resources first check
    617	 * if they have been previously allocated for this channel.
    618	 */
    619	if (!ioat_chan->ring)
    620		return;
    621
    622	ioat_stop(ioat_chan);
    623
    624	if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) {
    625		ioat_reset_hw(ioat_chan);
    626
    627		/* Put LTR to idle */
    628		if (ioat_dma->version >= IOAT_VER_3_4)
    629			writeb(IOAT_CHAN_LTR_SWSEL_IDLE,
    630			       ioat_chan->reg_base +
    631			       IOAT_CHAN_LTR_SWSEL_OFFSET);
    632	}
    633
    634	spin_lock_bh(&ioat_chan->cleanup_lock);
    635	spin_lock_bh(&ioat_chan->prep_lock);
    636	descs = ioat_ring_space(ioat_chan);
    637	dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
    638	for (i = 0; i < descs; i++) {
    639		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);
    640		ioat_free_ring_ent(desc, c);
    641	}
    642
    643	if (descs < total_descs)
    644		dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
    645			total_descs - descs);
    646
    647	for (i = 0; i < total_descs - descs; i++) {
    648		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i);
    649		dump_desc_dbg(ioat_chan, desc);
    650		ioat_free_ring_ent(desc, c);
    651	}
    652
    653	for (i = 0; i < ioat_chan->desc_chunks; i++) {
    654		dma_free_coherent(to_dev(ioat_chan), IOAT_CHUNK_SIZE,
    655				  ioat_chan->descs[i].virt,
    656				  ioat_chan->descs[i].hw);
    657		ioat_chan->descs[i].virt = NULL;
    658		ioat_chan->descs[i].hw = 0;
    659	}
    660	ioat_chan->desc_chunks = 0;
    661
    662	kfree(ioat_chan->ring);
    663	ioat_chan->ring = NULL;
    664	ioat_chan->alloc_order = 0;
    665	dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion,
    666		      ioat_chan->completion_dma);
    667	spin_unlock_bh(&ioat_chan->prep_lock);
    668	spin_unlock_bh(&ioat_chan->cleanup_lock);
    669
    670	ioat_chan->last_completion = 0;
    671	ioat_chan->completion_dma = 0;
    672	ioat_chan->dmacount = 0;
    673}
    674
    675/* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring
    676 * @chan: channel to be initialized
    677 */
    678static int ioat_alloc_chan_resources(struct dma_chan *c)
    679{
    680	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
    681	struct ioat_ring_ent **ring;
    682	u64 status;
    683	int order;
    684	int i = 0;
    685	u32 chanerr;
    686
    687	/* have we already been set up? */
    688	if (ioat_chan->ring)
    689		return 1 << ioat_chan->alloc_order;
    690
    691	/* Setup register to interrupt and write completion status on error */
    692	writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
    693
    694	/* allocate a completion writeback area */
    695	/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
    696	ioat_chan->completion =
    697		dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool,
    698				GFP_NOWAIT, &ioat_chan->completion_dma);
    699	if (!ioat_chan->completion)
    700		return -ENOMEM;
    701
    702	writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,
    703	       ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
    704	writel(((u64)ioat_chan->completion_dma) >> 32,
    705	       ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
    706
    707	order = IOAT_MAX_ORDER;
    708	ring = ioat_alloc_ring(c, order, GFP_NOWAIT);
    709	if (!ring)
    710		return -ENOMEM;
    711
    712	spin_lock_bh(&ioat_chan->cleanup_lock);
    713	spin_lock_bh(&ioat_chan->prep_lock);
    714	ioat_chan->ring = ring;
    715	ioat_chan->head = 0;
    716	ioat_chan->issued = 0;
    717	ioat_chan->tail = 0;
    718	ioat_chan->alloc_order = order;
    719	set_bit(IOAT_RUN, &ioat_chan->state);
    720	spin_unlock_bh(&ioat_chan->prep_lock);
    721	spin_unlock_bh(&ioat_chan->cleanup_lock);
    722
    723	/* Setting up LTR values for 3.4 or later */
    724	if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) {
    725		u32 lat_val;
    726
    727		lat_val = IOAT_CHAN_LTR_ACTIVE_SNVAL |
    728			IOAT_CHAN_LTR_ACTIVE_SNLATSCALE |
    729			IOAT_CHAN_LTR_ACTIVE_SNREQMNT;
    730		writel(lat_val, ioat_chan->reg_base +
    731				IOAT_CHAN_LTR_ACTIVE_OFFSET);
    732
    733		lat_val = IOAT_CHAN_LTR_IDLE_SNVAL |
    734			  IOAT_CHAN_LTR_IDLE_SNLATSCALE |
    735			  IOAT_CHAN_LTR_IDLE_SNREQMNT;
    736		writel(lat_val, ioat_chan->reg_base +
    737				IOAT_CHAN_LTR_IDLE_OFFSET);
    738
    739		/* Select to active */
    740		writeb(IOAT_CHAN_LTR_SWSEL_ACTIVE,
    741		       ioat_chan->reg_base +
    742		       IOAT_CHAN_LTR_SWSEL_OFFSET);
    743	}
    744
    745	ioat_start_null_desc(ioat_chan);
    746
    747	/* check that we got off the ground */
    748	do {
    749		udelay(1);
    750		status = ioat_chansts(ioat_chan);
    751	} while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
    752
    753	if (is_ioat_active(status) || is_ioat_idle(status))
    754		return 1 << ioat_chan->alloc_order;
    755
    756	chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
    757
    758	dev_WARN(to_dev(ioat_chan),
    759		 "failed to start channel chanerr: %#x\n", chanerr);
    760	ioat_free_chan_resources(c);
    761	return -EFAULT;
    762}
    763
    764/* common channel initialization */
    765static void
    766ioat_init_channel(struct ioatdma_device *ioat_dma,
    767		  struct ioatdma_chan *ioat_chan, int idx)
    768{
    769	struct dma_device *dma = &ioat_dma->dma_dev;
    770
    771	ioat_chan->ioat_dma = ioat_dma;
    772	ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));
    773	spin_lock_init(&ioat_chan->cleanup_lock);
    774	ioat_chan->dma_chan.device = dma;
    775	dma_cookie_init(&ioat_chan->dma_chan);
    776	list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
    777	ioat_dma->idx[idx] = ioat_chan;
    778	timer_setup(&ioat_chan->timer, ioat_timer_event, 0);
    779	tasklet_setup(&ioat_chan->cleanup_task, ioat_cleanup_event);
    780}
    781
    782#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
    783static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
    784{
    785	int i, src_idx;
    786	struct page *dest;
    787	struct page *xor_srcs[IOAT_NUM_SRC_TEST];
    788	struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
    789	dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
    790	dma_addr_t dest_dma;
    791	struct dma_async_tx_descriptor *tx;
    792	struct dma_chan *dma_chan;
    793	dma_cookie_t cookie;
    794	u8 cmp_byte = 0;
    795	u32 cmp_word;
    796	u32 xor_val_result;
    797	int err = 0;
    798	struct completion cmp;
    799	unsigned long tmo;
    800	struct device *dev = &ioat_dma->pdev->dev;
    801	struct dma_device *dma = &ioat_dma->dma_dev;
    802	u8 op = 0;
    803
    804	dev_dbg(dev, "%s\n", __func__);
    805
    806	if (!dma_has_cap(DMA_XOR, dma->cap_mask))
    807		return 0;
    808
    809	for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
    810		xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
    811		if (!xor_srcs[src_idx]) {
    812			while (src_idx--)
    813				__free_page(xor_srcs[src_idx]);
    814			return -ENOMEM;
    815		}
    816	}
    817
    818	dest = alloc_page(GFP_KERNEL);
    819	if (!dest) {
    820		while (src_idx--)
    821			__free_page(xor_srcs[src_idx]);
    822		return -ENOMEM;
    823	}
    824
    825	/* Fill in src buffers */
    826	for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
    827		u8 *ptr = page_address(xor_srcs[src_idx]);
    828
    829		for (i = 0; i < PAGE_SIZE; i++)
    830			ptr[i] = (1 << src_idx);
    831	}
    832
    833	for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
    834		cmp_byte ^= (u8) (1 << src_idx);
    835
    836	cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
    837			(cmp_byte << 8) | cmp_byte;
    838
    839	memset(page_address(dest), 0, PAGE_SIZE);
    840
    841	dma_chan = container_of(dma->channels.next, struct dma_chan,
    842				device_node);
    843	if (dma->device_alloc_chan_resources(dma_chan) < 1) {
    844		err = -ENODEV;
    845		goto out;
    846	}
    847
    848	/* test xor */
    849	op = IOAT_OP_XOR;
    850
    851	dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
    852	if (dma_mapping_error(dev, dest_dma)) {
    853		err = -ENOMEM;
    854		goto free_resources;
    855	}
    856
    857	for (i = 0; i < IOAT_NUM_SRC_TEST; i++) {
    858		dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
    859					   DMA_TO_DEVICE);
    860		if (dma_mapping_error(dev, dma_srcs[i])) {
    861			err = -ENOMEM;
    862			goto dma_unmap;
    863		}
    864	}
    865	tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
    866				      IOAT_NUM_SRC_TEST, PAGE_SIZE,
    867				      DMA_PREP_INTERRUPT);
    868
    869	if (!tx) {
    870		dev_err(dev, "Self-test xor prep failed\n");
    871		err = -ENODEV;
    872		goto dma_unmap;
    873	}
    874
    875	async_tx_ack(tx);
    876	init_completion(&cmp);
    877	tx->callback = ioat_dma_test_callback;
    878	tx->callback_param = &cmp;
    879	cookie = tx->tx_submit(tx);
    880	if (cookie < 0) {
    881		dev_err(dev, "Self-test xor setup failed\n");
    882		err = -ENODEV;
    883		goto dma_unmap;
    884	}
    885	dma->device_issue_pending(dma_chan);
    886
    887	tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
    888
    889	if (tmo == 0 ||
    890	    dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
    891		dev_err(dev, "Self-test xor timed out\n");
    892		err = -ENODEV;
    893		goto dma_unmap;
    894	}
    895
    896	for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
    897		dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
    898
    899	dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
    900	for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
    901		u32 *ptr = page_address(dest);
    902
    903		if (ptr[i] != cmp_word) {
    904			dev_err(dev, "Self-test xor failed compare\n");
    905			err = -ENODEV;
    906			goto free_resources;
    907		}
    908	}
    909	dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
    910
    911	dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
    912
    913	/* skip validate if the capability is not present */
    914	if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
    915		goto free_resources;
    916
    917	op = IOAT_OP_XOR_VAL;
    918
    919	/* validate the sources with the destintation page */
    920	for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
    921		xor_val_srcs[i] = xor_srcs[i];
    922	xor_val_srcs[i] = dest;
    923
    924	xor_val_result = 1;
    925
    926	for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
    927		dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
    928					   DMA_TO_DEVICE);
    929		if (dma_mapping_error(dev, dma_srcs[i])) {
    930			err = -ENOMEM;
    931			goto dma_unmap;
    932		}
    933	}
    934	tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
    935					  IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
    936					  &xor_val_result, DMA_PREP_INTERRUPT);
    937	if (!tx) {
    938		dev_err(dev, "Self-test zero prep failed\n");
    939		err = -ENODEV;
    940		goto dma_unmap;
    941	}
    942
    943	async_tx_ack(tx);
    944	init_completion(&cmp);
    945	tx->callback = ioat_dma_test_callback;
    946	tx->callback_param = &cmp;
    947	cookie = tx->tx_submit(tx);
    948	if (cookie < 0) {
    949		dev_err(dev, "Self-test zero setup failed\n");
    950		err = -ENODEV;
    951		goto dma_unmap;
    952	}
    953	dma->device_issue_pending(dma_chan);
    954
    955	tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
    956
    957	if (tmo == 0 ||
    958	    dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
    959		dev_err(dev, "Self-test validate timed out\n");
    960		err = -ENODEV;
    961		goto dma_unmap;
    962	}
    963
    964	for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
    965		dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
    966
    967	if (xor_val_result != 0) {
    968		dev_err(dev, "Self-test validate failed compare\n");
    969		err = -ENODEV;
    970		goto free_resources;
    971	}
    972
    973	memset(page_address(dest), 0, PAGE_SIZE);
    974
    975	/* test for non-zero parity sum */
    976	op = IOAT_OP_XOR_VAL;
    977
    978	xor_val_result = 0;
    979	for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
    980		dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
    981					   DMA_TO_DEVICE);
    982		if (dma_mapping_error(dev, dma_srcs[i])) {
    983			err = -ENOMEM;
    984			goto dma_unmap;
    985		}
    986	}
    987	tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
    988					  IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
    989					  &xor_val_result, DMA_PREP_INTERRUPT);
    990	if (!tx) {
    991		dev_err(dev, "Self-test 2nd zero prep failed\n");
    992		err = -ENODEV;
    993		goto dma_unmap;
    994	}
    995
    996	async_tx_ack(tx);
    997	init_completion(&cmp);
    998	tx->callback = ioat_dma_test_callback;
    999	tx->callback_param = &cmp;
   1000	cookie = tx->tx_submit(tx);
   1001	if (cookie < 0) {
   1002		dev_err(dev, "Self-test  2nd zero setup failed\n");
   1003		err = -ENODEV;
   1004		goto dma_unmap;
   1005	}
   1006	dma->device_issue_pending(dma_chan);
   1007
   1008	tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
   1009
   1010	if (tmo == 0 ||
   1011	    dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
   1012		dev_err(dev, "Self-test 2nd validate timed out\n");
   1013		err = -ENODEV;
   1014		goto dma_unmap;
   1015	}
   1016
   1017	if (xor_val_result != SUM_CHECK_P_RESULT) {
   1018		dev_err(dev, "Self-test validate failed compare\n");
   1019		err = -ENODEV;
   1020		goto dma_unmap;
   1021	}
   1022
   1023	for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
   1024		dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
   1025
   1026	goto free_resources;
   1027dma_unmap:
   1028	if (op == IOAT_OP_XOR) {
   1029		while (--i >= 0)
   1030			dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
   1031				       DMA_TO_DEVICE);
   1032		dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
   1033	} else if (op == IOAT_OP_XOR_VAL) {
   1034		while (--i >= 0)
   1035			dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
   1036				       DMA_TO_DEVICE);
   1037	}
   1038free_resources:
   1039	dma->device_free_chan_resources(dma_chan);
   1040out:
   1041	src_idx = IOAT_NUM_SRC_TEST;
   1042	while (src_idx--)
   1043		__free_page(xor_srcs[src_idx]);
   1044	__free_page(dest);
   1045	return err;
   1046}
   1047
   1048static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma)
   1049{
   1050	int rc;
   1051
   1052	rc = ioat_dma_self_test(ioat_dma);
   1053	if (rc)
   1054		return rc;
   1055
   1056	rc = ioat_xor_val_self_test(ioat_dma);
   1057
   1058	return rc;
   1059}
   1060
   1061static void ioat_intr_quirk(struct ioatdma_device *ioat_dma)
   1062{
   1063	struct dma_device *dma;
   1064	struct dma_chan *c;
   1065	struct ioatdma_chan *ioat_chan;
   1066	u32 errmask;
   1067
   1068	dma = &ioat_dma->dma_dev;
   1069
   1070	/*
   1071	 * if we have descriptor write back error status, we mask the
   1072	 * error interrupts
   1073	 */
   1074	if (ioat_dma->cap & IOAT_CAP_DWBES) {
   1075		list_for_each_entry(c, &dma->channels, device_node) {
   1076			ioat_chan = to_ioat_chan(c);
   1077			errmask = readl(ioat_chan->reg_base +
   1078					IOAT_CHANERR_MASK_OFFSET);
   1079			errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR |
   1080				   IOAT_CHANERR_XOR_Q_ERR;
   1081			writel(errmask, ioat_chan->reg_base +
   1082					IOAT_CHANERR_MASK_OFFSET);
   1083		}
   1084	}
   1085}
   1086
   1087static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
   1088{
   1089	struct pci_dev *pdev = ioat_dma->pdev;
   1090	int dca_en = system_has_dca_enabled(pdev);
   1091	struct dma_device *dma;
   1092	struct dma_chan *c;
   1093	struct ioatdma_chan *ioat_chan;
   1094	int err;
   1095	u16 val16;
   1096
   1097	dma = &ioat_dma->dma_dev;
   1098	dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock;
   1099	dma->device_issue_pending = ioat_issue_pending;
   1100	dma->device_alloc_chan_resources = ioat_alloc_chan_resources;
   1101	dma->device_free_chan_resources = ioat_free_chan_resources;
   1102
   1103	dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
   1104	dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock;
   1105
   1106	ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET);
   1107
   1108	if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev))
   1109		ioat_dma->cap &=
   1110			~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS);
   1111
   1112	/* dca is incompatible with raid operations */
   1113	if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))
   1114		ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
   1115
   1116	if (ioat_dma->cap & IOAT_CAP_XOR) {
   1117		dma->max_xor = 8;
   1118
   1119		dma_cap_set(DMA_XOR, dma->cap_mask);
   1120		dma->device_prep_dma_xor = ioat_prep_xor;
   1121
   1122		dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
   1123		dma->device_prep_dma_xor_val = ioat_prep_xor_val;
   1124	}
   1125
   1126	if (ioat_dma->cap & IOAT_CAP_PQ) {
   1127
   1128		dma->device_prep_dma_pq = ioat_prep_pq;
   1129		dma->device_prep_dma_pq_val = ioat_prep_pq_val;
   1130		dma_cap_set(DMA_PQ, dma->cap_mask);
   1131		dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
   1132
   1133		if (ioat_dma->cap & IOAT_CAP_RAID16SS)
   1134			dma_set_maxpq(dma, 16, 0);
   1135		else
   1136			dma_set_maxpq(dma, 8, 0);
   1137
   1138		if (!(ioat_dma->cap & IOAT_CAP_XOR)) {
   1139			dma->device_prep_dma_xor = ioat_prep_pqxor;
   1140			dma->device_prep_dma_xor_val = ioat_prep_pqxor_val;
   1141			dma_cap_set(DMA_XOR, dma->cap_mask);
   1142			dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
   1143
   1144			if (ioat_dma->cap & IOAT_CAP_RAID16SS)
   1145				dma->max_xor = 16;
   1146			else
   1147				dma->max_xor = 8;
   1148		}
   1149	}
   1150
   1151	dma->device_tx_status = ioat_tx_status;
   1152
   1153	/* starting with CB3.3 super extended descriptors are supported */
   1154	if (ioat_dma->cap & IOAT_CAP_RAID16SS) {
   1155		char pool_name[14];
   1156		int i;
   1157
   1158		for (i = 0; i < MAX_SED_POOLS; i++) {
   1159			snprintf(pool_name, 14, "ioat_hw%d_sed", i);
   1160
   1161			/* allocate SED DMA pool */
   1162			ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name,
   1163					&pdev->dev,
   1164					SED_SIZE * (i + 1), 64, 0);
   1165			if (!ioat_dma->sed_hw_pool[i])
   1166				return -ENOMEM;
   1167
   1168		}
   1169	}
   1170
   1171	if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ)))
   1172		dma_cap_set(DMA_PRIVATE, dma->cap_mask);
   1173
   1174	err = ioat_probe(ioat_dma);
   1175	if (err)
   1176		return err;
   1177
   1178	list_for_each_entry(c, &dma->channels, device_node) {
   1179		ioat_chan = to_ioat_chan(c);
   1180		writel(IOAT_DMA_DCA_ANY_CPU,
   1181		       ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
   1182	}
   1183
   1184	err = ioat_register(ioat_dma);
   1185	if (err)
   1186		return err;
   1187
   1188	ioat_kobject_add(ioat_dma, &ioat_ktype);
   1189
   1190	if (dca)
   1191		ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);
   1192
   1193	/* disable relaxed ordering */
   1194	err = pcie_capability_read_word(pdev, IOAT_DEVCTRL_OFFSET, &val16);
   1195	if (err)
   1196		return pcibios_err_to_errno(err);
   1197
   1198	/* clear relaxed ordering enable */
   1199	val16 &= ~IOAT_DEVCTRL_ROE;
   1200	err = pcie_capability_write_word(pdev, IOAT_DEVCTRL_OFFSET, val16);
   1201	if (err)
   1202		return pcibios_err_to_errno(err);
   1203
   1204	if (ioat_dma->cap & IOAT_CAP_DPS)
   1205		writeb(ioat_pending_level + 1,
   1206		       ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET);
   1207
   1208	return 0;
   1209}
   1210
   1211static void ioat_shutdown(struct pci_dev *pdev)
   1212{
   1213	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
   1214	struct ioatdma_chan *ioat_chan;
   1215	int i;
   1216
   1217	if (!ioat_dma)
   1218		return;
   1219
   1220	for (i = 0; i < IOAT_MAX_CHANS; i++) {
   1221		ioat_chan = ioat_dma->idx[i];
   1222		if (!ioat_chan)
   1223			continue;
   1224
   1225		spin_lock_bh(&ioat_chan->prep_lock);
   1226		set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
   1227		spin_unlock_bh(&ioat_chan->prep_lock);
   1228		/*
   1229		 * Synchronization rule for del_timer_sync():
   1230		 *  - The caller must not hold locks which would prevent
   1231		 *    completion of the timer's handler.
   1232		 * So prep_lock cannot be held before calling it.
   1233		 */
   1234		del_timer_sync(&ioat_chan->timer);
   1235
   1236		/* this should quiesce then reset */
   1237		ioat_reset_hw(ioat_chan);
   1238	}
   1239
   1240	ioat_disable_interrupts(ioat_dma);
   1241}
   1242
   1243static void ioat_resume(struct ioatdma_device *ioat_dma)
   1244{
   1245	struct ioatdma_chan *ioat_chan;
   1246	u32 chanerr;
   1247	int i;
   1248
   1249	for (i = 0; i < IOAT_MAX_CHANS; i++) {
   1250		ioat_chan = ioat_dma->idx[i];
   1251		if (!ioat_chan)
   1252			continue;
   1253
   1254		spin_lock_bh(&ioat_chan->prep_lock);
   1255		clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
   1256		spin_unlock_bh(&ioat_chan->prep_lock);
   1257
   1258		chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
   1259		writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
   1260
   1261		/* no need to reset as shutdown already did that */
   1262	}
   1263}
   1264
   1265#define DRV_NAME "ioatdma"
   1266
   1267static pci_ers_result_t ioat_pcie_error_detected(struct pci_dev *pdev,
   1268						 pci_channel_state_t error)
   1269{
   1270	dev_dbg(&pdev->dev, "%s: PCIe AER error %d\n", DRV_NAME, error);
   1271
   1272	/* quiesce and block I/O */
   1273	ioat_shutdown(pdev);
   1274
   1275	return PCI_ERS_RESULT_NEED_RESET;
   1276}
   1277
   1278static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev)
   1279{
   1280	pci_ers_result_t result = PCI_ERS_RESULT_RECOVERED;
   1281
   1282	dev_dbg(&pdev->dev, "%s post reset handling\n", DRV_NAME);
   1283
   1284	if (pci_enable_device_mem(pdev) < 0) {
   1285		dev_err(&pdev->dev,
   1286			"Failed to enable PCIe device after reset.\n");
   1287		result = PCI_ERS_RESULT_DISCONNECT;
   1288	} else {
   1289		pci_set_master(pdev);
   1290		pci_restore_state(pdev);
   1291		pci_save_state(pdev);
   1292		pci_wake_from_d3(pdev, false);
   1293	}
   1294
   1295	return result;
   1296}
   1297
   1298static void ioat_pcie_error_resume(struct pci_dev *pdev)
   1299{
   1300	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
   1301
   1302	dev_dbg(&pdev->dev, "%s: AER handling resuming\n", DRV_NAME);
   1303
   1304	/* initialize and bring everything back */
   1305	ioat_resume(ioat_dma);
   1306}
   1307
   1308static const struct pci_error_handlers ioat_err_handler = {
   1309	.error_detected = ioat_pcie_error_detected,
   1310	.slot_reset = ioat_pcie_error_slot_reset,
   1311	.resume = ioat_pcie_error_resume,
   1312};
   1313
   1314static struct pci_driver ioat_pci_driver = {
   1315	.name		= DRV_NAME,
   1316	.id_table	= ioat_pci_tbl,
   1317	.probe		= ioat_pci_probe,
   1318	.remove		= ioat_remove,
   1319	.shutdown	= ioat_shutdown,
   1320	.err_handler	= &ioat_err_handler,
   1321};
   1322
   1323static void release_ioatdma(struct dma_device *device)
   1324{
   1325	struct ioatdma_device *d = to_ioatdma_device(device);
   1326	int i;
   1327
   1328	for (i = 0; i < IOAT_MAX_CHANS; i++)
   1329		kfree(d->idx[i]);
   1330
   1331	dma_pool_destroy(d->completion_pool);
   1332	kfree(d);
   1333}
   1334
   1335static struct ioatdma_device *
   1336alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
   1337{
   1338	struct ioatdma_device *d = kzalloc(sizeof(*d), GFP_KERNEL);
   1339
   1340	if (!d)
   1341		return NULL;
   1342	d->pdev = pdev;
   1343	d->reg_base = iobase;
   1344	d->dma_dev.device_release = release_ioatdma;
   1345	return d;
   1346}
   1347
   1348static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
   1349{
   1350	void __iomem * const *iomap;
   1351	struct device *dev = &pdev->dev;
   1352	struct ioatdma_device *device;
   1353	int err;
   1354
   1355	err = pcim_enable_device(pdev);
   1356	if (err)
   1357		return err;
   1358
   1359	err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
   1360	if (err)
   1361		return err;
   1362	iomap = pcim_iomap_table(pdev);
   1363	if (!iomap)
   1364		return -ENOMEM;
   1365
   1366	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
   1367	if (err)
   1368		return err;
   1369
   1370	device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
   1371	if (!device)
   1372		return -ENOMEM;
   1373	pci_set_master(pdev);
   1374	pci_set_drvdata(pdev, device);
   1375
   1376	device->version = readb(device->reg_base + IOAT_VER_OFFSET);
   1377	if (device->version >= IOAT_VER_3_4)
   1378		ioat_dca_enabled = 0;
   1379	if (device->version >= IOAT_VER_3_0) {
   1380		if (is_skx_ioat(pdev))
   1381			device->version = IOAT_VER_3_2;
   1382		err = ioat3_dma_probe(device, ioat_dca_enabled);
   1383
   1384		if (device->version >= IOAT_VER_3_3)
   1385			pci_enable_pcie_error_reporting(pdev);
   1386	} else
   1387		return -ENODEV;
   1388
   1389	if (err) {
   1390		dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
   1391		pci_disable_pcie_error_reporting(pdev);
   1392		return -ENODEV;
   1393	}
   1394
   1395	return 0;
   1396}
   1397
   1398static void ioat_remove(struct pci_dev *pdev)
   1399{
   1400	struct ioatdma_device *device = pci_get_drvdata(pdev);
   1401
   1402	if (!device)
   1403		return;
   1404
   1405	ioat_shutdown(pdev);
   1406
   1407	dev_err(&pdev->dev, "Removing dma and dca services\n");
   1408	if (device->dca) {
   1409		unregister_dca_provider(device->dca, &pdev->dev);
   1410		free_dca_provider(device->dca);
   1411		device->dca = NULL;
   1412	}
   1413
   1414	pci_disable_pcie_error_reporting(pdev);
   1415	ioat_dma_remove(device);
   1416}
   1417
   1418static int __init ioat_init_module(void)
   1419{
   1420	int err = -ENOMEM;
   1421
   1422	pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
   1423		DRV_NAME, IOAT_DMA_VERSION);
   1424
   1425	ioat_cache = kmem_cache_create("ioat", sizeof(struct ioat_ring_ent),
   1426					0, SLAB_HWCACHE_ALIGN, NULL);
   1427	if (!ioat_cache)
   1428		return -ENOMEM;
   1429
   1430	ioat_sed_cache = KMEM_CACHE(ioat_sed_ent, 0);
   1431	if (!ioat_sed_cache)
   1432		goto err_ioat_cache;
   1433
   1434	err = pci_register_driver(&ioat_pci_driver);
   1435	if (err)
   1436		goto err_ioat3_cache;
   1437
   1438	return 0;
   1439
   1440 err_ioat3_cache:
   1441	kmem_cache_destroy(ioat_sed_cache);
   1442
   1443 err_ioat_cache:
   1444	kmem_cache_destroy(ioat_cache);
   1445
   1446	return err;
   1447}
   1448module_init(ioat_init_module);
   1449
   1450static void __exit ioat_exit_module(void)
   1451{
   1452	pci_unregister_driver(&ioat_pci_driver);
   1453	kmem_cache_destroy(ioat_cache);
   1454}
   1455module_exit(ioat_exit_module);