cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

mac-fcc.c (15374B)


      1/*
      2 * FCC driver for Motorola MPC82xx (PQ2).
      3 *
      4 * Copyright (c) 2003 Intracom S.A.
      5 *  by Pantelis Antoniou <panto@intracom.gr>
      6 *
      7 * 2005 (c) MontaVista Software, Inc.
      8 * Vitaly Bordug <vbordug@ru.mvista.com>
      9 *
     10 * This file is licensed under the terms of the GNU General Public License
     11 * version 2. This program is licensed "as is" without any warranty of any
     12 * kind, whether express or implied.
     13 */
     14
     15#include <linux/module.h>
     16#include <linux/kernel.h>
     17#include <linux/types.h>
     18#include <linux/string.h>
     19#include <linux/ptrace.h>
     20#include <linux/errno.h>
     21#include <linux/ioport.h>
     22#include <linux/interrupt.h>
     23#include <linux/delay.h>
     24#include <linux/netdevice.h>
     25#include <linux/etherdevice.h>
     26#include <linux/skbuff.h>
     27#include <linux/spinlock.h>
     28#include <linux/mii.h>
     29#include <linux/ethtool.h>
     30#include <linux/bitops.h>
     31#include <linux/fs.h>
     32#include <linux/platform_device.h>
     33#include <linux/phy.h>
     34#include <linux/of_address.h>
     35#include <linux/of_device.h>
     36#include <linux/of_irq.h>
     37#include <linux/gfp.h>
     38#include <linux/pgtable.h>
     39
     40#include <asm/immap_cpm2.h>
     41#include <asm/mpc8260.h>
     42#include <asm/cpm2.h>
     43
     44#include <asm/irq.h>
     45#include <linux/uaccess.h>
     46
     47#include "fs_enet.h"
     48
     49/*************************************************/
     50
     51/* FCC access macros */
     52
     53/* write, read, set bits, clear bits */
     54#define W32(_p, _m, _v)	out_be32(&(_p)->_m, (_v))
     55#define R32(_p, _m)	in_be32(&(_p)->_m)
     56#define S32(_p, _m, _v)	W32(_p, _m, R32(_p, _m) | (_v))
     57#define C32(_p, _m, _v)	W32(_p, _m, R32(_p, _m) & ~(_v))
     58
     59#define W16(_p, _m, _v)	out_be16(&(_p)->_m, (_v))
     60#define R16(_p, _m)	in_be16(&(_p)->_m)
     61#define S16(_p, _m, _v)	W16(_p, _m, R16(_p, _m) | (_v))
     62#define C16(_p, _m, _v)	W16(_p, _m, R16(_p, _m) & ~(_v))
     63
     64#define W8(_p, _m, _v)	out_8(&(_p)->_m, (_v))
     65#define R8(_p, _m)	in_8(&(_p)->_m)
     66#define S8(_p, _m, _v)	W8(_p, _m, R8(_p, _m) | (_v))
     67#define C8(_p, _m, _v)	W8(_p, _m, R8(_p, _m) & ~(_v))
     68
     69/*************************************************/
     70
     71#define FCC_MAX_MULTICAST_ADDRS	64
     72
     73#define mk_mii_read(REG)	(0x60020000 | ((REG & 0x1f) << 18))
     74#define mk_mii_write(REG, VAL)	(0x50020000 | ((REG & 0x1f) << 18) | (VAL & 0xffff))
     75#define mk_mii_end		0
     76
     77#define MAX_CR_CMD_LOOPS	10000
     78
     79static inline int fcc_cr_cmd(struct fs_enet_private *fep, u32 op)
     80{
     81	const struct fs_platform_info *fpi = fep->fpi;
     82
     83	return cpm_command(fpi->cp_command, op);
     84}
     85
     86static int do_pd_setup(struct fs_enet_private *fep)
     87{
     88	struct platform_device *ofdev = to_platform_device(fep->dev);
     89	struct fs_platform_info *fpi = fep->fpi;
     90	int ret = -EINVAL;
     91
     92	fep->interrupt = irq_of_parse_and_map(ofdev->dev.of_node, 0);
     93	if (!fep->interrupt)
     94		goto out;
     95
     96	fep->fcc.fccp = of_iomap(ofdev->dev.of_node, 0);
     97	if (!fep->fcc.fccp)
     98		goto out;
     99
    100	fep->fcc.ep = of_iomap(ofdev->dev.of_node, 1);
    101	if (!fep->fcc.ep)
    102		goto out_fccp;
    103
    104	fep->fcc.fcccp = of_iomap(ofdev->dev.of_node, 2);
    105	if (!fep->fcc.fcccp)
    106		goto out_ep;
    107
    108	fep->fcc.mem = (void __iomem *)cpm2_immr;
    109	fpi->dpram_offset = cpm_dpalloc(128, 32);
    110	if (IS_ERR_VALUE(fpi->dpram_offset)) {
    111		ret = fpi->dpram_offset;
    112		goto out_fcccp;
    113	}
    114
    115	return 0;
    116
    117out_fcccp:
    118	iounmap(fep->fcc.fcccp);
    119out_ep:
    120	iounmap(fep->fcc.ep);
    121out_fccp:
    122	iounmap(fep->fcc.fccp);
    123out:
    124	return ret;
    125}
    126
    127#define FCC_NAPI_EVENT_MSK	(FCC_ENET_RXF | FCC_ENET_RXB | FCC_ENET_TXB)
    128#define FCC_EVENT		(FCC_ENET_RXF | FCC_ENET_TXB)
    129#define FCC_ERR_EVENT_MSK	(FCC_ENET_TXE)
    130
    131static int setup_data(struct net_device *dev)
    132{
    133	struct fs_enet_private *fep = netdev_priv(dev);
    134
    135	if (do_pd_setup(fep) != 0)
    136		return -EINVAL;
    137
    138	fep->ev_napi = FCC_NAPI_EVENT_MSK;
    139	fep->ev = FCC_EVENT;
    140	fep->ev_err = FCC_ERR_EVENT_MSK;
    141
    142	return 0;
    143}
    144
    145static int allocate_bd(struct net_device *dev)
    146{
    147	struct fs_enet_private *fep = netdev_priv(dev);
    148	const struct fs_platform_info *fpi = fep->fpi;
    149
    150	fep->ring_base = (void __iomem __force *)dma_alloc_coherent(fep->dev,
    151					    (fpi->tx_ring + fpi->rx_ring) *
    152					    sizeof(cbd_t), &fep->ring_mem_addr,
    153					    GFP_KERNEL);
    154	if (fep->ring_base == NULL)
    155		return -ENOMEM;
    156
    157	return 0;
    158}
    159
    160static void free_bd(struct net_device *dev)
    161{
    162	struct fs_enet_private *fep = netdev_priv(dev);
    163	const struct fs_platform_info *fpi = fep->fpi;
    164
    165	if (fep->ring_base)
    166		dma_free_coherent(fep->dev,
    167			(fpi->tx_ring + fpi->rx_ring) * sizeof(cbd_t),
    168			(void __force *)fep->ring_base, fep->ring_mem_addr);
    169}
    170
    171static void cleanup_data(struct net_device *dev)
    172{
    173	/* nothing */
    174}
    175
    176static void set_promiscuous_mode(struct net_device *dev)
    177{
    178	struct fs_enet_private *fep = netdev_priv(dev);
    179	fcc_t __iomem *fccp = fep->fcc.fccp;
    180
    181	S32(fccp, fcc_fpsmr, FCC_PSMR_PRO);
    182}
    183
    184static void set_multicast_start(struct net_device *dev)
    185{
    186	struct fs_enet_private *fep = netdev_priv(dev);
    187	fcc_enet_t __iomem *ep = fep->fcc.ep;
    188
    189	W32(ep, fen_gaddrh, 0);
    190	W32(ep, fen_gaddrl, 0);
    191}
    192
    193static void set_multicast_one(struct net_device *dev, const u8 *mac)
    194{
    195	struct fs_enet_private *fep = netdev_priv(dev);
    196	fcc_enet_t __iomem *ep = fep->fcc.ep;
    197	u16 taddrh, taddrm, taddrl;
    198
    199	taddrh = ((u16)mac[5] << 8) | mac[4];
    200	taddrm = ((u16)mac[3] << 8) | mac[2];
    201	taddrl = ((u16)mac[1] << 8) | mac[0];
    202
    203	W16(ep, fen_taddrh, taddrh);
    204	W16(ep, fen_taddrm, taddrm);
    205	W16(ep, fen_taddrl, taddrl);
    206	fcc_cr_cmd(fep, CPM_CR_SET_GADDR);
    207}
    208
    209static void set_multicast_finish(struct net_device *dev)
    210{
    211	struct fs_enet_private *fep = netdev_priv(dev);
    212	fcc_t __iomem *fccp = fep->fcc.fccp;
    213	fcc_enet_t __iomem *ep = fep->fcc.ep;
    214
    215	/* clear promiscuous always */
    216	C32(fccp, fcc_fpsmr, FCC_PSMR_PRO);
    217
    218	/* if all multi or too many multicasts; just enable all */
    219	if ((dev->flags & IFF_ALLMULTI) != 0 ||
    220	    netdev_mc_count(dev) > FCC_MAX_MULTICAST_ADDRS) {
    221
    222		W32(ep, fen_gaddrh, 0xffffffff);
    223		W32(ep, fen_gaddrl, 0xffffffff);
    224	}
    225
    226	/* read back */
    227	fep->fcc.gaddrh = R32(ep, fen_gaddrh);
    228	fep->fcc.gaddrl = R32(ep, fen_gaddrl);
    229}
    230
    231static void set_multicast_list(struct net_device *dev)
    232{
    233	struct netdev_hw_addr *ha;
    234
    235	if ((dev->flags & IFF_PROMISC) == 0) {
    236		set_multicast_start(dev);
    237		netdev_for_each_mc_addr(ha, dev)
    238			set_multicast_one(dev, ha->addr);
    239		set_multicast_finish(dev);
    240	} else
    241		set_promiscuous_mode(dev);
    242}
    243
    244static void restart(struct net_device *dev)
    245{
    246	struct fs_enet_private *fep = netdev_priv(dev);
    247	const struct fs_platform_info *fpi = fep->fpi;
    248	fcc_t __iomem *fccp = fep->fcc.fccp;
    249	fcc_c_t __iomem *fcccp = fep->fcc.fcccp;
    250	fcc_enet_t __iomem *ep = fep->fcc.ep;
    251	dma_addr_t rx_bd_base_phys, tx_bd_base_phys;
    252	u16 paddrh, paddrm, paddrl;
    253	const unsigned char *mac;
    254	int i;
    255
    256	C32(fccp, fcc_gfmr, FCC_GFMR_ENR | FCC_GFMR_ENT);
    257
    258	/* clear everything (slow & steady does it) */
    259	for (i = 0; i < sizeof(*ep); i++)
    260		out_8((u8 __iomem *)ep + i, 0);
    261
    262	/* get physical address */
    263	rx_bd_base_phys = fep->ring_mem_addr;
    264	tx_bd_base_phys = rx_bd_base_phys + sizeof(cbd_t) * fpi->rx_ring;
    265
    266	/* point to bds */
    267	W32(ep, fen_genfcc.fcc_rbase, rx_bd_base_phys);
    268	W32(ep, fen_genfcc.fcc_tbase, tx_bd_base_phys);
    269
    270	/* Set maximum bytes per receive buffer.
    271	 * It must be a multiple of 32.
    272	 */
    273	W16(ep, fen_genfcc.fcc_mrblr, PKT_MAXBLR_SIZE);
    274
    275	W32(ep, fen_genfcc.fcc_rstate, (CPMFCR_GBL | CPMFCR_EB) << 24);
    276	W32(ep, fen_genfcc.fcc_tstate, (CPMFCR_GBL | CPMFCR_EB) << 24);
    277
    278	/* Allocate space in the reserved FCC area of DPRAM for the
    279	 * internal buffers.  No one uses this space (yet), so we
    280	 * can do this.  Later, we will add resource management for
    281	 * this area.
    282	 */
    283
    284	W16(ep, fen_genfcc.fcc_riptr, fpi->dpram_offset);
    285	W16(ep, fen_genfcc.fcc_tiptr, fpi->dpram_offset + 32);
    286
    287	W16(ep, fen_padptr, fpi->dpram_offset + 64);
    288
    289	/* fill with special symbol...  */
    290	memset_io(fep->fcc.mem + fpi->dpram_offset + 64, 0x88, 32);
    291
    292	W32(ep, fen_genfcc.fcc_rbptr, 0);
    293	W32(ep, fen_genfcc.fcc_tbptr, 0);
    294	W32(ep, fen_genfcc.fcc_rcrc, 0);
    295	W32(ep, fen_genfcc.fcc_tcrc, 0);
    296	W16(ep, fen_genfcc.fcc_res1, 0);
    297	W32(ep, fen_genfcc.fcc_res2, 0);
    298
    299	/* no CAM */
    300	W32(ep, fen_camptr, 0);
    301
    302	/* Set CRC preset and mask */
    303	W32(ep, fen_cmask, 0xdebb20e3);
    304	W32(ep, fen_cpres, 0xffffffff);
    305
    306	W32(ep, fen_crcec, 0);		/* CRC Error counter       */
    307	W32(ep, fen_alec, 0);		/* alignment error counter */
    308	W32(ep, fen_disfc, 0);		/* discard frame counter   */
    309	W16(ep, fen_retlim, 15);	/* Retry limit threshold   */
    310	W16(ep, fen_pper, 0);		/* Normal persistence      */
    311
    312	/* set group address */
    313	W32(ep, fen_gaddrh, fep->fcc.gaddrh);
    314	W32(ep, fen_gaddrl, fep->fcc.gaddrh);
    315
    316	/* Clear hash filter tables */
    317	W32(ep, fen_iaddrh, 0);
    318	W32(ep, fen_iaddrl, 0);
    319
    320	/* Clear the Out-of-sequence TxBD  */
    321	W16(ep, fen_tfcstat, 0);
    322	W16(ep, fen_tfclen, 0);
    323	W32(ep, fen_tfcptr, 0);
    324
    325	W16(ep, fen_mflr, PKT_MAXBUF_SIZE);	/* maximum frame length register */
    326	W16(ep, fen_minflr, PKT_MINBUF_SIZE);	/* minimum frame length register */
    327
    328	/* set address */
    329	mac = dev->dev_addr;
    330	paddrh = ((u16)mac[5] << 8) | mac[4];
    331	paddrm = ((u16)mac[3] << 8) | mac[2];
    332	paddrl = ((u16)mac[1] << 8) | mac[0];
    333
    334	W16(ep, fen_paddrh, paddrh);
    335	W16(ep, fen_paddrm, paddrm);
    336	W16(ep, fen_paddrl, paddrl);
    337
    338	W16(ep, fen_taddrh, 0);
    339	W16(ep, fen_taddrm, 0);
    340	W16(ep, fen_taddrl, 0);
    341
    342	W16(ep, fen_maxd1, 1520);	/* maximum DMA1 length */
    343	W16(ep, fen_maxd2, 1520);	/* maximum DMA2 length */
    344
    345	/* Clear stat counters, in case we ever enable RMON */
    346	W32(ep, fen_octc, 0);
    347	W32(ep, fen_colc, 0);
    348	W32(ep, fen_broc, 0);
    349	W32(ep, fen_mulc, 0);
    350	W32(ep, fen_uspc, 0);
    351	W32(ep, fen_frgc, 0);
    352	W32(ep, fen_ospc, 0);
    353	W32(ep, fen_jbrc, 0);
    354	W32(ep, fen_p64c, 0);
    355	W32(ep, fen_p65c, 0);
    356	W32(ep, fen_p128c, 0);
    357	W32(ep, fen_p256c, 0);
    358	W32(ep, fen_p512c, 0);
    359	W32(ep, fen_p1024c, 0);
    360
    361	W16(ep, fen_rfthr, 0);	/* Suggested by manual */
    362	W16(ep, fen_rfcnt, 0);
    363	W16(ep, fen_cftype, 0);
    364
    365	fs_init_bds(dev);
    366
    367	/* adjust to speed (for RMII mode) */
    368	if (fpi->use_rmii) {
    369		if (dev->phydev->speed == 100)
    370			C8(fcccp, fcc_gfemr, 0x20);
    371		else
    372			S8(fcccp, fcc_gfemr, 0x20);
    373	}
    374
    375	fcc_cr_cmd(fep, CPM_CR_INIT_TRX);
    376
    377	/* clear events */
    378	W16(fccp, fcc_fcce, 0xffff);
    379
    380	/* Enable interrupts we wish to service */
    381	W16(fccp, fcc_fccm, FCC_ENET_TXE | FCC_ENET_RXF | FCC_ENET_TXB);
    382
    383	/* Set GFMR to enable Ethernet operating mode */
    384	W32(fccp, fcc_gfmr, FCC_GFMR_TCI | FCC_GFMR_MODE_ENET);
    385
    386	/* set sync/delimiters */
    387	W16(fccp, fcc_fdsr, 0xd555);
    388
    389	W32(fccp, fcc_fpsmr, FCC_PSMR_ENCRC);
    390
    391	if (fpi->use_rmii)
    392		S32(fccp, fcc_fpsmr, FCC_PSMR_RMII);
    393
    394	/* adjust to duplex mode */
    395	if (dev->phydev->duplex)
    396		S32(fccp, fcc_fpsmr, FCC_PSMR_FDE | FCC_PSMR_LPB);
    397	else
    398		C32(fccp, fcc_fpsmr, FCC_PSMR_FDE | FCC_PSMR_LPB);
    399
    400	/* Restore multicast and promiscuous settings */
    401	set_multicast_list(dev);
    402
    403	S32(fccp, fcc_gfmr, FCC_GFMR_ENR | FCC_GFMR_ENT);
    404}
    405
    406static void stop(struct net_device *dev)
    407{
    408	struct fs_enet_private *fep = netdev_priv(dev);
    409	fcc_t __iomem *fccp = fep->fcc.fccp;
    410
    411	/* stop ethernet */
    412	C32(fccp, fcc_gfmr, FCC_GFMR_ENR | FCC_GFMR_ENT);
    413
    414	/* clear events */
    415	W16(fccp, fcc_fcce, 0xffff);
    416
    417	/* clear interrupt mask */
    418	W16(fccp, fcc_fccm, 0);
    419
    420	fs_cleanup_bds(dev);
    421}
    422
    423static void napi_clear_event_fs(struct net_device *dev)
    424{
    425	struct fs_enet_private *fep = netdev_priv(dev);
    426	fcc_t __iomem *fccp = fep->fcc.fccp;
    427
    428	W16(fccp, fcc_fcce, FCC_NAPI_EVENT_MSK);
    429}
    430
    431static void napi_enable_fs(struct net_device *dev)
    432{
    433	struct fs_enet_private *fep = netdev_priv(dev);
    434	fcc_t __iomem *fccp = fep->fcc.fccp;
    435
    436	S16(fccp, fcc_fccm, FCC_NAPI_EVENT_MSK);
    437}
    438
    439static void napi_disable_fs(struct net_device *dev)
    440{
    441	struct fs_enet_private *fep = netdev_priv(dev);
    442	fcc_t __iomem *fccp = fep->fcc.fccp;
    443
    444	C16(fccp, fcc_fccm, FCC_NAPI_EVENT_MSK);
    445}
    446
    447static void rx_bd_done(struct net_device *dev)
    448{
    449	/* nothing */
    450}
    451
    452static void tx_kickstart(struct net_device *dev)
    453{
    454	struct fs_enet_private *fep = netdev_priv(dev);
    455	fcc_t __iomem *fccp = fep->fcc.fccp;
    456
    457	S16(fccp, fcc_ftodr, 0x8000);
    458}
    459
    460static u32 get_int_events(struct net_device *dev)
    461{
    462	struct fs_enet_private *fep = netdev_priv(dev);
    463	fcc_t __iomem *fccp = fep->fcc.fccp;
    464
    465	return (u32)R16(fccp, fcc_fcce);
    466}
    467
    468static void clear_int_events(struct net_device *dev, u32 int_events)
    469{
    470	struct fs_enet_private *fep = netdev_priv(dev);
    471	fcc_t __iomem *fccp = fep->fcc.fccp;
    472
    473	W16(fccp, fcc_fcce, int_events & 0xffff);
    474}
    475
    476static void ev_error(struct net_device *dev, u32 int_events)
    477{
    478	struct fs_enet_private *fep = netdev_priv(dev);
    479
    480	dev_warn(fep->dev, "FS_ENET ERROR(s) 0x%x\n", int_events);
    481}
    482
    483static int get_regs(struct net_device *dev, void *p, int *sizep)
    484{
    485	struct fs_enet_private *fep = netdev_priv(dev);
    486
    487	if (*sizep < sizeof(fcc_t) + sizeof(fcc_enet_t) + 1)
    488		return -EINVAL;
    489
    490	memcpy_fromio(p, fep->fcc.fccp, sizeof(fcc_t));
    491	p = (char *)p + sizeof(fcc_t);
    492
    493	memcpy_fromio(p, fep->fcc.ep, sizeof(fcc_enet_t));
    494	p = (char *)p + sizeof(fcc_enet_t);
    495
    496	memcpy_fromio(p, fep->fcc.fcccp, 1);
    497	return 0;
    498}
    499
    500static int get_regs_len(struct net_device *dev)
    501{
    502	return sizeof(fcc_t) + sizeof(fcc_enet_t) + 1;
    503}
    504
    505/* Some transmit errors cause the transmitter to shut
    506 * down.  We now issue a restart transmit.
    507 * Also, to workaround 8260 device erratum CPM37, we must
    508 * disable and then re-enable the transmitterfollowing a
    509 * Late Collision, Underrun, or Retry Limit error.
    510 * In addition, tbptr may point beyond BDs beyond still marked
    511 * as ready due to internal pipelining, so we need to look back
    512 * through the BDs and adjust tbptr to point to the last BD
    513 * marked as ready.  This may result in some buffers being
    514 * retransmitted.
    515 */
    516static void tx_restart(struct net_device *dev)
    517{
    518	struct fs_enet_private *fep = netdev_priv(dev);
    519	fcc_t __iomem *fccp = fep->fcc.fccp;
    520	const struct fs_platform_info *fpi = fep->fpi;
    521	fcc_enet_t __iomem *ep = fep->fcc.ep;
    522	cbd_t __iomem *curr_tbptr;
    523	cbd_t __iomem *recheck_bd;
    524	cbd_t __iomem *prev_bd;
    525	cbd_t __iomem *last_tx_bd;
    526
    527	last_tx_bd = fep->tx_bd_base + (fpi->tx_ring - 1);
    528
    529	/* get the current bd held in TBPTR  and scan back from this point */
    530	recheck_bd = curr_tbptr = (cbd_t __iomem *)
    531		((R32(ep, fen_genfcc.fcc_tbptr) - fep->ring_mem_addr) +
    532		fep->ring_base);
    533
    534	prev_bd = (recheck_bd == fep->tx_bd_base) ? last_tx_bd : recheck_bd - 1;
    535
    536	/* Move through the bds in reverse, look for the earliest buffer
    537	 * that is not ready.  Adjust TBPTR to the following buffer */
    538	while ((CBDR_SC(prev_bd) & BD_ENET_TX_READY) != 0) {
    539		/* Go back one buffer */
    540		recheck_bd = prev_bd;
    541
    542		/* update the previous buffer */
    543		prev_bd = (prev_bd == fep->tx_bd_base) ? last_tx_bd : prev_bd - 1;
    544
    545		/* We should never see all bds marked as ready, check anyway */
    546		if (recheck_bd == curr_tbptr)
    547			break;
    548	}
    549	/* Now update the TBPTR and dirty flag to the current buffer */
    550	W32(ep, fen_genfcc.fcc_tbptr,
    551		(uint) (((void *)recheck_bd - fep->ring_base) +
    552		fep->ring_mem_addr));
    553	fep->dirty_tx = recheck_bd;
    554
    555	C32(fccp, fcc_gfmr, FCC_GFMR_ENT);
    556	udelay(10);
    557	S32(fccp, fcc_gfmr, FCC_GFMR_ENT);
    558
    559	fcc_cr_cmd(fep, CPM_CR_RESTART_TX);
    560}
    561
    562/*************************************************************************/
    563
    564const struct fs_ops fs_fcc_ops = {
    565	.setup_data		= setup_data,
    566	.cleanup_data		= cleanup_data,
    567	.set_multicast_list	= set_multicast_list,
    568	.restart		= restart,
    569	.stop			= stop,
    570	.napi_clear_event	= napi_clear_event_fs,
    571	.napi_enable		= napi_enable_fs,
    572	.napi_disable		= napi_disable_fs,
    573	.rx_bd_done		= rx_bd_done,
    574	.tx_kickstart		= tx_kickstart,
    575	.get_int_events		= get_int_events,
    576	.clear_int_events	= clear_int_events,
    577	.ev_error		= ev_error,
    578	.get_regs		= get_regs,
    579	.get_regs_len		= get_regs_len,
    580	.tx_restart		= tx_restart,
    581	.allocate_bd		= allocate_bd,
    582	.free_bd		= free_bd,
    583};