cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

sba_iommu.c (58517B)


      1// SPDX-License-Identifier: GPL-2.0-or-later
      2/*
      3**  System Bus Adapter (SBA) I/O MMU manager
      4**
      5**	(c) Copyright 2000-2004 Grant Grundler <grundler @ parisc-linux x org>
      6**	(c) Copyright 2004 Naresh Kumar Inna <knaresh at india x hp x com>
      7**	(c) Copyright 2000-2004 Hewlett-Packard Company
      8**
      9**	Portions (c) 1999 Dave S. Miller (from sparc64 I/O MMU code)
     10**
     11**
     12**
     13** This module initializes the IOC (I/O Controller) found on B1000/C3000/
     14** J5000/J7000/N-class/L-class machines and their successors.
     15**
     16** FIXME: add DMA hint support programming in both sba and lba modules.
     17*/
     18
     19#include <linux/types.h>
     20#include <linux/kernel.h>
     21#include <linux/spinlock.h>
     22#include <linux/slab.h>
     23#include <linux/init.h>
     24
     25#include <linux/mm.h>
     26#include <linux/string.h>
     27#include <linux/pci.h>
     28#include <linux/dma-map-ops.h>
     29#include <linux/scatterlist.h>
     30#include <linux/iommu-helper.h>
     31
     32#include <asm/byteorder.h>
     33#include <asm/io.h>
     34#include <asm/dma.h>		/* for DMA_CHUNK_SIZE */
     35
     36#include <asm/hardware.h>	/* for register_parisc_driver() stuff */
     37
     38#include <linux/proc_fs.h>
     39#include <linux/seq_file.h>
     40#include <linux/module.h>
     41
     42#include <asm/ropes.h>
     43#include <asm/mckinley.h>	/* for proc_mckinley_root */
     44#include <asm/runway.h>		/* for proc_runway_root */
     45#include <asm/page.h>		/* for PAGE0 */
     46#include <asm/pdc.h>		/* for PDC_MODEL_* */
     47#include <asm/pdcpat.h>		/* for is_pdc_pat() */
     48#include <asm/parisc-device.h>
     49
     50#include "iommu.h"
     51
     52#define MODULE_NAME "SBA"
     53
     54/*
     55** The number of debug flags is a clue - this code is fragile.
     56** Don't even think about messing with it unless you have
     57** plenty of 710's to sacrifice to the computer gods. :^)
     58*/
     59#undef DEBUG_SBA_INIT
     60#undef DEBUG_SBA_RUN
     61#undef DEBUG_SBA_RUN_SG
     62#undef DEBUG_SBA_RESOURCE
     63#undef ASSERT_PDIR_SANITY
     64#undef DEBUG_LARGE_SG_ENTRIES
     65#undef DEBUG_DMB_TRAP
     66
     67#ifdef DEBUG_SBA_INIT
     68#define DBG_INIT(x...)	printk(x)
     69#else
     70#define DBG_INIT(x...)
     71#endif
     72
     73#ifdef DEBUG_SBA_RUN
     74#define DBG_RUN(x...)	printk(x)
     75#else
     76#define DBG_RUN(x...)
     77#endif
     78
     79#ifdef DEBUG_SBA_RUN_SG
     80#define DBG_RUN_SG(x...)	printk(x)
     81#else
     82#define DBG_RUN_SG(x...)
     83#endif
     84
     85
     86#ifdef DEBUG_SBA_RESOURCE
     87#define DBG_RES(x...)	printk(x)
     88#else
     89#define DBG_RES(x...)
     90#endif
     91
     92#define SBA_INLINE	__inline__
     93
     94#define DEFAULT_DMA_HINT_REG	0
     95
     96struct sba_device *sba_list;
     97EXPORT_SYMBOL_GPL(sba_list);
     98
     99static unsigned long ioc_needs_fdc = 0;
    100
    101/* global count of IOMMUs in the system */
    102static unsigned int global_ioc_cnt = 0;
    103
    104/* PA8700 (Piranha 2.2) bug workaround */
    105static unsigned long piranha_bad_128k = 0;
    106
    107/* Looks nice and keeps the compiler happy */
    108#define SBA_DEV(d) ((struct sba_device *) (d))
    109
    110#ifdef CONFIG_AGP_PARISC
    111#define SBA_AGP_SUPPORT
    112#endif /*CONFIG_AGP_PARISC*/
    113
    114#ifdef SBA_AGP_SUPPORT
    115static int sba_reserve_agpgart = 1;
    116module_param(sba_reserve_agpgart, int, 0444);
    117MODULE_PARM_DESC(sba_reserve_agpgart, "Reserve half of IO pdir as AGPGART");
    118#endif
    119
    120
    121/************************************
    122** SBA register read and write support
    123**
    124** BE WARNED: register writes are posted.
    125**  (ie follow writes which must reach HW with a read)
    126**
    127** Superdome (in particular, REO) allows only 64-bit CSR accesses.
    128*/
    129#define READ_REG32(addr)	readl(addr)
    130#define READ_REG64(addr)	readq(addr)
    131#define WRITE_REG32(val, addr)	writel((val), (addr))
    132#define WRITE_REG64(val, addr)	writeq((val), (addr))
    133
    134#ifdef CONFIG_64BIT
    135#define READ_REG(addr)		READ_REG64(addr)
    136#define WRITE_REG(value, addr)	WRITE_REG64(value, addr)
    137#else
    138#define READ_REG(addr)		READ_REG32(addr)
    139#define WRITE_REG(value, addr)	WRITE_REG32(value, addr)
    140#endif
    141
    142#ifdef DEBUG_SBA_INIT
    143
    144/* NOTE: When CONFIG_64BIT isn't defined, READ_REG64() is two 32-bit reads */
    145
    146/**
    147 * sba_dump_ranges - debugging only - print ranges assigned to this IOA
    148 * @hpa: base address of the sba
    149 *
    150 * Print the MMIO and IO Port address ranges forwarded by an Astro/Ike/RIO
    151 * IO Adapter (aka Bus Converter).
    152 */
    153static void
    154sba_dump_ranges(void __iomem *hpa)
    155{
    156	DBG_INIT("SBA at 0x%p\n", hpa);
    157	DBG_INIT("IOS_DIST_BASE   : %Lx\n", READ_REG64(hpa+IOS_DIST_BASE));
    158	DBG_INIT("IOS_DIST_MASK   : %Lx\n", READ_REG64(hpa+IOS_DIST_MASK));
    159	DBG_INIT("IOS_DIST_ROUTE  : %Lx\n", READ_REG64(hpa+IOS_DIST_ROUTE));
    160	DBG_INIT("\n");
    161	DBG_INIT("IOS_DIRECT_BASE : %Lx\n", READ_REG64(hpa+IOS_DIRECT_BASE));
    162	DBG_INIT("IOS_DIRECT_MASK : %Lx\n", READ_REG64(hpa+IOS_DIRECT_MASK));
    163	DBG_INIT("IOS_DIRECT_ROUTE: %Lx\n", READ_REG64(hpa+IOS_DIRECT_ROUTE));
    164}
    165
    166/**
    167 * sba_dump_tlb - debugging only - print IOMMU operating parameters
    168 * @hpa: base address of the IOMMU
    169 *
    170 * Print the size/location of the IO MMU PDIR.
    171 */
    172static void sba_dump_tlb(void __iomem *hpa)
    173{
    174	DBG_INIT("IO TLB at 0x%p\n", hpa);
    175	DBG_INIT("IOC_IBASE    : 0x%Lx\n", READ_REG64(hpa+IOC_IBASE));
    176	DBG_INIT("IOC_IMASK    : 0x%Lx\n", READ_REG64(hpa+IOC_IMASK));
    177	DBG_INIT("IOC_TCNFG    : 0x%Lx\n", READ_REG64(hpa+IOC_TCNFG));
    178	DBG_INIT("IOC_PDIR_BASE: 0x%Lx\n", READ_REG64(hpa+IOC_PDIR_BASE));
    179	DBG_INIT("\n");
    180}
    181#else
    182#define sba_dump_ranges(x)
    183#define sba_dump_tlb(x)
    184#endif	/* DEBUG_SBA_INIT */
    185
    186
    187#ifdef ASSERT_PDIR_SANITY
    188
    189/**
    190 * sba_dump_pdir_entry - debugging only - print one IOMMU PDIR entry
    191 * @ioc: IO MMU structure which owns the pdir we are interested in.
    192 * @msg: text to print ont the output line.
    193 * @pide: pdir index.
    194 *
    195 * Print one entry of the IO MMU PDIR in human readable form.
    196 */
    197static void
    198sba_dump_pdir_entry(struct ioc *ioc, char *msg, uint pide)
    199{
    200	/* start printing from lowest pde in rval */
    201	u64 *ptr = &(ioc->pdir_base[pide & (~0U * BITS_PER_LONG)]);
    202	unsigned long *rptr = (unsigned long *) &(ioc->res_map[(pide >>3) & ~(sizeof(unsigned long) - 1)]);
    203	uint rcnt;
    204
    205	printk(KERN_DEBUG "SBA: %s rp %p bit %d rval 0x%lx\n",
    206		 msg,
    207		 rptr, pide & (BITS_PER_LONG - 1), *rptr);
    208
    209	rcnt = 0;
    210	while (rcnt < BITS_PER_LONG) {
    211		printk(KERN_DEBUG "%s %2d %p %016Lx\n",
    212			(rcnt == (pide & (BITS_PER_LONG - 1)))
    213				? "    -->" : "       ",
    214			rcnt, ptr, *ptr );
    215		rcnt++;
    216		ptr++;
    217	}
    218	printk(KERN_DEBUG "%s", msg);
    219}
    220
    221
    222/**
    223 * sba_check_pdir - debugging only - consistency checker
    224 * @ioc: IO MMU structure which owns the pdir we are interested in.
    225 * @msg: text to print ont the output line.
    226 *
    227 * Verify the resource map and pdir state is consistent
    228 */
    229static int
    230sba_check_pdir(struct ioc *ioc, char *msg)
    231{
    232	u32 *rptr_end = (u32 *) &(ioc->res_map[ioc->res_size]);
    233	u32 *rptr = (u32 *) ioc->res_map;	/* resource map ptr */
    234	u64 *pptr = ioc->pdir_base;	/* pdir ptr */
    235	uint pide = 0;
    236
    237	while (rptr < rptr_end) {
    238		u32 rval = *rptr;
    239		int rcnt = 32;	/* number of bits we might check */
    240
    241		while (rcnt) {
    242			/* Get last byte and highest bit from that */
    243			u32 pde = ((u32) (((char *)pptr)[7])) << 24;
    244			if ((rval ^ pde) & 0x80000000)
    245			{
    246				/*
    247				** BUMMER!  -- res_map != pdir --
    248				** Dump rval and matching pdir entries
    249				*/
    250				sba_dump_pdir_entry(ioc, msg, pide);
    251				return(1);
    252			}
    253			rcnt--;
    254			rval <<= 1;	/* try the next bit */
    255			pptr++;
    256			pide++;
    257		}
    258		rptr++;	/* look at next word of res_map */
    259	}
    260	/* It'd be nice if we always got here :^) */
    261	return 0;
    262}
    263
    264
    265/**
    266 * sba_dump_sg - debugging only - print Scatter-Gather list
    267 * @ioc: IO MMU structure which owns the pdir we are interested in.
    268 * @startsg: head of the SG list
    269 * @nents: number of entries in SG list
    270 *
    271 * print the SG list so we can verify it's correct by hand.
    272 */
    273static void
    274sba_dump_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
    275{
    276	while (nents-- > 0) {
    277		printk(KERN_DEBUG " %d : %08lx/%05x %p/%05x\n",
    278				nents,
    279				(unsigned long) sg_dma_address(startsg),
    280				sg_dma_len(startsg),
    281				sg_virt(startsg), startsg->length);
    282		startsg++;
    283	}
    284}
    285
    286#endif /* ASSERT_PDIR_SANITY */
    287
    288
    289
    290
    291/**************************************************************
    292*
    293*   I/O Pdir Resource Management
    294*
    295*   Bits set in the resource map are in use.
    296*   Each bit can represent a number of pages.
    297*   LSbs represent lower addresses (IOVA's).
    298*
    299***************************************************************/
    300#define PAGES_PER_RANGE 1	/* could increase this to 4 or 8 if needed */
    301
    302/* Convert from IOVP to IOVA and vice versa. */
    303
    304#ifdef ZX1_SUPPORT
    305/* Pluto (aka ZX1) boxes need to set or clear the ibase bits appropriately */
    306#define SBA_IOVA(ioc,iovp,offset,hint_reg) ((ioc->ibase) | (iovp) | (offset))
    307#define SBA_IOVP(ioc,iova) ((iova) & (ioc)->iovp_mask)
    308#else
    309/* only support Astro and ancestors. Saves a few cycles in key places */
    310#define SBA_IOVA(ioc,iovp,offset,hint_reg) ((iovp) | (offset))
    311#define SBA_IOVP(ioc,iova) (iova)
    312#endif
    313
    314#define PDIR_INDEX(iovp)   ((iovp)>>IOVP_SHIFT)
    315
    316#define RESMAP_MASK(n)    (~0UL << (BITS_PER_LONG - (n)))
    317#define RESMAP_IDX_MASK   (sizeof(unsigned long) - 1)
    318
    319static unsigned long ptr_to_pide(struct ioc *ioc, unsigned long *res_ptr,
    320				 unsigned int bitshiftcnt)
    321{
    322	return (((unsigned long)res_ptr - (unsigned long)ioc->res_map) << 3)
    323		+ bitshiftcnt;
    324}
    325
    326/**
    327 * sba_search_bitmap - find free space in IO PDIR resource bitmap
    328 * @ioc: IO MMU structure which owns the pdir we are interested in.
    329 * @bits_wanted: number of entries we need.
    330 *
    331 * Find consecutive free bits in resource bitmap.
    332 * Each bit represents one entry in the IO Pdir.
    333 * Cool perf optimization: search for log2(size) bits at a time.
    334 */
    335static SBA_INLINE unsigned long
    336sba_search_bitmap(struct ioc *ioc, struct device *dev,
    337		  unsigned long bits_wanted)
    338{
    339	unsigned long *res_ptr = ioc->res_hint;
    340	unsigned long *res_end = (unsigned long *) &(ioc->res_map[ioc->res_size]);
    341	unsigned long pide = ~0UL, tpide;
    342	unsigned long boundary_size;
    343	unsigned long shift;
    344	int ret;
    345
    346	boundary_size = dma_get_seg_boundary_nr_pages(dev, IOVP_SHIFT);
    347
    348#if defined(ZX1_SUPPORT)
    349	BUG_ON(ioc->ibase & ~IOVP_MASK);
    350	shift = ioc->ibase >> IOVP_SHIFT;
    351#else
    352	shift = 0;
    353#endif
    354
    355	if (bits_wanted > (BITS_PER_LONG/2)) {
    356		/* Search word at a time - no mask needed */
    357		for(; res_ptr < res_end; ++res_ptr) {
    358			tpide = ptr_to_pide(ioc, res_ptr, 0);
    359			ret = iommu_is_span_boundary(tpide, bits_wanted,
    360						     shift,
    361						     boundary_size);
    362			if ((*res_ptr == 0) && !ret) {
    363				*res_ptr = RESMAP_MASK(bits_wanted);
    364				pide = tpide;
    365				break;
    366			}
    367		}
    368		/* point to the next word on next pass */
    369		res_ptr++;
    370		ioc->res_bitshift = 0;
    371	} else {
    372		/*
    373		** Search the resource bit map on well-aligned values.
    374		** "o" is the alignment.
    375		** We need the alignment to invalidate I/O TLB using
    376		** SBA HW features in the unmap path.
    377		*/
    378		unsigned long o = 1 << get_order(bits_wanted << PAGE_SHIFT);
    379		uint bitshiftcnt = ALIGN(ioc->res_bitshift, o);
    380		unsigned long mask;
    381
    382		if (bitshiftcnt >= BITS_PER_LONG) {
    383			bitshiftcnt = 0;
    384			res_ptr++;
    385		}
    386		mask = RESMAP_MASK(bits_wanted) >> bitshiftcnt;
    387
    388		DBG_RES("%s() o %ld %p", __func__, o, res_ptr);
    389		while(res_ptr < res_end)
    390		{ 
    391			DBG_RES("    %p %lx %lx\n", res_ptr, mask, *res_ptr);
    392			WARN_ON(mask == 0);
    393			tpide = ptr_to_pide(ioc, res_ptr, bitshiftcnt);
    394			ret = iommu_is_span_boundary(tpide, bits_wanted,
    395						     shift,
    396						     boundary_size);
    397			if ((((*res_ptr) & mask) == 0) && !ret) {
    398				*res_ptr |= mask;     /* mark resources busy! */
    399				pide = tpide;
    400				break;
    401			}
    402			mask >>= o;
    403			bitshiftcnt += o;
    404			if (mask == 0) {
    405				mask = RESMAP_MASK(bits_wanted);
    406				bitshiftcnt=0;
    407				res_ptr++;
    408			}
    409		}
    410		/* look in the same word on the next pass */
    411		ioc->res_bitshift = bitshiftcnt + bits_wanted;
    412	}
    413
    414	/* wrapped ? */
    415	if (res_end <= res_ptr) {
    416		ioc->res_hint = (unsigned long *) ioc->res_map;
    417		ioc->res_bitshift = 0;
    418	} else {
    419		ioc->res_hint = res_ptr;
    420	}
    421	return (pide);
    422}
    423
    424
    425/**
    426 * sba_alloc_range - find free bits and mark them in IO PDIR resource bitmap
    427 * @ioc: IO MMU structure which owns the pdir we are interested in.
    428 * @size: number of bytes to create a mapping for
    429 *
    430 * Given a size, find consecutive unmarked and then mark those bits in the
    431 * resource bit map.
    432 */
    433static int
    434sba_alloc_range(struct ioc *ioc, struct device *dev, size_t size)
    435{
    436	unsigned int pages_needed = size >> IOVP_SHIFT;
    437#ifdef SBA_COLLECT_STATS
    438	unsigned long cr_start = mfctl(16);
    439#endif
    440	unsigned long pide;
    441
    442	pide = sba_search_bitmap(ioc, dev, pages_needed);
    443	if (pide >= (ioc->res_size << 3)) {
    444		pide = sba_search_bitmap(ioc, dev, pages_needed);
    445		if (pide >= (ioc->res_size << 3))
    446			panic("%s: I/O MMU @ %p is out of mapping resources\n",
    447			      __FILE__, ioc->ioc_hpa);
    448	}
    449
    450#ifdef ASSERT_PDIR_SANITY
    451	/* verify the first enable bit is clear */
    452	if(0x00 != ((u8 *) ioc->pdir_base)[pide*sizeof(u64) + 7]) {
    453		sba_dump_pdir_entry(ioc, "sba_search_bitmap() botched it?", pide);
    454	}
    455#endif
    456
    457	DBG_RES("%s(%x) %d -> %lx hint %x/%x\n",
    458		__func__, size, pages_needed, pide,
    459		(uint) ((unsigned long) ioc->res_hint - (unsigned long) ioc->res_map),
    460		ioc->res_bitshift );
    461
    462#ifdef SBA_COLLECT_STATS
    463	{
    464		unsigned long cr_end = mfctl(16);
    465		unsigned long tmp = cr_end - cr_start;
    466		/* check for roll over */
    467		cr_start = (cr_end < cr_start) ?  -(tmp) : (tmp);
    468	}
    469	ioc->avg_search[ioc->avg_idx++] = cr_start;
    470	ioc->avg_idx &= SBA_SEARCH_SAMPLE - 1;
    471
    472	ioc->used_pages += pages_needed;
    473#endif
    474
    475	return (pide);
    476}
    477
    478
    479/**
    480 * sba_free_range - unmark bits in IO PDIR resource bitmap
    481 * @ioc: IO MMU structure which owns the pdir we are interested in.
    482 * @iova: IO virtual address which was previously allocated.
    483 * @size: number of bytes to create a mapping for
    484 *
    485 * clear bits in the ioc's resource map
    486 */
    487static SBA_INLINE void
    488sba_free_range(struct ioc *ioc, dma_addr_t iova, size_t size)
    489{
    490	unsigned long iovp = SBA_IOVP(ioc, iova);
    491	unsigned int pide = PDIR_INDEX(iovp);
    492	unsigned int ridx = pide >> 3;	/* convert bit to byte address */
    493	unsigned long *res_ptr = (unsigned long *) &((ioc)->res_map[ridx & ~RESMAP_IDX_MASK]);
    494
    495	int bits_not_wanted = size >> IOVP_SHIFT;
    496
    497	/* 3-bits "bit" address plus 2 (or 3) bits for "byte" == bit in word */
    498	unsigned long m = RESMAP_MASK(bits_not_wanted) >> (pide & (BITS_PER_LONG - 1));
    499
    500	DBG_RES("%s( ,%x,%x) %x/%lx %x %p %lx\n",
    501		__func__, (uint) iova, size,
    502		bits_not_wanted, m, pide, res_ptr, *res_ptr);
    503
    504#ifdef SBA_COLLECT_STATS
    505	ioc->used_pages -= bits_not_wanted;
    506#endif
    507
    508	*res_ptr &= ~m;
    509}
    510
    511
    512/**************************************************************
    513*
    514*   "Dynamic DMA Mapping" support (aka "Coherent I/O")
    515*
    516***************************************************************/
    517
    518#ifdef SBA_HINT_SUPPORT
    519#define SBA_DMA_HINT(ioc, val) ((val) << (ioc)->hint_shift_pdir)
    520#endif
    521
    522typedef unsigned long space_t;
    523#define KERNEL_SPACE 0
    524
    525/**
    526 * sba_io_pdir_entry - fill in one IO PDIR entry
    527 * @pdir_ptr:  pointer to IO PDIR entry
    528 * @sid: process Space ID - currently only support KERNEL_SPACE
    529 * @vba: Virtual CPU address of buffer to map
    530 * @hint: DMA hint set to use for this mapping
    531 *
    532 * SBA Mapping Routine
    533 *
    534 * Given a virtual address (vba, arg2) and space id, (sid, arg1)
    535 * sba_io_pdir_entry() loads the I/O PDIR entry pointed to by
    536 * pdir_ptr (arg0). 
    537 * Using the bass-ackwards HP bit numbering, Each IO Pdir entry
    538 * for Astro/Ike looks like:
    539 *
    540 *
    541 *  0                    19                                 51   55       63
    542 * +-+---------------------+----------------------------------+----+--------+
    543 * |V|        U            |            PPN[43:12]            | U  |   VI   |
    544 * +-+---------------------+----------------------------------+----+--------+
    545 *
    546 * Pluto is basically identical, supports fewer physical address bits:
    547 *
    548 *  0                       23                              51   55       63
    549 * +-+------------------------+-------------------------------+----+--------+
    550 * |V|        U               |         PPN[39:12]            | U  |   VI   |
    551 * +-+------------------------+-------------------------------+----+--------+
    552 *
    553 *  V  == Valid Bit  (Most Significant Bit is bit 0)
    554 *  U  == Unused
    555 * PPN == Physical Page Number
    556 * VI  == Virtual Index (aka Coherent Index)
    557 *
    558 * LPA instruction output is put into PPN field.
    559 * LCI (Load Coherence Index) instruction provides the "VI" bits.
    560 *
    561 * We pre-swap the bytes since PCX-W is Big Endian and the
    562 * IOMMU uses little endian for the pdir.
    563 */
    564
    565static void SBA_INLINE
    566sba_io_pdir_entry(u64 *pdir_ptr, space_t sid, unsigned long vba,
    567		  unsigned long hint)
    568{
    569	u64 pa; /* physical address */
    570	register unsigned ci; /* coherent index */
    571
    572	pa = lpa(vba);
    573	pa &= IOVP_MASK;
    574
    575	asm("lci 0(%1), %0" : "=r" (ci) : "r" (vba));
    576	pa |= (ci >> PAGE_SHIFT) & 0xff;  /* move CI (8 bits) into lowest byte */
    577
    578	pa |= SBA_PDIR_VALID_BIT;	/* set "valid" bit */
    579	*pdir_ptr = cpu_to_le64(pa);	/* swap and store into I/O Pdir */
    580
    581	/*
    582	 * If the PDC_MODEL capabilities has Non-coherent IO-PDIR bit set
    583	 * (bit #61, big endian), we have to flush and sync every time
    584	 * IO-PDIR is changed in Ike/Astro.
    585	 */
    586	asm_io_fdc(pdir_ptr);
    587}
    588
    589
    590/**
    591 * sba_mark_invalid - invalidate one or more IO PDIR entries
    592 * @ioc: IO MMU structure which owns the pdir we are interested in.
    593 * @iova:  IO Virtual Address mapped earlier
    594 * @byte_cnt:  number of bytes this mapping covers.
    595 *
    596 * Marking the IO PDIR entry(ies) as Invalid and invalidate
    597 * corresponding IO TLB entry. The Ike PCOM (Purge Command Register)
    598 * is to purge stale entries in the IO TLB when unmapping entries.
    599 *
    600 * The PCOM register supports purging of multiple pages, with a minium
    601 * of 1 page and a maximum of 2GB. Hardware requires the address be
    602 * aligned to the size of the range being purged. The size of the range
    603 * must be a power of 2. The "Cool perf optimization" in the
    604 * allocation routine helps keep that true.
    605 */
    606static SBA_INLINE void
    607sba_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt)
    608{
    609	u32 iovp = (u32) SBA_IOVP(ioc,iova);
    610	u64 *pdir_ptr = &ioc->pdir_base[PDIR_INDEX(iovp)];
    611
    612#ifdef ASSERT_PDIR_SANITY
    613	/* Assert first pdir entry is set.
    614	**
    615	** Even though this is a big-endian machine, the entries
    616	** in the iopdir are little endian. That's why we look at
    617	** the byte at +7 instead of at +0.
    618	*/
    619	if (0x80 != (((u8 *) pdir_ptr)[7])) {
    620		sba_dump_pdir_entry(ioc,"sba_mark_invalid()", PDIR_INDEX(iovp));
    621	}
    622#endif
    623
    624	if (byte_cnt > IOVP_SIZE)
    625	{
    626#if 0
    627		unsigned long entries_per_cacheline = ioc_needs_fdc ?
    628				L1_CACHE_ALIGN(((unsigned long) pdir_ptr))
    629					- (unsigned long) pdir_ptr;
    630				: 262144;
    631#endif
    632
    633		/* set "size" field for PCOM */
    634		iovp |= get_order(byte_cnt) + PAGE_SHIFT;
    635
    636		do {
    637			/* clear I/O Pdir entry "valid" bit first */
    638			((u8 *) pdir_ptr)[7] = 0;
    639			asm_io_fdc(pdir_ptr);
    640			if (ioc_needs_fdc) {
    641#if 0
    642				entries_per_cacheline = L1_CACHE_SHIFT - 3;
    643#endif
    644			}
    645			pdir_ptr++;
    646			byte_cnt -= IOVP_SIZE;
    647		} while (byte_cnt > IOVP_SIZE);
    648	} else
    649		iovp |= IOVP_SHIFT;     /* set "size" field for PCOM */
    650
    651	/*
    652	** clear I/O PDIR entry "valid" bit.
    653	** We have to R/M/W the cacheline regardless how much of the
    654	** pdir entry that we clobber.
    655	** The rest of the entry would be useful for debugging if we
    656	** could dump core on HPMC.
    657	*/
    658	((u8 *) pdir_ptr)[7] = 0;
    659	asm_io_fdc(pdir_ptr);
    660
    661	WRITE_REG( SBA_IOVA(ioc, iovp, 0, 0), ioc->ioc_hpa+IOC_PCOM);
    662}
    663
    664/**
    665 * sba_dma_supported - PCI driver can query DMA support
    666 * @dev: instance of PCI owned by the driver that's asking
    667 * @mask:  number of address bits this PCI device can handle
    668 *
    669 * See Documentation/core-api/dma-api-howto.rst
    670 */
    671static int sba_dma_supported( struct device *dev, u64 mask)
    672{
    673	struct ioc *ioc;
    674
    675	if (dev == NULL) {
    676		printk(KERN_ERR MODULE_NAME ": EISA/ISA/et al not supported\n");
    677		BUG();
    678		return(0);
    679	}
    680
    681	ioc = GET_IOC(dev);
    682	if (!ioc)
    683		return 0;
    684
    685	/*
    686	 * check if mask is >= than the current max IO Virt Address
    687	 * The max IO Virt address will *always* < 30 bits.
    688	 */
    689	return((int)(mask >= (ioc->ibase - 1 +
    690			(ioc->pdir_size / sizeof(u64) * IOVP_SIZE) )));
    691}
    692
    693
    694/**
    695 * sba_map_single - map one buffer and return IOVA for DMA
    696 * @dev: instance of PCI owned by the driver that's asking.
    697 * @addr:  driver buffer to map.
    698 * @size:  number of bytes to map in driver buffer.
    699 * @direction:  R/W or both.
    700 *
    701 * See Documentation/core-api/dma-api-howto.rst
    702 */
    703static dma_addr_t
    704sba_map_single(struct device *dev, void *addr, size_t size,
    705	       enum dma_data_direction direction)
    706{
    707	struct ioc *ioc;
    708	unsigned long flags; 
    709	dma_addr_t iovp;
    710	dma_addr_t offset;
    711	u64 *pdir_start;
    712	int pide;
    713
    714	ioc = GET_IOC(dev);
    715	if (!ioc)
    716		return DMA_MAPPING_ERROR;
    717
    718	/* save offset bits */
    719	offset = ((dma_addr_t) (long) addr) & ~IOVP_MASK;
    720
    721	/* round up to nearest IOVP_SIZE */
    722	size = (size + offset + ~IOVP_MASK) & IOVP_MASK;
    723
    724	spin_lock_irqsave(&ioc->res_lock, flags);
    725#ifdef ASSERT_PDIR_SANITY
    726	sba_check_pdir(ioc,"Check before sba_map_single()");
    727#endif
    728
    729#ifdef SBA_COLLECT_STATS
    730	ioc->msingle_calls++;
    731	ioc->msingle_pages += size >> IOVP_SHIFT;
    732#endif
    733	pide = sba_alloc_range(ioc, dev, size);
    734	iovp = (dma_addr_t) pide << IOVP_SHIFT;
    735
    736	DBG_RUN("%s() 0x%p -> 0x%lx\n",
    737		__func__, addr, (long) iovp | offset);
    738
    739	pdir_start = &(ioc->pdir_base[pide]);
    740
    741	while (size > 0) {
    742		sba_io_pdir_entry(pdir_start, KERNEL_SPACE, (unsigned long) addr, 0);
    743
    744		DBG_RUN("	pdir 0x%p %02x%02x%02x%02x%02x%02x%02x%02x\n",
    745			pdir_start,
    746			(u8) (((u8 *) pdir_start)[7]),
    747			(u8) (((u8 *) pdir_start)[6]),
    748			(u8) (((u8 *) pdir_start)[5]),
    749			(u8) (((u8 *) pdir_start)[4]),
    750			(u8) (((u8 *) pdir_start)[3]),
    751			(u8) (((u8 *) pdir_start)[2]),
    752			(u8) (((u8 *) pdir_start)[1]),
    753			(u8) (((u8 *) pdir_start)[0])
    754			);
    755
    756		addr += IOVP_SIZE;
    757		size -= IOVP_SIZE;
    758		pdir_start++;
    759	}
    760
    761	/* force FDC ops in io_pdir_entry() to be visible to IOMMU */
    762	asm_io_sync();
    763
    764#ifdef ASSERT_PDIR_SANITY
    765	sba_check_pdir(ioc,"Check after sba_map_single()");
    766#endif
    767	spin_unlock_irqrestore(&ioc->res_lock, flags);
    768
    769	/* form complete address */
    770	return SBA_IOVA(ioc, iovp, offset, DEFAULT_DMA_HINT_REG);
    771}
    772
    773
    774static dma_addr_t
    775sba_map_page(struct device *dev, struct page *page, unsigned long offset,
    776		size_t size, enum dma_data_direction direction,
    777		unsigned long attrs)
    778{
    779	return sba_map_single(dev, page_address(page) + offset, size,
    780			direction);
    781}
    782
    783
    784/**
    785 * sba_unmap_page - unmap one IOVA and free resources
    786 * @dev: instance of PCI owned by the driver that's asking.
    787 * @iova:  IOVA of driver buffer previously mapped.
    788 * @size:  number of bytes mapped in driver buffer.
    789 * @direction:  R/W or both.
    790 *
    791 * See Documentation/core-api/dma-api-howto.rst
    792 */
    793static void
    794sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size,
    795		enum dma_data_direction direction, unsigned long attrs)
    796{
    797	struct ioc *ioc;
    798#if DELAYED_RESOURCE_CNT > 0
    799	struct sba_dma_pair *d;
    800#endif
    801	unsigned long flags; 
    802	dma_addr_t offset;
    803
    804	DBG_RUN("%s() iovp 0x%lx/%x\n", __func__, (long) iova, size);
    805
    806	ioc = GET_IOC(dev);
    807	if (!ioc) {
    808		WARN_ON(!ioc);
    809		return;
    810	}
    811	offset = iova & ~IOVP_MASK;
    812	iova ^= offset;        /* clear offset bits */
    813	size += offset;
    814	size = ALIGN(size, IOVP_SIZE);
    815
    816	spin_lock_irqsave(&ioc->res_lock, flags);
    817
    818#ifdef SBA_COLLECT_STATS
    819	ioc->usingle_calls++;
    820	ioc->usingle_pages += size >> IOVP_SHIFT;
    821#endif
    822
    823	sba_mark_invalid(ioc, iova, size);
    824
    825#if DELAYED_RESOURCE_CNT > 0
    826	/* Delaying when we re-use a IO Pdir entry reduces the number
    827	 * of MMIO reads needed to flush writes to the PCOM register.
    828	 */
    829	d = &(ioc->saved[ioc->saved_cnt]);
    830	d->iova = iova;
    831	d->size = size;
    832	if (++(ioc->saved_cnt) >= DELAYED_RESOURCE_CNT) {
    833		int cnt = ioc->saved_cnt;
    834		while (cnt--) {
    835			sba_free_range(ioc, d->iova, d->size);
    836			d--;
    837		}
    838		ioc->saved_cnt = 0;
    839
    840		READ_REG(ioc->ioc_hpa+IOC_PCOM);	/* flush purges */
    841	}
    842#else /* DELAYED_RESOURCE_CNT == 0 */
    843	sba_free_range(ioc, iova, size);
    844
    845	/* If fdc's were issued, force fdc's to be visible now */
    846	asm_io_sync();
    847
    848	READ_REG(ioc->ioc_hpa+IOC_PCOM);	/* flush purges */
    849#endif /* DELAYED_RESOURCE_CNT == 0 */
    850
    851	spin_unlock_irqrestore(&ioc->res_lock, flags);
    852
    853	/* XXX REVISIT for 2.5 Linux - need syncdma for zero-copy support.
    854	** For Astro based systems this isn't a big deal WRT performance.
    855	** As long as 2.4 kernels copyin/copyout data from/to userspace,
    856	** we don't need the syncdma. The issue here is I/O MMU cachelines
    857	** are *not* coherent in all cases.  May be hwrev dependent.
    858	** Need to investigate more.
    859	asm volatile("syncdma");	
    860	*/
    861}
    862
    863
    864/**
    865 * sba_alloc - allocate/map shared mem for DMA
    866 * @hwdev: instance of PCI owned by the driver that's asking.
    867 * @size:  number of bytes mapped in driver buffer.
    868 * @dma_handle:  IOVA of new buffer.
    869 *
    870 * See Documentation/core-api/dma-api-howto.rst
    871 */
    872static void *sba_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle,
    873		gfp_t gfp, unsigned long attrs)
    874{
    875	void *ret;
    876
    877	if (!hwdev) {
    878		/* only support PCI */
    879		*dma_handle = 0;
    880		return NULL;
    881	}
    882
    883        ret = (void *) __get_free_pages(gfp, get_order(size));
    884
    885	if (ret) {
    886		memset(ret, 0, size);
    887		*dma_handle = sba_map_single(hwdev, ret, size, 0);
    888	}
    889
    890	return ret;
    891}
    892
    893
    894/**
    895 * sba_free - free/unmap shared mem for DMA
    896 * @hwdev: instance of PCI owned by the driver that's asking.
    897 * @size:  number of bytes mapped in driver buffer.
    898 * @vaddr:  virtual address IOVA of "consistent" buffer.
    899 * @dma_handler:  IO virtual address of "consistent" buffer.
    900 *
    901 * See Documentation/core-api/dma-api-howto.rst
    902 */
    903static void
    904sba_free(struct device *hwdev, size_t size, void *vaddr,
    905		    dma_addr_t dma_handle, unsigned long attrs)
    906{
    907	sba_unmap_page(hwdev, dma_handle, size, 0, 0);
    908	free_pages((unsigned long) vaddr, get_order(size));
    909}
    910
    911
    912/*
    913** Since 0 is a valid pdir_base index value, can't use that
    914** to determine if a value is valid or not. Use a flag to indicate
    915** the SG list entry contains a valid pdir index.
    916*/
    917#define PIDE_FLAG 0x80000000UL
    918
    919#ifdef SBA_COLLECT_STATS
    920#define IOMMU_MAP_STATS
    921#endif
    922#include "iommu-helpers.h"
    923
    924#ifdef DEBUG_LARGE_SG_ENTRIES
    925int dump_run_sg = 0;
    926#endif
    927
    928
    929/**
    930 * sba_map_sg - map Scatter/Gather list
    931 * @dev: instance of PCI owned by the driver that's asking.
    932 * @sglist:  array of buffer/length pairs
    933 * @nents:  number of entries in list
    934 * @direction:  R/W or both.
    935 *
    936 * See Documentation/core-api/dma-api-howto.rst
    937 */
    938static int
    939sba_map_sg(struct device *dev, struct scatterlist *sglist, int nents,
    940	   enum dma_data_direction direction, unsigned long attrs)
    941{
    942	struct ioc *ioc;
    943	int coalesced, filled = 0;
    944	unsigned long flags;
    945
    946	DBG_RUN_SG("%s() START %d entries\n", __func__, nents);
    947
    948	ioc = GET_IOC(dev);
    949	if (!ioc)
    950		return -EINVAL;
    951
    952	/* Fast path single entry scatterlists. */
    953	if (nents == 1) {
    954		sg_dma_address(sglist) = sba_map_single(dev, sg_virt(sglist),
    955						sglist->length, direction);
    956		sg_dma_len(sglist)     = sglist->length;
    957		return 1;
    958	}
    959
    960	spin_lock_irqsave(&ioc->res_lock, flags);
    961
    962#ifdef ASSERT_PDIR_SANITY
    963	if (sba_check_pdir(ioc,"Check before sba_map_sg()"))
    964	{
    965		sba_dump_sg(ioc, sglist, nents);
    966		panic("Check before sba_map_sg()");
    967	}
    968#endif
    969
    970#ifdef SBA_COLLECT_STATS
    971	ioc->msg_calls++;
    972#endif
    973
    974	/*
    975	** First coalesce the chunks and allocate I/O pdir space
    976	**
    977	** If this is one DMA stream, we can properly map using the
    978	** correct virtual address associated with each DMA page.
    979	** w/o this association, we wouldn't have coherent DMA!
    980	** Access to the virtual address is what forces a two pass algorithm.
    981	*/
    982	coalesced = iommu_coalesce_chunks(ioc, dev, sglist, nents, sba_alloc_range);
    983
    984	/*
    985	** Program the I/O Pdir
    986	**
    987	** map the virtual addresses to the I/O Pdir
    988	** o dma_address will contain the pdir index
    989	** o dma_len will contain the number of bytes to map 
    990	** o address contains the virtual address.
    991	*/
    992	filled = iommu_fill_pdir(ioc, sglist, nents, 0, sba_io_pdir_entry);
    993
    994	/* force FDC ops in io_pdir_entry() to be visible to IOMMU */
    995	asm_io_sync();
    996
    997#ifdef ASSERT_PDIR_SANITY
    998	if (sba_check_pdir(ioc,"Check after sba_map_sg()"))
    999	{
   1000		sba_dump_sg(ioc, sglist, nents);
   1001		panic("Check after sba_map_sg()\n");
   1002	}
   1003#endif
   1004
   1005	spin_unlock_irqrestore(&ioc->res_lock, flags);
   1006
   1007	DBG_RUN_SG("%s() DONE %d mappings\n", __func__, filled);
   1008
   1009	return filled;
   1010}
   1011
   1012
   1013/**
   1014 * sba_unmap_sg - unmap Scatter/Gather list
   1015 * @dev: instance of PCI owned by the driver that's asking.
   1016 * @sglist:  array of buffer/length pairs
   1017 * @nents:  number of entries in list
   1018 * @direction:  R/W or both.
   1019 *
   1020 * See Documentation/core-api/dma-api-howto.rst
   1021 */
   1022static void 
   1023sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
   1024	     enum dma_data_direction direction, unsigned long attrs)
   1025{
   1026	struct ioc *ioc;
   1027#ifdef ASSERT_PDIR_SANITY
   1028	unsigned long flags;
   1029#endif
   1030
   1031	DBG_RUN_SG("%s() START %d entries,  %p,%x\n",
   1032		__func__, nents, sg_virt(sglist), sglist->length);
   1033
   1034	ioc = GET_IOC(dev);
   1035	if (!ioc) {
   1036		WARN_ON(!ioc);
   1037		return;
   1038	}
   1039
   1040#ifdef SBA_COLLECT_STATS
   1041	ioc->usg_calls++;
   1042#endif
   1043
   1044#ifdef ASSERT_PDIR_SANITY
   1045	spin_lock_irqsave(&ioc->res_lock, flags);
   1046	sba_check_pdir(ioc,"Check before sba_unmap_sg()");
   1047	spin_unlock_irqrestore(&ioc->res_lock, flags);
   1048#endif
   1049
   1050	while (nents && sg_dma_len(sglist)) {
   1051
   1052		sba_unmap_page(dev, sg_dma_address(sglist), sg_dma_len(sglist),
   1053				direction, 0);
   1054#ifdef SBA_COLLECT_STATS
   1055		ioc->usg_pages += ((sg_dma_address(sglist) & ~IOVP_MASK) + sg_dma_len(sglist) + IOVP_SIZE - 1) >> PAGE_SHIFT;
   1056		ioc->usingle_calls--;	/* kluge since call is unmap_sg() */
   1057#endif
   1058		++sglist;
   1059		nents--;
   1060	}
   1061
   1062	DBG_RUN_SG("%s() DONE (nents %d)\n", __func__,  nents);
   1063
   1064#ifdef ASSERT_PDIR_SANITY
   1065	spin_lock_irqsave(&ioc->res_lock, flags);
   1066	sba_check_pdir(ioc,"Check after sba_unmap_sg()");
   1067	spin_unlock_irqrestore(&ioc->res_lock, flags);
   1068#endif
   1069
   1070}
   1071
   1072static const struct dma_map_ops sba_ops = {
   1073	.dma_supported =	sba_dma_supported,
   1074	.alloc =		sba_alloc,
   1075	.free =			sba_free,
   1076	.map_page =		sba_map_page,
   1077	.unmap_page =		sba_unmap_page,
   1078	.map_sg =		sba_map_sg,
   1079	.unmap_sg =		sba_unmap_sg,
   1080	.get_sgtable =		dma_common_get_sgtable,
   1081	.alloc_pages =		dma_common_alloc_pages,
   1082	.free_pages =		dma_common_free_pages,
   1083};
   1084
   1085
   1086/**************************************************************************
   1087**
   1088**   SBA PAT PDC support
   1089**
   1090**   o call pdc_pat_cell_module()
   1091**   o store ranges in PCI "resource" structures
   1092**
   1093**************************************************************************/
   1094
   1095static void
   1096sba_get_pat_resources(struct sba_device *sba_dev)
   1097{
   1098#if 0
   1099/*
   1100** TODO/REVISIT/FIXME: support for directed ranges requires calls to
   1101**      PAT PDC to program the SBA/LBA directed range registers...this
   1102**      burden may fall on the LBA code since it directly supports the
   1103**      PCI subsystem. It's not clear yet. - ggg
   1104*/
   1105PAT_MOD(mod)->mod_info.mod_pages   = PAT_GET_MOD_PAGES(temp);
   1106	FIXME : ???
   1107PAT_MOD(mod)->mod_info.dvi         = PAT_GET_DVI(temp);
   1108	Tells where the dvi bits are located in the address.
   1109PAT_MOD(mod)->mod_info.ioc         = PAT_GET_IOC(temp);
   1110	FIXME : ???
   1111#endif
   1112}
   1113
   1114
   1115/**************************************************************
   1116*
   1117*   Initialization and claim
   1118*
   1119***************************************************************/
   1120#define PIRANHA_ADDR_MASK	0x00160000UL /* bit 17,18,20 */
   1121#define PIRANHA_ADDR_VAL	0x00060000UL /* bit 17,18 on */
   1122static void *
   1123sba_alloc_pdir(unsigned int pdir_size)
   1124{
   1125        unsigned long pdir_base;
   1126	unsigned long pdir_order = get_order(pdir_size);
   1127
   1128	pdir_base = __get_free_pages(GFP_KERNEL, pdir_order);
   1129	if (NULL == (void *) pdir_base)	{
   1130		panic("%s() could not allocate I/O Page Table\n",
   1131			__func__);
   1132	}
   1133
   1134	/* If this is not PA8700 (PCX-W2)
   1135	**	OR newer than ver 2.2
   1136	**	OR in a system that doesn't need VINDEX bits from SBA,
   1137	**
   1138	** then we aren't exposed to the HW bug.
   1139	*/
   1140	if ( ((boot_cpu_data.pdc.cpuid >> 5) & 0x7f) != 0x13
   1141			|| (boot_cpu_data.pdc.versions > 0x202)
   1142			|| (boot_cpu_data.pdc.capabilities & 0x08L) )
   1143		return (void *) pdir_base;
   1144
   1145	/*
   1146	 * PA8700 (PCX-W2, aka piranha) silent data corruption fix
   1147	 *
   1148	 * An interaction between PA8700 CPU (Ver 2.2 or older) and
   1149	 * Ike/Astro can cause silent data corruption. This is only
   1150	 * a problem if the I/O PDIR is located in memory such that
   1151	 * (little-endian)  bits 17 and 18 are on and bit 20 is off.
   1152	 *
   1153	 * Since the max IO Pdir size is 2MB, by cleverly allocating the
   1154	 * right physical address, we can either avoid (IOPDIR <= 1MB)
   1155	 * or minimize (2MB IO Pdir) the problem if we restrict the
   1156	 * IO Pdir to a maximum size of 2MB-128K (1902K).
   1157	 *
   1158	 * Because we always allocate 2^N sized IO pdirs, either of the
   1159	 * "bad" regions will be the last 128K if at all. That's easy
   1160	 * to test for.
   1161	 * 
   1162	 */
   1163	if (pdir_order <= (19-12)) {
   1164		if (((virt_to_phys(pdir_base)+pdir_size-1) & PIRANHA_ADDR_MASK) == PIRANHA_ADDR_VAL) {
   1165			/* allocate a new one on 512k alignment */
   1166			unsigned long new_pdir = __get_free_pages(GFP_KERNEL, (19-12));
   1167			/* release original */
   1168			free_pages(pdir_base, pdir_order);
   1169
   1170			pdir_base = new_pdir;
   1171
   1172			/* release excess */
   1173			while (pdir_order < (19-12)) {
   1174				new_pdir += pdir_size;
   1175				free_pages(new_pdir, pdir_order);
   1176				pdir_order +=1;
   1177				pdir_size <<=1;
   1178			}
   1179		}
   1180	} else {
   1181		/*
   1182		** 1MB or 2MB Pdir
   1183		** Needs to be aligned on an "odd" 1MB boundary.
   1184		*/
   1185		unsigned long new_pdir = __get_free_pages(GFP_KERNEL, pdir_order+1); /* 2 or 4MB */
   1186
   1187		/* release original */
   1188		free_pages( pdir_base, pdir_order);
   1189
   1190		/* release first 1MB */
   1191		free_pages(new_pdir, 20-12);
   1192
   1193		pdir_base = new_pdir + 1024*1024;
   1194
   1195		if (pdir_order > (20-12)) {
   1196			/*
   1197			** 2MB Pdir.
   1198			**
   1199			** Flag tells init_bitmap() to mark bad 128k as used
   1200			** and to reduce the size by 128k.
   1201			*/
   1202			piranha_bad_128k = 1;
   1203
   1204			new_pdir += 3*1024*1024;
   1205			/* release last 1MB */
   1206			free_pages(new_pdir, 20-12);
   1207
   1208			/* release unusable 128KB */
   1209			free_pages(new_pdir - 128*1024 , 17-12);
   1210
   1211			pdir_size -= 128*1024;
   1212		}
   1213	}
   1214
   1215	memset((void *) pdir_base, 0, pdir_size);
   1216	return (void *) pdir_base;
   1217}
   1218
   1219struct ibase_data_struct {
   1220	struct ioc *ioc;
   1221	int ioc_num;
   1222};
   1223
   1224static int setup_ibase_imask_callback(struct device *dev, void *data)
   1225{
   1226	/* lba_set_iregs() is in drivers/parisc/lba_pci.c */
   1227        extern void lba_set_iregs(struct parisc_device *, u32, u32);
   1228	struct parisc_device *lba = to_parisc_device(dev);
   1229	struct ibase_data_struct *ibd = data;
   1230	int rope_num = (lba->hpa.start >> 13) & 0xf;
   1231	if (rope_num >> 3 == ibd->ioc_num)
   1232		lba_set_iregs(lba, ibd->ioc->ibase, ibd->ioc->imask);
   1233	return 0;
   1234}
   1235
   1236/* setup Mercury or Elroy IBASE/IMASK registers. */
   1237static void 
   1238setup_ibase_imask(struct parisc_device *sba, struct ioc *ioc, int ioc_num)
   1239{
   1240	struct ibase_data_struct ibase_data = {
   1241		.ioc		= ioc,
   1242		.ioc_num	= ioc_num,
   1243	};
   1244
   1245	device_for_each_child(&sba->dev, &ibase_data,
   1246			      setup_ibase_imask_callback);
   1247}
   1248
   1249#ifdef SBA_AGP_SUPPORT
   1250static int
   1251sba_ioc_find_quicksilver(struct device *dev, void *data)
   1252{
   1253	int *agp_found = data;
   1254	struct parisc_device *lba = to_parisc_device(dev);
   1255
   1256	if (IS_QUICKSILVER(lba))
   1257		*agp_found = 1;
   1258	return 0;
   1259}
   1260#endif
   1261
   1262static void
   1263sba_ioc_init_pluto(struct parisc_device *sba, struct ioc *ioc, int ioc_num)
   1264{
   1265	u32 iova_space_mask;
   1266	u32 iova_space_size;
   1267	int iov_order, tcnfg;
   1268#ifdef SBA_AGP_SUPPORT
   1269	int agp_found = 0;
   1270#endif
   1271	/*
   1272	** Firmware programs the base and size of a "safe IOVA space"
   1273	** (one that doesn't overlap memory or LMMIO space) in the
   1274	** IBASE and IMASK registers.
   1275	*/
   1276	ioc->ibase = READ_REG(ioc->ioc_hpa + IOC_IBASE) & ~0x1fffffULL;
   1277	iova_space_size = ~(READ_REG(ioc->ioc_hpa + IOC_IMASK) & 0xFFFFFFFFUL) + 1;
   1278
   1279	if ((ioc->ibase < 0xfed00000UL) && ((ioc->ibase + iova_space_size) > 0xfee00000UL)) {
   1280		printk("WARNING: IOV space overlaps local config and interrupt message, truncating\n");
   1281		iova_space_size /= 2;
   1282	}
   1283
   1284	/*
   1285	** iov_order is always based on a 1GB IOVA space since we want to
   1286	** turn on the other half for AGP GART.
   1287	*/
   1288	iov_order = get_order(iova_space_size >> (IOVP_SHIFT - PAGE_SHIFT));
   1289	ioc->pdir_size = (iova_space_size / IOVP_SIZE) * sizeof(u64);
   1290
   1291	DBG_INIT("%s() hpa 0x%p IOV %dMB (%d bits)\n",
   1292		__func__, ioc->ioc_hpa, iova_space_size >> 20,
   1293		iov_order + PAGE_SHIFT);
   1294
   1295	ioc->pdir_base = (void *) __get_free_pages(GFP_KERNEL,
   1296						   get_order(ioc->pdir_size));
   1297	if (!ioc->pdir_base)
   1298		panic("Couldn't allocate I/O Page Table\n");
   1299
   1300	memset(ioc->pdir_base, 0, ioc->pdir_size);
   1301
   1302	DBG_INIT("%s() pdir %p size %x\n",
   1303			__func__, ioc->pdir_base, ioc->pdir_size);
   1304
   1305#ifdef SBA_HINT_SUPPORT
   1306	ioc->hint_shift_pdir = iov_order + PAGE_SHIFT;
   1307	ioc->hint_mask_pdir = ~(0x3 << (iov_order + PAGE_SHIFT));
   1308
   1309	DBG_INIT("	hint_shift_pdir %x hint_mask_pdir %lx\n",
   1310		ioc->hint_shift_pdir, ioc->hint_mask_pdir);
   1311#endif
   1312
   1313	WARN_ON((((unsigned long) ioc->pdir_base) & PAGE_MASK) != (unsigned long) ioc->pdir_base);
   1314	WRITE_REG(virt_to_phys(ioc->pdir_base), ioc->ioc_hpa + IOC_PDIR_BASE);
   1315
   1316	/* build IMASK for IOC and Elroy */
   1317	iova_space_mask =  0xffffffff;
   1318	iova_space_mask <<= (iov_order + PAGE_SHIFT);
   1319	ioc->imask = iova_space_mask;
   1320#ifdef ZX1_SUPPORT
   1321	ioc->iovp_mask = ~(iova_space_mask + PAGE_SIZE - 1);
   1322#endif
   1323	sba_dump_tlb(ioc->ioc_hpa);
   1324
   1325	setup_ibase_imask(sba, ioc, ioc_num);
   1326
   1327	WRITE_REG(ioc->imask, ioc->ioc_hpa + IOC_IMASK);
   1328
   1329#ifdef CONFIG_64BIT
   1330	/*
   1331	** Setting the upper bits makes checking for bypass addresses
   1332	** a little faster later on.
   1333	*/
   1334	ioc->imask |= 0xFFFFFFFF00000000UL;
   1335#endif
   1336
   1337	/* Set I/O PDIR Page size to system page size */
   1338	switch (PAGE_SHIFT) {
   1339		case 12: tcnfg = 0; break;	/*  4K */
   1340		case 13: tcnfg = 1; break;	/*  8K */
   1341		case 14: tcnfg = 2; break;	/* 16K */
   1342		case 16: tcnfg = 3; break;	/* 64K */
   1343		default:
   1344			panic(__FILE__ "Unsupported system page size %d",
   1345				1 << PAGE_SHIFT);
   1346			break;
   1347	}
   1348	WRITE_REG(tcnfg, ioc->ioc_hpa + IOC_TCNFG);
   1349
   1350	/*
   1351	** Program the IOC's ibase and enable IOVA translation
   1352	** Bit zero == enable bit.
   1353	*/
   1354	WRITE_REG(ioc->ibase | 1, ioc->ioc_hpa + IOC_IBASE);
   1355
   1356	/*
   1357	** Clear I/O TLB of any possible entries.
   1358	** (Yes. This is a bit paranoid...but so what)
   1359	*/
   1360	WRITE_REG(ioc->ibase | 31, ioc->ioc_hpa + IOC_PCOM);
   1361
   1362#ifdef SBA_AGP_SUPPORT
   1363
   1364	/*
   1365	** If an AGP device is present, only use half of the IOV space
   1366	** for PCI DMA.  Unfortunately we can't know ahead of time
   1367	** whether GART support will actually be used, for now we
   1368	** can just key on any AGP device found in the system.
   1369	** We program the next pdir index after we stop w/ a key for
   1370	** the GART code to handshake on.
   1371	*/
   1372	device_for_each_child(&sba->dev, &agp_found, sba_ioc_find_quicksilver);
   1373
   1374	if (agp_found && sba_reserve_agpgart) {
   1375		printk(KERN_INFO "%s: reserving %dMb of IOVA space for agpgart\n",
   1376		       __func__, (iova_space_size/2) >> 20);
   1377		ioc->pdir_size /= 2;
   1378		ioc->pdir_base[PDIR_INDEX(iova_space_size/2)] = SBA_AGPGART_COOKIE;
   1379	}
   1380#endif /*SBA_AGP_SUPPORT*/
   1381}
   1382
   1383static void
   1384sba_ioc_init(struct parisc_device *sba, struct ioc *ioc, int ioc_num)
   1385{
   1386	u32 iova_space_size, iova_space_mask;
   1387	unsigned int pdir_size, iov_order, tcnfg;
   1388
   1389	/*
   1390	** Determine IOVA Space size from memory size.
   1391	**
   1392	** Ideally, PCI drivers would register the maximum number
   1393	** of DMA they can have outstanding for each device they
   1394	** own.  Next best thing would be to guess how much DMA
   1395	** can be outstanding based on PCI Class/sub-class. Both
   1396	** methods still require some "extra" to support PCI
   1397	** Hot-Plug/Removal of PCI cards. (aka PCI OLARD).
   1398	**
   1399	** While we have 32-bits "IOVA" space, top two 2 bits are used
   1400	** for DMA hints - ergo only 30 bits max.
   1401	*/
   1402
   1403	iova_space_size = (u32) (totalram_pages()/global_ioc_cnt);
   1404
   1405	/* limit IOVA space size to 1MB-1GB */
   1406	if (iova_space_size < (1 << (20 - PAGE_SHIFT))) {
   1407		iova_space_size = 1 << (20 - PAGE_SHIFT);
   1408	}
   1409	else if (iova_space_size > (1 << (30 - PAGE_SHIFT))) {
   1410		iova_space_size = 1 << (30 - PAGE_SHIFT);
   1411	}
   1412
   1413	/*
   1414	** iova space must be log2() in size.
   1415	** thus, pdir/res_map will also be log2().
   1416	** PIRANHA BUG: Exception is when IO Pdir is 2MB (gets reduced)
   1417	*/
   1418	iov_order = get_order(iova_space_size << PAGE_SHIFT);
   1419
   1420	/* iova_space_size is now bytes, not pages */
   1421	iova_space_size = 1 << (iov_order + PAGE_SHIFT);
   1422
   1423	ioc->pdir_size = pdir_size = (iova_space_size/IOVP_SIZE) * sizeof(u64);
   1424
   1425	DBG_INIT("%s() hpa 0x%lx mem %ldMB IOV %dMB (%d bits)\n",
   1426			__func__,
   1427			ioc->ioc_hpa,
   1428			(unsigned long) totalram_pages() >> (20 - PAGE_SHIFT),
   1429			iova_space_size>>20,
   1430			iov_order + PAGE_SHIFT);
   1431
   1432	ioc->pdir_base = sba_alloc_pdir(pdir_size);
   1433
   1434	DBG_INIT("%s() pdir %p size %x\n",
   1435			__func__, ioc->pdir_base, pdir_size);
   1436
   1437#ifdef SBA_HINT_SUPPORT
   1438	/* FIXME : DMA HINTs not used */
   1439	ioc->hint_shift_pdir = iov_order + PAGE_SHIFT;
   1440	ioc->hint_mask_pdir = ~(0x3 << (iov_order + PAGE_SHIFT));
   1441
   1442	DBG_INIT("	hint_shift_pdir %x hint_mask_pdir %lx\n",
   1443			ioc->hint_shift_pdir, ioc->hint_mask_pdir);
   1444#endif
   1445
   1446	WRITE_REG64(virt_to_phys(ioc->pdir_base), ioc->ioc_hpa + IOC_PDIR_BASE);
   1447
   1448	/* build IMASK for IOC and Elroy */
   1449	iova_space_mask =  0xffffffff;
   1450	iova_space_mask <<= (iov_order + PAGE_SHIFT);
   1451
   1452	/*
   1453	** On C3000 w/512MB mem, HP-UX 10.20 reports:
   1454	**     ibase=0, imask=0xFE000000, size=0x2000000.
   1455	*/
   1456	ioc->ibase = 0;
   1457	ioc->imask = iova_space_mask;	/* save it */
   1458#ifdef ZX1_SUPPORT
   1459	ioc->iovp_mask = ~(iova_space_mask + PAGE_SIZE - 1);
   1460#endif
   1461
   1462	DBG_INIT("%s() IOV base 0x%lx mask 0x%0lx\n",
   1463		__func__, ioc->ibase, ioc->imask);
   1464
   1465	/*
   1466	** FIXME: Hint registers are programmed with default hint
   1467	** values during boot, so hints should be sane even if we
   1468	** can't reprogram them the way drivers want.
   1469	*/
   1470
   1471	setup_ibase_imask(sba, ioc, ioc_num);
   1472
   1473	/*
   1474	** Program the IOC's ibase and enable IOVA translation
   1475	*/
   1476	WRITE_REG(ioc->ibase | 1, ioc->ioc_hpa+IOC_IBASE);
   1477	WRITE_REG(ioc->imask, ioc->ioc_hpa+IOC_IMASK);
   1478
   1479	/* Set I/O PDIR Page size to system page size */
   1480	switch (PAGE_SHIFT) {
   1481		case 12: tcnfg = 0; break;	/*  4K */
   1482		case 13: tcnfg = 1; break;	/*  8K */
   1483		case 14: tcnfg = 2; break;	/* 16K */
   1484		case 16: tcnfg = 3; break;	/* 64K */
   1485		default:
   1486			panic(__FILE__ "Unsupported system page size %d",
   1487				1 << PAGE_SHIFT);
   1488			break;
   1489	}
   1490	/* Set I/O PDIR Page size to PAGE_SIZE (4k/16k/...) */
   1491	WRITE_REG(tcnfg, ioc->ioc_hpa+IOC_TCNFG);
   1492
   1493	/*
   1494	** Clear I/O TLB of any possible entries.
   1495	** (Yes. This is a bit paranoid...but so what)
   1496	*/
   1497	WRITE_REG(0 | 31, ioc->ioc_hpa+IOC_PCOM);
   1498
   1499	ioc->ibase = 0; /* used by SBA_IOVA and related macros */	
   1500
   1501	DBG_INIT("%s() DONE\n", __func__);
   1502}
   1503
   1504
   1505
   1506/**************************************************************************
   1507**
   1508**   SBA initialization code (HW and SW)
   1509**
   1510**   o identify SBA chip itself
   1511**   o initialize SBA chip modes (HardFail)
   1512**   o initialize SBA chip modes (HardFail)
   1513**   o FIXME: initialize DMA hints for reasonable defaults
   1514**
   1515**************************************************************************/
   1516
   1517static void __iomem *ioc_remap(struct sba_device *sba_dev, unsigned int offset)
   1518{
   1519	return ioremap(sba_dev->dev->hpa.start + offset, SBA_FUNC_SIZE);
   1520}
   1521
   1522static void sba_hw_init(struct sba_device *sba_dev)
   1523{ 
   1524	int i;
   1525	int num_ioc;
   1526	u64 ioc_ctl;
   1527
   1528	if (!is_pdc_pat()) {
   1529		/* Shutdown the USB controller on Astro-based workstations.
   1530		** Once we reprogram the IOMMU, the next DMA performed by
   1531		** USB will HPMC the box. USB is only enabled if a
   1532		** keyboard is present and found.
   1533		**
   1534		** With serial console, j6k v5.0 firmware says:
   1535		**   mem_kbd hpa 0xfee003f8 sba 0x0 pad 0x0 cl_class 0x7
   1536		**
   1537		** FIXME: Using GFX+USB console at power up but direct
   1538		**	linux to serial console is still broken.
   1539		**	USB could generate DMA so we must reset USB.
   1540		**	The proper sequence would be:
   1541		**	o block console output
   1542		**	o reset USB device
   1543		**	o reprogram serial port
   1544		**	o unblock console output
   1545		*/
   1546		if (PAGE0->mem_kbd.cl_class == CL_KEYBD) {
   1547			pdc_io_reset_devices();
   1548		}
   1549
   1550	}
   1551
   1552
   1553#if 0
   1554printk("sba_hw_init(): mem_boot 0x%x 0x%x 0x%x 0x%x\n", PAGE0->mem_boot.hpa,
   1555	PAGE0->mem_boot.spa, PAGE0->mem_boot.pad, PAGE0->mem_boot.cl_class);
   1556
   1557	/*
   1558	** Need to deal with DMA from LAN.
   1559	**	Maybe use page zero boot device as a handle to talk
   1560	**	to PDC about which device to shutdown.
   1561	**
   1562	** Netbooting, j6k v5.0 firmware says:
   1563	** 	mem_boot hpa 0xf4008000 sba 0x0 pad 0x0 cl_class 0x1002
   1564	** ARGH! invalid class.
   1565	*/
   1566	if ((PAGE0->mem_boot.cl_class != CL_RANDOM)
   1567		&& (PAGE0->mem_boot.cl_class != CL_SEQU)) {
   1568			pdc_io_reset();
   1569	}
   1570#endif
   1571
   1572	if (!IS_PLUTO(sba_dev->dev)) {
   1573		ioc_ctl = READ_REG(sba_dev->sba_hpa+IOC_CTRL);
   1574		DBG_INIT("%s() hpa 0x%lx ioc_ctl 0x%Lx ->",
   1575			__func__, sba_dev->sba_hpa, ioc_ctl);
   1576		ioc_ctl &= ~(IOC_CTRL_RM | IOC_CTRL_NC | IOC_CTRL_CE);
   1577		ioc_ctl |= IOC_CTRL_DD | IOC_CTRL_D4 | IOC_CTRL_TC;
   1578			/* j6700 v1.6 firmware sets 0x294f */
   1579			/* A500 firmware sets 0x4d */
   1580
   1581		WRITE_REG(ioc_ctl, sba_dev->sba_hpa+IOC_CTRL);
   1582
   1583#ifdef DEBUG_SBA_INIT
   1584		ioc_ctl = READ_REG64(sba_dev->sba_hpa+IOC_CTRL);
   1585		DBG_INIT(" 0x%Lx\n", ioc_ctl);
   1586#endif
   1587	} /* if !PLUTO */
   1588
   1589	if (IS_ASTRO(sba_dev->dev)) {
   1590		int err;
   1591		sba_dev->ioc[0].ioc_hpa = ioc_remap(sba_dev, ASTRO_IOC_OFFSET);
   1592		num_ioc = 1;
   1593
   1594		sba_dev->chip_resv.name = "Astro Intr Ack";
   1595		sba_dev->chip_resv.start = PCI_F_EXTEND | 0xfef00000UL;
   1596		sba_dev->chip_resv.end   = PCI_F_EXTEND | (0xff000000UL - 1) ;
   1597		err = request_resource(&iomem_resource, &(sba_dev->chip_resv));
   1598		BUG_ON(err < 0);
   1599
   1600	} else if (IS_PLUTO(sba_dev->dev)) {
   1601		int err;
   1602
   1603		sba_dev->ioc[0].ioc_hpa = ioc_remap(sba_dev, PLUTO_IOC_OFFSET);
   1604		num_ioc = 1;
   1605
   1606		sba_dev->chip_resv.name = "Pluto Intr/PIOP/VGA";
   1607		sba_dev->chip_resv.start = PCI_F_EXTEND | 0xfee00000UL;
   1608		sba_dev->chip_resv.end   = PCI_F_EXTEND | (0xff200000UL - 1);
   1609		err = request_resource(&iomem_resource, &(sba_dev->chip_resv));
   1610		WARN_ON(err < 0);
   1611
   1612		sba_dev->iommu_resv.name = "IOVA Space";
   1613		sba_dev->iommu_resv.start = 0x40000000UL;
   1614		sba_dev->iommu_resv.end   = 0x50000000UL - 1;
   1615		err = request_resource(&iomem_resource, &(sba_dev->iommu_resv));
   1616		WARN_ON(err < 0);
   1617	} else {
   1618		/* IKE, REO */
   1619		sba_dev->ioc[0].ioc_hpa = ioc_remap(sba_dev, IKE_IOC_OFFSET(0));
   1620		sba_dev->ioc[1].ioc_hpa = ioc_remap(sba_dev, IKE_IOC_OFFSET(1));
   1621		num_ioc = 2;
   1622
   1623		/* TODO - LOOKUP Ike/Stretch chipset mem map */
   1624	}
   1625	/* XXX: What about Reo Grande? */
   1626
   1627	sba_dev->num_ioc = num_ioc;
   1628	for (i = 0; i < num_ioc; i++) {
   1629		void __iomem *ioc_hpa = sba_dev->ioc[i].ioc_hpa;
   1630		unsigned int j;
   1631
   1632		for (j=0; j < sizeof(u64) * ROPES_PER_IOC; j+=sizeof(u64)) {
   1633
   1634			/*
   1635			 * Clear ROPE(N)_CONFIG AO bit.
   1636			 * Disables "NT Ordering" (~= !"Relaxed Ordering")
   1637			 * Overrides bit 1 in DMA Hint Sets.
   1638			 * Improves netperf UDP_STREAM by ~10% for bcm5701.
   1639			 */
   1640			if (IS_PLUTO(sba_dev->dev)) {
   1641				void __iomem *rope_cfg;
   1642				unsigned long cfg_val;
   1643
   1644				rope_cfg = ioc_hpa + IOC_ROPE0_CFG + j;
   1645				cfg_val = READ_REG(rope_cfg);
   1646				cfg_val &= ~IOC_ROPE_AO;
   1647				WRITE_REG(cfg_val, rope_cfg);
   1648			}
   1649
   1650			/*
   1651			** Make sure the box crashes on rope errors.
   1652			*/
   1653			WRITE_REG(HF_ENABLE, ioc_hpa + ROPE0_CTL + j);
   1654		}
   1655
   1656		/* flush out the last writes */
   1657		READ_REG(sba_dev->ioc[i].ioc_hpa + ROPE7_CTL);
   1658
   1659		DBG_INIT("	ioc[%d] ROPE_CFG 0x%Lx  ROPE_DBG 0x%Lx\n",
   1660				i,
   1661				READ_REG(sba_dev->ioc[i].ioc_hpa + 0x40),
   1662				READ_REG(sba_dev->ioc[i].ioc_hpa + 0x50)
   1663			);
   1664		DBG_INIT("	STATUS_CONTROL 0x%Lx  FLUSH_CTRL 0x%Lx\n",
   1665				READ_REG(sba_dev->ioc[i].ioc_hpa + 0x108),
   1666				READ_REG(sba_dev->ioc[i].ioc_hpa + 0x400)
   1667			);
   1668
   1669		if (IS_PLUTO(sba_dev->dev)) {
   1670			sba_ioc_init_pluto(sba_dev->dev, &(sba_dev->ioc[i]), i);
   1671		} else {
   1672			sba_ioc_init(sba_dev->dev, &(sba_dev->ioc[i]), i);
   1673		}
   1674	}
   1675}
   1676
   1677static void
   1678sba_common_init(struct sba_device *sba_dev)
   1679{
   1680	int i;
   1681
   1682	/* add this one to the head of the list (order doesn't matter)
   1683	** This will be useful for debugging - especially if we get coredumps
   1684	*/
   1685	sba_dev->next = sba_list;
   1686	sba_list = sba_dev;
   1687
   1688	for(i=0; i< sba_dev->num_ioc; i++) {
   1689		int res_size;
   1690#ifdef DEBUG_DMB_TRAP
   1691		extern void iterate_pages(unsigned long , unsigned long ,
   1692					  void (*)(pte_t * , unsigned long),
   1693					  unsigned long );
   1694		void set_data_memory_break(pte_t * , unsigned long);
   1695#endif
   1696		/* resource map size dictated by pdir_size */
   1697		res_size = sba_dev->ioc[i].pdir_size/sizeof(u64); /* entries */
   1698
   1699		/* Second part of PIRANHA BUG */
   1700		if (piranha_bad_128k) {
   1701			res_size -= (128*1024)/sizeof(u64);
   1702		}
   1703
   1704		res_size >>= 3;  /* convert bit count to byte count */
   1705		DBG_INIT("%s() res_size 0x%x\n",
   1706			__func__, res_size);
   1707
   1708		sba_dev->ioc[i].res_size = res_size;
   1709		sba_dev->ioc[i].res_map = (char *) __get_free_pages(GFP_KERNEL, get_order(res_size));
   1710
   1711#ifdef DEBUG_DMB_TRAP
   1712		iterate_pages( sba_dev->ioc[i].res_map, res_size,
   1713				set_data_memory_break, 0);
   1714#endif
   1715
   1716		if (NULL == sba_dev->ioc[i].res_map)
   1717		{
   1718			panic("%s:%s() could not allocate resource map\n",
   1719			      __FILE__, __func__ );
   1720		}
   1721
   1722		memset(sba_dev->ioc[i].res_map, 0, res_size);
   1723		/* next available IOVP - circular search */
   1724		sba_dev->ioc[i].res_hint = (unsigned long *)
   1725				&(sba_dev->ioc[i].res_map[L1_CACHE_BYTES]);
   1726
   1727#ifdef ASSERT_PDIR_SANITY
   1728		/* Mark first bit busy - ie no IOVA 0 */
   1729		sba_dev->ioc[i].res_map[0] = 0x80;
   1730		sba_dev->ioc[i].pdir_base[0] = 0xeeffc0addbba0080ULL;
   1731#endif
   1732
   1733		/* Third (and last) part of PIRANHA BUG */
   1734		if (piranha_bad_128k) {
   1735			/* region from +1408K to +1536 is un-usable. */
   1736
   1737			int idx_start = (1408*1024/sizeof(u64)) >> 3;
   1738			int idx_end   = (1536*1024/sizeof(u64)) >> 3;
   1739			long *p_start = (long *) &(sba_dev->ioc[i].res_map[idx_start]);
   1740			long *p_end   = (long *) &(sba_dev->ioc[i].res_map[idx_end]);
   1741
   1742			/* mark that part of the io pdir busy */
   1743			while (p_start < p_end)
   1744				*p_start++ = -1;
   1745				
   1746		}
   1747
   1748#ifdef DEBUG_DMB_TRAP
   1749		iterate_pages( sba_dev->ioc[i].res_map, res_size,
   1750				set_data_memory_break, 0);
   1751		iterate_pages( sba_dev->ioc[i].pdir_base, sba_dev->ioc[i].pdir_size,
   1752				set_data_memory_break, 0);
   1753#endif
   1754
   1755		DBG_INIT("%s() %d res_map %x %p\n",
   1756			__func__, i, res_size, sba_dev->ioc[i].res_map);
   1757	}
   1758
   1759	spin_lock_init(&sba_dev->sba_lock);
   1760	ioc_needs_fdc = boot_cpu_data.pdc.capabilities & PDC_MODEL_IOPDIR_FDC;
   1761
   1762#ifdef DEBUG_SBA_INIT
   1763	/*
   1764	 * If the PDC_MODEL capabilities has Non-coherent IO-PDIR bit set
   1765	 * (bit #61, big endian), we have to flush and sync every time
   1766	 * IO-PDIR is changed in Ike/Astro.
   1767	 */
   1768	if (ioc_needs_fdc) {
   1769		printk(KERN_INFO MODULE_NAME " FDC/SYNC required.\n");
   1770	} else {
   1771		printk(KERN_INFO MODULE_NAME " IOC has cache coherent PDIR.\n");
   1772	}
   1773#endif
   1774}
   1775
   1776#ifdef CONFIG_PROC_FS
   1777static int sba_proc_info(struct seq_file *m, void *p)
   1778{
   1779	struct sba_device *sba_dev = sba_list;
   1780	struct ioc *ioc = &sba_dev->ioc[0];	/* FIXME: Multi-IOC support! */
   1781	int total_pages = (int) (ioc->res_size << 3); /* 8 bits per byte */
   1782#ifdef SBA_COLLECT_STATS
   1783	unsigned long avg = 0, min, max;
   1784#endif
   1785	int i;
   1786
   1787	seq_printf(m, "%s rev %d.%d\n",
   1788		   sba_dev->name,
   1789		   (sba_dev->hw_rev & 0x7) + 1,
   1790		   (sba_dev->hw_rev & 0x18) >> 3);
   1791	seq_printf(m, "IO PDIR size    : %d bytes (%d entries)\n",
   1792		   (int)((ioc->res_size << 3) * sizeof(u64)), /* 8 bits/byte */
   1793		   total_pages);
   1794
   1795	seq_printf(m, "Resource bitmap : %d bytes (%d pages)\n",
   1796		   ioc->res_size, ioc->res_size << 3);   /* 8 bits per byte */
   1797
   1798	seq_printf(m, "LMMIO_BASE/MASK/ROUTE %08x %08x %08x\n",
   1799		   READ_REG32(sba_dev->sba_hpa + LMMIO_DIST_BASE),
   1800		   READ_REG32(sba_dev->sba_hpa + LMMIO_DIST_MASK),
   1801		   READ_REG32(sba_dev->sba_hpa + LMMIO_DIST_ROUTE));
   1802
   1803	for (i=0; i<4; i++)
   1804		seq_printf(m, "DIR%d_BASE/MASK/ROUTE %08x %08x %08x\n",
   1805			   i,
   1806			   READ_REG32(sba_dev->sba_hpa + LMMIO_DIRECT0_BASE  + i*0x18),
   1807			   READ_REG32(sba_dev->sba_hpa + LMMIO_DIRECT0_MASK  + i*0x18),
   1808			   READ_REG32(sba_dev->sba_hpa + LMMIO_DIRECT0_ROUTE + i*0x18));
   1809
   1810#ifdef SBA_COLLECT_STATS
   1811	seq_printf(m, "IO PDIR entries : %ld free  %ld used (%d%%)\n",
   1812		   total_pages - ioc->used_pages, ioc->used_pages,
   1813		   (int)(ioc->used_pages * 100 / total_pages));
   1814
   1815	min = max = ioc->avg_search[0];
   1816	for (i = 0; i < SBA_SEARCH_SAMPLE; i++) {
   1817		avg += ioc->avg_search[i];
   1818		if (ioc->avg_search[i] > max) max = ioc->avg_search[i];
   1819		if (ioc->avg_search[i] < min) min = ioc->avg_search[i];
   1820	}
   1821	avg /= SBA_SEARCH_SAMPLE;
   1822	seq_printf(m, "  Bitmap search : %ld/%ld/%ld (min/avg/max CPU Cycles)\n",
   1823		   min, avg, max);
   1824
   1825	seq_printf(m, "pci_map_single(): %12ld calls  %12ld pages (avg %d/1000)\n",
   1826		   ioc->msingle_calls, ioc->msingle_pages,
   1827		   (int)((ioc->msingle_pages * 1000)/ioc->msingle_calls));
   1828
   1829	/* KLUGE - unmap_sg calls unmap_single for each mapped page */
   1830	min = ioc->usingle_calls;
   1831	max = ioc->usingle_pages - ioc->usg_pages;
   1832	seq_printf(m, "pci_unmap_single: %12ld calls  %12ld pages (avg %d/1000)\n",
   1833		   min, max, (int)((max * 1000)/min));
   1834
   1835	seq_printf(m, "pci_map_sg()    : %12ld calls  %12ld pages (avg %d/1000)\n",
   1836		   ioc->msg_calls, ioc->msg_pages,
   1837		   (int)((ioc->msg_pages * 1000)/ioc->msg_calls));
   1838
   1839	seq_printf(m, "pci_unmap_sg()  : %12ld calls  %12ld pages (avg %d/1000)\n",
   1840		   ioc->usg_calls, ioc->usg_pages,
   1841		   (int)((ioc->usg_pages * 1000)/ioc->usg_calls));
   1842#endif
   1843
   1844	return 0;
   1845}
   1846
   1847static int
   1848sba_proc_bitmap_info(struct seq_file *m, void *p)
   1849{
   1850	struct sba_device *sba_dev = sba_list;
   1851	struct ioc *ioc = &sba_dev->ioc[0];	/* FIXME: Multi-IOC support! */
   1852
   1853	seq_hex_dump(m, "   ", DUMP_PREFIX_NONE, 32, 4, ioc->res_map,
   1854		     ioc->res_size, false);
   1855	seq_putc(m, '\n');
   1856
   1857	return 0;
   1858}
   1859#endif /* CONFIG_PROC_FS */
   1860
   1861static const struct parisc_device_id sba_tbl[] __initconst = {
   1862	{ HPHW_IOA, HVERSION_REV_ANY_ID, ASTRO_RUNWAY_PORT, 0xb },
   1863	{ HPHW_BCPORT, HVERSION_REV_ANY_ID, IKE_MERCED_PORT, 0xc },
   1864	{ HPHW_BCPORT, HVERSION_REV_ANY_ID, REO_MERCED_PORT, 0xc },
   1865	{ HPHW_BCPORT, HVERSION_REV_ANY_ID, REOG_MERCED_PORT, 0xc },
   1866	{ HPHW_IOA, HVERSION_REV_ANY_ID, PLUTO_MCKINLEY_PORT, 0xc },
   1867	{ 0, }
   1868};
   1869
   1870static int sba_driver_callback(struct parisc_device *);
   1871
   1872static struct parisc_driver sba_driver __refdata = {
   1873	.name =		MODULE_NAME,
   1874	.id_table =	sba_tbl,
   1875	.probe =	sba_driver_callback,
   1876};
   1877
   1878/*
   1879** Determine if sba should claim this chip (return 0) or not (return 1).
   1880** If so, initialize the chip and tell other partners in crime they
   1881** have work to do.
   1882*/
   1883static int __init sba_driver_callback(struct parisc_device *dev)
   1884{
   1885	struct sba_device *sba_dev;
   1886	u32 func_class;
   1887	int i;
   1888	char *version;
   1889	void __iomem *sba_addr = ioremap(dev->hpa.start, SBA_FUNC_SIZE);
   1890#ifdef CONFIG_PROC_FS
   1891	struct proc_dir_entry *root;
   1892#endif
   1893
   1894	sba_dump_ranges(sba_addr);
   1895
   1896	/* Read HW Rev First */
   1897	func_class = READ_REG(sba_addr + SBA_FCLASS);
   1898
   1899	if (IS_ASTRO(dev)) {
   1900		unsigned long fclass;
   1901		static char astro_rev[]="Astro ?.?";
   1902
   1903		/* Astro is broken...Read HW Rev First */
   1904		fclass = READ_REG(sba_addr);
   1905
   1906		astro_rev[6] = '1' + (char) (fclass & 0x7);
   1907		astro_rev[8] = '0' + (char) ((fclass & 0x18) >> 3);
   1908		version = astro_rev;
   1909
   1910	} else if (IS_IKE(dev)) {
   1911		static char ike_rev[] = "Ike rev ?";
   1912		ike_rev[8] = '0' + (char) (func_class & 0xff);
   1913		version = ike_rev;
   1914	} else if (IS_PLUTO(dev)) {
   1915		static char pluto_rev[]="Pluto ?.?";
   1916		pluto_rev[6] = '0' + (char) ((func_class & 0xf0) >> 4); 
   1917		pluto_rev[8] = '0' + (char) (func_class & 0x0f); 
   1918		version = pluto_rev;
   1919	} else {
   1920		static char reo_rev[] = "REO rev ?";
   1921		reo_rev[8] = '0' + (char) (func_class & 0xff);
   1922		version = reo_rev;
   1923	}
   1924
   1925	if (!global_ioc_cnt) {
   1926		global_ioc_cnt = count_parisc_driver(&sba_driver);
   1927
   1928		/* Astro and Pluto have one IOC per SBA */
   1929		if ((!IS_ASTRO(dev)) || (!IS_PLUTO(dev)))
   1930			global_ioc_cnt *= 2;
   1931	}
   1932
   1933	printk(KERN_INFO "%s found %s at 0x%llx\n",
   1934		MODULE_NAME, version, (unsigned long long)dev->hpa.start);
   1935
   1936	sba_dev = kzalloc(sizeof(struct sba_device), GFP_KERNEL);
   1937	if (!sba_dev) {
   1938		printk(KERN_ERR MODULE_NAME " - couldn't alloc sba_device\n");
   1939		return -ENOMEM;
   1940	}
   1941
   1942	parisc_set_drvdata(dev, sba_dev);
   1943
   1944	for(i=0; i<MAX_IOC; i++)
   1945		spin_lock_init(&(sba_dev->ioc[i].res_lock));
   1946
   1947	sba_dev->dev = dev;
   1948	sba_dev->hw_rev = func_class;
   1949	sba_dev->name = dev->name;
   1950	sba_dev->sba_hpa = sba_addr;
   1951
   1952	sba_get_pat_resources(sba_dev);
   1953	sba_hw_init(sba_dev);
   1954	sba_common_init(sba_dev);
   1955
   1956	hppa_dma_ops = &sba_ops;
   1957
   1958#ifdef CONFIG_PROC_FS
   1959	switch (dev->id.hversion) {
   1960	case PLUTO_MCKINLEY_PORT:
   1961		root = proc_mckinley_root;
   1962		break;
   1963	case ASTRO_RUNWAY_PORT:
   1964	case IKE_MERCED_PORT:
   1965	default:
   1966		root = proc_runway_root;
   1967		break;
   1968	}
   1969
   1970	proc_create_single("sba_iommu", 0, root, sba_proc_info);
   1971	proc_create_single("sba_iommu-bitmap", 0, root, sba_proc_bitmap_info);
   1972#endif
   1973	return 0;
   1974}
   1975
   1976/*
   1977** One time initialization to let the world know the SBA was found.
   1978** This is the only routine which is NOT static.
   1979** Must be called exactly once before pci_init().
   1980*/
   1981void __init sba_init(void)
   1982{
   1983	register_parisc_driver(&sba_driver);
   1984}
   1985
   1986
   1987/**
   1988 * sba_get_iommu - Assign the iommu pointer for the pci bus controller.
   1989 * @dev: The parisc device.
   1990 *
   1991 * Returns the appropriate IOMMU data for the given parisc PCI controller.
   1992 * This is cached and used later for PCI DMA Mapping.
   1993 */
   1994void * sba_get_iommu(struct parisc_device *pci_hba)
   1995{
   1996	struct parisc_device *sba_dev = parisc_parent(pci_hba);
   1997	struct sba_device *sba = dev_get_drvdata(&sba_dev->dev);
   1998	char t = sba_dev->id.hw_type;
   1999	int iocnum = (pci_hba->hw_path >> 3);	/* rope # */
   2000
   2001	WARN_ON((t != HPHW_IOA) && (t != HPHW_BCPORT));
   2002
   2003	return &(sba->ioc[iocnum]);
   2004}
   2005
   2006
   2007/**
   2008 * sba_directed_lmmio - return first directed LMMIO range routed to rope
   2009 * @pa_dev: The parisc device.
   2010 * @r: resource PCI host controller wants start/end fields assigned.
   2011 *
   2012 * For the given parisc PCI controller, determine if any direct ranges
   2013 * are routed down the corresponding rope.
   2014 */
   2015void sba_directed_lmmio(struct parisc_device *pci_hba, struct resource *r)
   2016{
   2017	struct parisc_device *sba_dev = parisc_parent(pci_hba);
   2018	struct sba_device *sba = dev_get_drvdata(&sba_dev->dev);
   2019	char t = sba_dev->id.hw_type;
   2020	int i;
   2021	int rope = (pci_hba->hw_path & (ROPES_PER_IOC-1));  /* rope # */
   2022
   2023	BUG_ON((t!=HPHW_IOA) && (t!=HPHW_BCPORT));
   2024
   2025	r->start = r->end = 0;
   2026
   2027	/* Astro has 4 directed ranges. Not sure about Ike/Pluto/et al */
   2028	for (i=0; i<4; i++) {
   2029		int base, size;
   2030		void __iomem *reg = sba->sba_hpa + i*0x18;
   2031
   2032		base = READ_REG32(reg + LMMIO_DIRECT0_BASE);
   2033		if ((base & 1) == 0)
   2034			continue;	/* not enabled */
   2035
   2036		size = READ_REG32(reg + LMMIO_DIRECT0_ROUTE);
   2037
   2038		if ((size & (ROPES_PER_IOC-1)) != rope)
   2039			continue;	/* directed down different rope */
   2040		
   2041		r->start = (base & ~1UL) | PCI_F_EXTEND;
   2042		size = ~ READ_REG32(reg + LMMIO_DIRECT0_MASK);
   2043		r->end = r->start + size;
   2044		r->flags = IORESOURCE_MEM;
   2045	}
   2046}
   2047
   2048
   2049/**
   2050 * sba_distributed_lmmio - return portion of distributed LMMIO range
   2051 * @pa_dev: The parisc device.
   2052 * @r: resource PCI host controller wants start/end fields assigned.
   2053 *
   2054 * For the given parisc PCI controller, return portion of distributed LMMIO
   2055 * range. The distributed LMMIO is always present and it's just a question
   2056 * of the base address and size of the range.
   2057 */
   2058void sba_distributed_lmmio(struct parisc_device *pci_hba, struct resource *r )
   2059{
   2060	struct parisc_device *sba_dev = parisc_parent(pci_hba);
   2061	struct sba_device *sba = dev_get_drvdata(&sba_dev->dev);
   2062	char t = sba_dev->id.hw_type;
   2063	int base, size;
   2064	int rope = (pci_hba->hw_path & (ROPES_PER_IOC-1));  /* rope # */
   2065
   2066	BUG_ON((t!=HPHW_IOA) && (t!=HPHW_BCPORT));
   2067
   2068	r->start = r->end = 0;
   2069
   2070	base = READ_REG32(sba->sba_hpa + LMMIO_DIST_BASE);
   2071	if ((base & 1) == 0) {
   2072		BUG();	/* Gah! Distr Range wasn't enabled! */
   2073		return;
   2074	}
   2075
   2076	r->start = (base & ~1UL) | PCI_F_EXTEND;
   2077
   2078	size = (~READ_REG32(sba->sba_hpa + LMMIO_DIST_MASK)) / ROPES_PER_IOC;
   2079	r->start += rope * (size + 1);	/* adjust base for this rope */
   2080	r->end = r->start + size;
   2081	r->flags = IORESOURCE_MEM;
   2082}