cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

nv50.c (11230B)


      1/*
      2 * Copyright 2012 Red Hat Inc.
      3 *
      4 * Permission is hereby granted, free of charge, to any person obtaining a
      5 * copy of this software and associated documentation files (the "Software"),
      6 * to deal in the Software without restriction, including without limitation
      7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      8 * and/or sell copies of the Software, and to permit persons to whom the
      9 * Software is furnished to do so, subject to the following conditions:
     10 *
     11 * The above copyright notice and this permission notice shall be included in
     12 * all copies or substantial portions of the Software.
     13 *
     14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     20 * OTHER DEALINGS IN THE SOFTWARE.
     21 *
     22 * Authors: Ben Skeggs
     23 */
     24#define nv50_instmem(p) container_of((p), struct nv50_instmem, base)
     25#include "priv.h"
     26
     27#include <core/memory.h>
     28#include <subdev/bar.h>
     29#include <subdev/fb.h>
     30#include <subdev/mmu.h>
     31
     32struct nv50_instmem {
     33	struct nvkm_instmem base;
     34	u64 addr;
     35
     36	/* Mappings that can be evicted when BAR2 space has been exhausted. */
     37	struct list_head lru;
     38};
     39
     40/******************************************************************************
     41 * instmem object implementation
     42 *****************************************************************************/
     43#define nv50_instobj(p) container_of((p), struct nv50_instobj, base.memory)
     44
     45struct nv50_instobj {
     46	struct nvkm_instobj base;
     47	struct nv50_instmem *imem;
     48	struct nvkm_memory *ram;
     49	struct nvkm_vma *bar;
     50	refcount_t maps;
     51	void *map;
     52	struct list_head lru;
     53};
     54
     55static void
     56nv50_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
     57{
     58	struct nv50_instobj *iobj = nv50_instobj(memory);
     59	struct nv50_instmem *imem = iobj->imem;
     60	struct nvkm_device *device = imem->base.subdev.device;
     61	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
     62	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
     63	unsigned long flags;
     64
     65	spin_lock_irqsave(&imem->base.lock, flags);
     66	if (unlikely(imem->addr != base)) {
     67		nvkm_wr32(device, 0x001700, base >> 16);
     68		imem->addr = base;
     69	}
     70	nvkm_wr32(device, 0x700000 + addr, data);
     71	spin_unlock_irqrestore(&imem->base.lock, flags);
     72}
     73
     74static u32
     75nv50_instobj_rd32_slow(struct nvkm_memory *memory, u64 offset)
     76{
     77	struct nv50_instobj *iobj = nv50_instobj(memory);
     78	struct nv50_instmem *imem = iobj->imem;
     79	struct nvkm_device *device = imem->base.subdev.device;
     80	u64 base = (nvkm_memory_addr(iobj->ram) + offset) & 0xffffff00000ULL;
     81	u64 addr = (nvkm_memory_addr(iobj->ram) + offset) & 0x000000fffffULL;
     82	u32 data;
     83	unsigned long flags;
     84
     85	spin_lock_irqsave(&imem->base.lock, flags);
     86	if (unlikely(imem->addr != base)) {
     87		nvkm_wr32(device, 0x001700, base >> 16);
     88		imem->addr = base;
     89	}
     90	data = nvkm_rd32(device, 0x700000 + addr);
     91	spin_unlock_irqrestore(&imem->base.lock, flags);
     92	return data;
     93}
     94
     95static const struct nvkm_memory_ptrs
     96nv50_instobj_slow = {
     97	.rd32 = nv50_instobj_rd32_slow,
     98	.wr32 = nv50_instobj_wr32_slow,
     99};
    100
    101static void
    102nv50_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data)
    103{
    104	iowrite32_native(data, nv50_instobj(memory)->map + offset);
    105}
    106
    107static u32
    108nv50_instobj_rd32(struct nvkm_memory *memory, u64 offset)
    109{
    110	return ioread32_native(nv50_instobj(memory)->map + offset);
    111}
    112
    113static const struct nvkm_memory_ptrs
    114nv50_instobj_fast = {
    115	.rd32 = nv50_instobj_rd32,
    116	.wr32 = nv50_instobj_wr32,
    117};
    118
    119static void
    120nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
    121{
    122	struct nv50_instmem *imem = iobj->imem;
    123	struct nv50_instobj *eobj;
    124	struct nvkm_memory *memory = &iobj->base.memory;
    125	struct nvkm_subdev *subdev = &imem->base.subdev;
    126	struct nvkm_device *device = subdev->device;
    127	struct nvkm_vma *bar = NULL, *ebar;
    128	u64 size = nvkm_memory_size(memory);
    129	void *emap;
    130	int ret;
    131
    132	/* Attempt to allocate BAR2 address-space and map the object
    133	 * into it.  The lock has to be dropped while doing this due
    134	 * to the possibility of recursion for page table allocation.
    135	 */
    136	mutex_unlock(&imem->base.mutex);
    137	while ((ret = nvkm_vmm_get(vmm, 12, size, &bar))) {
    138		/* Evict unused mappings, and keep retrying until we either
    139		 * succeed,or there's no more objects left on the LRU.
    140		 */
    141		mutex_lock(&imem->base.mutex);
    142		eobj = list_first_entry_or_null(&imem->lru, typeof(*eobj), lru);
    143		if (eobj) {
    144			nvkm_debug(subdev, "evict %016llx %016llx @ %016llx\n",
    145				   nvkm_memory_addr(&eobj->base.memory),
    146				   nvkm_memory_size(&eobj->base.memory),
    147				   eobj->bar->addr);
    148			list_del_init(&eobj->lru);
    149			ebar = eobj->bar;
    150			eobj->bar = NULL;
    151			emap = eobj->map;
    152			eobj->map = NULL;
    153		}
    154		mutex_unlock(&imem->base.mutex);
    155		if (!eobj)
    156			break;
    157		iounmap(emap);
    158		nvkm_vmm_put(vmm, &ebar);
    159	}
    160
    161	if (ret == 0)
    162		ret = nvkm_memory_map(memory, 0, vmm, bar, NULL, 0);
    163	mutex_lock(&imem->base.mutex);
    164	if (ret || iobj->bar) {
    165		/* We either failed, or another thread beat us. */
    166		mutex_unlock(&imem->base.mutex);
    167		nvkm_vmm_put(vmm, &bar);
    168		mutex_lock(&imem->base.mutex);
    169		return;
    170	}
    171
    172	/* Make the mapping visible to the host. */
    173	iobj->bar = bar;
    174	iobj->map = ioremap_wc(device->func->resource_addr(device, 3) +
    175			       (u32)iobj->bar->addr, size);
    176	if (!iobj->map) {
    177		nvkm_warn(subdev, "PRAMIN ioremap failed\n");
    178		nvkm_vmm_put(vmm, &iobj->bar);
    179	}
    180}
    181
    182static int
    183nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
    184		 struct nvkm_vma *vma, void *argv, u32 argc)
    185{
    186	memory = nv50_instobj(memory)->ram;
    187	return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
    188}
    189
    190static void
    191nv50_instobj_release(struct nvkm_memory *memory)
    192{
    193	struct nv50_instobj *iobj = nv50_instobj(memory);
    194	struct nv50_instmem *imem = iobj->imem;
    195	struct nvkm_subdev *subdev = &imem->base.subdev;
    196
    197	wmb();
    198	nvkm_bar_flush(subdev->device->bar);
    199
    200	if (refcount_dec_and_mutex_lock(&iobj->maps, &imem->base.mutex)) {
    201		/* Add the now-unused mapping to the LRU instead of directly
    202		 * unmapping it here, in case we need to map it again later.
    203		 */
    204		if (likely(iobj->lru.next) && iobj->map) {
    205			BUG_ON(!list_empty(&iobj->lru));
    206			list_add_tail(&iobj->lru, &imem->lru);
    207		}
    208
    209		/* Switch back to NULL accessors when last map is gone. */
    210		iobj->base.memory.ptrs = NULL;
    211		mutex_unlock(&imem->base.mutex);
    212	}
    213}
    214
    215static void __iomem *
    216nv50_instobj_acquire(struct nvkm_memory *memory)
    217{
    218	struct nv50_instobj *iobj = nv50_instobj(memory);
    219	struct nvkm_instmem *imem = &iobj->imem->base;
    220	struct nvkm_vmm *vmm;
    221	void __iomem *map = NULL;
    222
    223	/* Already mapped? */
    224	if (refcount_inc_not_zero(&iobj->maps))
    225		return iobj->map;
    226
    227	/* Take the lock, and re-check that another thread hasn't
    228	 * already mapped the object in the meantime.
    229	 */
    230	mutex_lock(&imem->mutex);
    231	if (refcount_inc_not_zero(&iobj->maps)) {
    232		mutex_unlock(&imem->mutex);
    233		return iobj->map;
    234	}
    235
    236	/* Attempt to get a direct CPU mapping of the object. */
    237	if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
    238		if (!iobj->map)
    239			nv50_instobj_kmap(iobj, vmm);
    240		map = iobj->map;
    241	}
    242
    243	if (!refcount_inc_not_zero(&iobj->maps)) {
    244		/* Exclude object from eviction while it's being accessed. */
    245		if (likely(iobj->lru.next))
    246			list_del_init(&iobj->lru);
    247
    248		if (map)
    249			iobj->base.memory.ptrs = &nv50_instobj_fast;
    250		else
    251			iobj->base.memory.ptrs = &nv50_instobj_slow;
    252		refcount_set(&iobj->maps, 1);
    253	}
    254
    255	mutex_unlock(&imem->mutex);
    256	return map;
    257}
    258
    259static void
    260nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
    261{
    262	struct nv50_instobj *iobj = nv50_instobj(memory);
    263	struct nvkm_instmem *imem = &iobj->imem->base;
    264
    265	/* Exclude bootstrapped objects (ie. the page tables for the
    266	 * instmem BAR itself) from eviction.
    267	 */
    268	mutex_lock(&imem->mutex);
    269	if (likely(iobj->lru.next)) {
    270		list_del_init(&iobj->lru);
    271		iobj->lru.next = NULL;
    272	}
    273
    274	nv50_instobj_kmap(iobj, vmm);
    275	nvkm_instmem_boot(imem);
    276	mutex_unlock(&imem->mutex);
    277}
    278
    279static u64
    280nv50_instobj_size(struct nvkm_memory *memory)
    281{
    282	return nvkm_memory_size(nv50_instobj(memory)->ram);
    283}
    284
    285static u64
    286nv50_instobj_addr(struct nvkm_memory *memory)
    287{
    288	return nvkm_memory_addr(nv50_instobj(memory)->ram);
    289}
    290
    291static u64
    292nv50_instobj_bar2(struct nvkm_memory *memory)
    293{
    294	struct nv50_instobj *iobj = nv50_instobj(memory);
    295	u64 addr = ~0ULL;
    296	if (nv50_instobj_acquire(&iobj->base.memory)) {
    297		iobj->lru.next = NULL; /* Exclude from eviction. */
    298		addr = iobj->bar->addr;
    299	}
    300	nv50_instobj_release(&iobj->base.memory);
    301	return addr;
    302}
    303
    304static enum nvkm_memory_target
    305nv50_instobj_target(struct nvkm_memory *memory)
    306{
    307	return nvkm_memory_target(nv50_instobj(memory)->ram);
    308}
    309
    310static void *
    311nv50_instobj_dtor(struct nvkm_memory *memory)
    312{
    313	struct nv50_instobj *iobj = nv50_instobj(memory);
    314	struct nvkm_instmem *imem = &iobj->imem->base;
    315	struct nvkm_vma *bar;
    316	void *map;
    317
    318	mutex_lock(&imem->mutex);
    319	if (likely(iobj->lru.next))
    320		list_del(&iobj->lru);
    321	map = iobj->map;
    322	bar = iobj->bar;
    323	mutex_unlock(&imem->mutex);
    324
    325	if (map) {
    326		struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device);
    327		iounmap(map);
    328		if (likely(vmm)) /* Can be NULL during BAR destructor. */
    329			nvkm_vmm_put(vmm, &bar);
    330	}
    331
    332	nvkm_memory_unref(&iobj->ram);
    333	nvkm_instobj_dtor(imem, &iobj->base);
    334	return iobj;
    335}
    336
    337static const struct nvkm_memory_func
    338nv50_instobj_func = {
    339	.dtor = nv50_instobj_dtor,
    340	.target = nv50_instobj_target,
    341	.bar2 = nv50_instobj_bar2,
    342	.addr = nv50_instobj_addr,
    343	.size = nv50_instobj_size,
    344	.boot = nv50_instobj_boot,
    345	.acquire = nv50_instobj_acquire,
    346	.release = nv50_instobj_release,
    347	.map = nv50_instobj_map,
    348};
    349
    350static int
    351nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero,
    352		 struct nvkm_memory **pmemory)
    353{
    354	struct nv50_instmem *imem = nv50_instmem(base);
    355	struct nv50_instobj *iobj;
    356	struct nvkm_device *device = imem->base.subdev.device;
    357	u8 page = max(order_base_2(align), 12);
    358
    359	if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL)))
    360		return -ENOMEM;
    361	*pmemory = &iobj->base.memory;
    362
    363	nvkm_instobj_ctor(&nv50_instobj_func, &imem->base, &iobj->base);
    364	iobj->imem = imem;
    365	refcount_set(&iobj->maps, 0);
    366	INIT_LIST_HEAD(&iobj->lru);
    367
    368	return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram);
    369}
    370
    371/******************************************************************************
    372 * instmem subdev implementation
    373 *****************************************************************************/
    374
    375static void
    376nv50_instmem_fini(struct nvkm_instmem *base)
    377{
    378	nv50_instmem(base)->addr = ~0ULL;
    379}
    380
    381static const struct nvkm_instmem_func
    382nv50_instmem = {
    383	.fini = nv50_instmem_fini,
    384	.memory_new = nv50_instobj_new,
    385	.zero = false,
    386};
    387
    388int
    389nv50_instmem_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
    390		 struct nvkm_instmem **pimem)
    391{
    392	struct nv50_instmem *imem;
    393
    394	if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))
    395		return -ENOMEM;
    396	nvkm_instmem_ctor(&nv50_instmem, device, type, inst, &imem->base);
    397	INIT_LIST_HEAD(&imem->lru);
    398	*pimem = &imem->base;
    399	return 0;
    400}