cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

i915_gem_migrate.c (10005B)


      1// SPDX-License-Identifier: MIT
      2/*
      3 * Copyright © 2020-2021 Intel Corporation
      4 */
      5
      6#include "gt/intel_migrate.h"
      7#include "gt/intel_gpu_commands.h"
      8#include "gem/i915_gem_ttm_move.h"
      9
     10#include "i915_deps.h"
     11
     12#include "selftests/igt_spinner.h"
     13
     14static int igt_fill_check_buffer(struct drm_i915_gem_object *obj,
     15				 bool fill)
     16{
     17	struct drm_i915_private *i915 = to_i915(obj->base.dev);
     18	unsigned int i, count = obj->base.size / sizeof(u32);
     19	enum i915_map_type map_type =
     20		i915_coherent_map_type(i915, obj, false);
     21	u32 *cur;
     22	int err = 0;
     23
     24	assert_object_held(obj);
     25	cur = i915_gem_object_pin_map(obj, map_type);
     26	if (IS_ERR(cur))
     27		return PTR_ERR(cur);
     28
     29	if (fill)
     30		for (i = 0; i < count; ++i)
     31			*cur++ = i;
     32	else
     33		for (i = 0; i < count; ++i)
     34			if (*cur++ != i) {
     35				pr_err("Object content mismatch at location %d of %d\n", i, count);
     36				err = -EINVAL;
     37				break;
     38			}
     39
     40	i915_gem_object_unpin_map(obj);
     41
     42	return err;
     43}
     44
     45static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src,
     46			      enum intel_region_id dst)
     47{
     48	struct drm_i915_private *i915 = gt->i915;
     49	struct intel_memory_region *src_mr = i915->mm.regions[src];
     50	struct intel_memory_region *dst_mr = i915->mm.regions[dst];
     51	struct drm_i915_gem_object *obj;
     52	struct i915_gem_ww_ctx ww;
     53	int err = 0;
     54
     55	GEM_BUG_ON(!src_mr);
     56	GEM_BUG_ON(!dst_mr);
     57
     58	/* Switch object backing-store on create */
     59	obj = i915_gem_object_create_region(src_mr, dst_mr->min_page_size, 0, 0);
     60	if (IS_ERR(obj))
     61		return PTR_ERR(obj);
     62
     63	for_i915_gem_ww(&ww, err, true) {
     64		err = i915_gem_object_lock(obj, &ww);
     65		if (err)
     66			continue;
     67
     68		err = igt_fill_check_buffer(obj, true);
     69		if (err)
     70			continue;
     71
     72		err = i915_gem_object_migrate(obj, &ww, dst);
     73		if (err)
     74			continue;
     75
     76		err = i915_gem_object_pin_pages(obj);
     77		if (err)
     78			continue;
     79
     80		if (i915_gem_object_can_migrate(obj, src))
     81			err = -EINVAL;
     82
     83		i915_gem_object_unpin_pages(obj);
     84		err = i915_gem_object_wait_migration(obj, true);
     85		if (err)
     86			continue;
     87
     88		err = igt_fill_check_buffer(obj, false);
     89	}
     90	i915_gem_object_put(obj);
     91
     92	return err;
     93}
     94
     95static int igt_smem_create_migrate(void *arg)
     96{
     97	return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_SMEM);
     98}
     99
    100static int igt_lmem_create_migrate(void *arg)
    101{
    102	return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM_0);
    103}
    104
    105static int igt_same_create_migrate(void *arg)
    106{
    107	return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_LMEM_0);
    108}
    109
    110static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww,
    111				  struct drm_i915_gem_object *obj,
    112				  struct i915_vma *vma)
    113{
    114	int err;
    115
    116	err = i915_gem_object_lock(obj, ww);
    117	if (err)
    118		return err;
    119
    120	if (vma) {
    121		err = i915_vma_pin_ww(vma, ww, obj->base.size, 0,
    122				      0UL | PIN_OFFSET_FIXED |
    123				      PIN_USER);
    124		if (err) {
    125			if (err != -EINTR && err != ERESTARTSYS &&
    126			    err != -EDEADLK)
    127				pr_err("Failed to pin vma.\n");
    128			return err;
    129		}
    130
    131		i915_vma_unpin(vma);
    132	}
    133
    134	/*
    135	 * Migration will implicitly unbind (asynchronously) any bound
    136	 * vmas.
    137	 */
    138	if (i915_gem_object_is_lmem(obj)) {
    139		err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM);
    140		if (err) {
    141			pr_err("Object failed migration to smem\n");
    142			if (err)
    143				return err;
    144		}
    145
    146		if (i915_gem_object_is_lmem(obj)) {
    147			pr_err("object still backed by lmem\n");
    148			err = -EINVAL;
    149		}
    150
    151		if (!i915_gem_object_has_struct_page(obj)) {
    152			pr_err("object not backed by struct page\n");
    153			err = -EINVAL;
    154		}
    155
    156	} else {
    157		err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM_0);
    158		if (err) {
    159			pr_err("Object failed migration to lmem\n");
    160			if (err)
    161				return err;
    162		}
    163
    164		if (i915_gem_object_has_struct_page(obj)) {
    165			pr_err("object still backed by struct page\n");
    166			err = -EINVAL;
    167		}
    168
    169		if (!i915_gem_object_is_lmem(obj)) {
    170			pr_err("object not backed by lmem\n");
    171			err = -EINVAL;
    172		}
    173	}
    174
    175	return err;
    176}
    177
    178static int __igt_lmem_pages_migrate(struct intel_gt *gt,
    179				    struct i915_address_space *vm,
    180				    struct i915_deps *deps,
    181				    struct igt_spinner *spin,
    182				    struct dma_fence *spin_fence)
    183{
    184	struct drm_i915_private *i915 = gt->i915;
    185	struct drm_i915_gem_object *obj;
    186	struct i915_vma *vma = NULL;
    187	struct i915_gem_ww_ctx ww;
    188	struct i915_request *rq;
    189	int err;
    190	int i;
    191
    192	/* From LMEM to shmem and back again */
    193
    194	obj = i915_gem_object_create_lmem(i915, SZ_2M, 0);
    195	if (IS_ERR(obj))
    196		return PTR_ERR(obj);
    197
    198	if (vm) {
    199		vma = i915_vma_instance(obj, vm, NULL);
    200		if (IS_ERR(vma)) {
    201			err = PTR_ERR(vma);
    202			goto out_put;
    203		}
    204	}
    205
    206	/* Initial GPU fill, sync, CPU initialization. */
    207	for_i915_gem_ww(&ww, err, true) {
    208		err = i915_gem_object_lock(obj, &ww);
    209		if (err)
    210			continue;
    211
    212		err = ____i915_gem_object_get_pages(obj);
    213		if (err)
    214			continue;
    215
    216		err = intel_migrate_clear(&gt->migrate, &ww, deps,
    217					  obj->mm.pages->sgl, obj->cache_level,
    218					  i915_gem_object_is_lmem(obj),
    219					  0xdeadbeaf, &rq);
    220		if (rq) {
    221			err = dma_resv_reserve_fences(obj->base.resv, 1);
    222			if (!err)
    223				dma_resv_add_fence(obj->base.resv, &rq->fence,
    224						   DMA_RESV_USAGE_KERNEL);
    225			i915_request_put(rq);
    226		}
    227		if (err)
    228			continue;
    229
    230		if (!vma) {
    231			err = igt_fill_check_buffer(obj, true);
    232			if (err)
    233				continue;
    234		}
    235	}
    236	if (err)
    237		goto out_put;
    238
    239	/*
    240	 * Migrate to and from smem without explicitly syncing.
    241	 * Finalize with data in smem for fast readout.
    242	 */
    243	for (i = 1; i <= 5; ++i) {
    244		for_i915_gem_ww(&ww, err, true)
    245			err = lmem_pages_migrate_one(&ww, obj, vma);
    246		if (err)
    247			goto out_put;
    248	}
    249
    250	err = i915_gem_object_lock_interruptible(obj, NULL);
    251	if (err)
    252		goto out_put;
    253
    254	if (spin) {
    255		if (dma_fence_is_signaled(spin_fence)) {
    256			pr_err("Spinner was terminated by hangcheck.\n");
    257			err = -EBUSY;
    258			goto out_unlock;
    259		}
    260		igt_spinner_end(spin);
    261	}
    262
    263	/* Finally sync migration and check content. */
    264	err = i915_gem_object_wait_migration(obj, true);
    265	if (err)
    266		goto out_unlock;
    267
    268	if (vma) {
    269		err = i915_vma_wait_for_bind(vma);
    270		if (err)
    271			goto out_unlock;
    272	} else {
    273		err = igt_fill_check_buffer(obj, false);
    274	}
    275
    276out_unlock:
    277	i915_gem_object_unlock(obj);
    278out_put:
    279	i915_gem_object_put(obj);
    280
    281	return err;
    282}
    283
    284static int igt_lmem_pages_failsafe_migrate(void *arg)
    285{
    286	int fail_gpu, fail_alloc, ret;
    287	struct intel_gt *gt = arg;
    288
    289	for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
    290		for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) {
    291			pr_info("Simulated failure modes: gpu: %d, alloc: %d\n",
    292				fail_gpu, fail_alloc);
    293			i915_ttm_migrate_set_failure_modes(fail_gpu,
    294							   fail_alloc);
    295			ret = __igt_lmem_pages_migrate(gt, NULL, NULL, NULL, NULL);
    296			if (ret)
    297				goto out_err;
    298		}
    299	}
    300
    301out_err:
    302	i915_ttm_migrate_set_failure_modes(false, false);
    303	return ret;
    304}
    305
    306/*
    307 * This subtest tests that unbinding at migration is indeed performed
    308 * async. We launch a spinner and a number of migrations depending on
    309 * that spinner to have terminated. Before each migration we bind a
    310 * vma, which should then be async unbound by the migration operation.
    311 * If we are able to schedule migrations without blocking while the
    312 * spinner is still running, those unbinds are indeed async and non-
    313 * blocking.
    314 *
    315 * Note that each async bind operation is awaiting the previous migration
    316 * due to the moving fence resulting from the migration.
    317 */
    318static int igt_async_migrate(struct intel_gt *gt)
    319{
    320	struct intel_engine_cs *engine;
    321	enum intel_engine_id id;
    322	struct i915_ppgtt *ppgtt;
    323	struct igt_spinner spin;
    324	int err;
    325
    326	ppgtt = i915_ppgtt_create(gt, 0);
    327	if (IS_ERR(ppgtt))
    328		return PTR_ERR(ppgtt);
    329
    330	if (igt_spinner_init(&spin, gt)) {
    331		err = -ENOMEM;
    332		goto out_spin;
    333	}
    334
    335	for_each_engine(engine, gt, id) {
    336		struct ttm_operation_ctx ctx = {
    337			.interruptible = true
    338		};
    339		struct dma_fence *spin_fence;
    340		struct intel_context *ce;
    341		struct i915_request *rq;
    342		struct i915_deps deps;
    343
    344		ce = intel_context_create(engine);
    345		if (IS_ERR(ce)) {
    346			err = PTR_ERR(ce);
    347			goto out_ce;
    348		}
    349
    350		/*
    351		 * Use MI_NOOP, making the spinner non-preemptible. If there
    352		 * is a code path where we fail async operation due to the
    353		 * running spinner, we will block and fail to end the
    354		 * spinner resulting in a deadlock. But with a non-
    355		 * preemptible spinner, hangcheck will terminate the spinner
    356		 * for us, and we will later detect that and fail the test.
    357		 */
    358		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
    359		intel_context_put(ce);
    360		if (IS_ERR(rq)) {
    361			err = PTR_ERR(rq);
    362			goto out_ce;
    363		}
    364
    365		i915_deps_init(&deps, GFP_KERNEL);
    366		err = i915_deps_add_dependency(&deps, &rq->fence, &ctx);
    367		spin_fence = dma_fence_get(&rq->fence);
    368		i915_request_add(rq);
    369		if (err)
    370			goto out_ce;
    371
    372		err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin,
    373					       spin_fence);
    374		i915_deps_fini(&deps);
    375		dma_fence_put(spin_fence);
    376		if (err)
    377			goto out_ce;
    378	}
    379
    380out_ce:
    381	igt_spinner_fini(&spin);
    382out_spin:
    383	i915_vm_put(&ppgtt->vm);
    384
    385	return err;
    386}
    387
    388/*
    389 * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while
    390 * arming the migration error check and block async migration. This
    391 * will cause us to deadlock and hangcheck will terminate the spinner
    392 * causing the test to fail.
    393 */
    394#define ASYNC_FAIL_ALLOC 1
    395static int igt_lmem_async_migrate(void *arg)
    396{
    397	int fail_gpu, fail_alloc, ret;
    398	struct intel_gt *gt = arg;
    399
    400	for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
    401		for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) {
    402			pr_info("Simulated failure modes: gpu: %d, alloc: %d\n",
    403				fail_gpu, fail_alloc);
    404			i915_ttm_migrate_set_failure_modes(fail_gpu,
    405							   fail_alloc);
    406			ret = igt_async_migrate(gt);
    407			if (ret)
    408				goto out_err;
    409		}
    410	}
    411
    412out_err:
    413	i915_ttm_migrate_set_failure_modes(false, false);
    414	return ret;
    415}
    416
    417int i915_gem_migrate_live_selftests(struct drm_i915_private *i915)
    418{
    419	static const struct i915_subtest tests[] = {
    420		SUBTEST(igt_smem_create_migrate),
    421		SUBTEST(igt_lmem_create_migrate),
    422		SUBTEST(igt_same_create_migrate),
    423		SUBTEST(igt_lmem_pages_failsafe_migrate),
    424		SUBTEST(igt_lmem_async_migrate),
    425	};
    426
    427	if (!HAS_LMEM(i915))
    428		return 0;
    429
    430	return intel_gt_live_subtests(tests, to_gt(i915));
    431}