cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

i915_gem_context.c (64094B)


      1/*
      2 * SPDX-License-Identifier: MIT
      3 *
      4 * Copyright © 2011-2012 Intel Corporation
      5 */
      6
      7/*
      8 * This file implements HW context support. On gen5+ a HW context consists of an
      9 * opaque GPU object which is referenced at times of context saves and restores.
     10 * With RC6 enabled, the context is also referenced as the GPU enters and exists
     11 * from RC6 (GPU has it's own internal power context, except on gen5). Though
     12 * something like a context does exist for the media ring, the code only
     13 * supports contexts for the render ring.
     14 *
     15 * In software, there is a distinction between contexts created by the user,
     16 * and the default HW context. The default HW context is used by GPU clients
     17 * that do not request setup of their own hardware context. The default
     18 * context's state is never restored to help prevent programming errors. This
     19 * would happen if a client ran and piggy-backed off another clients GPU state.
     20 * The default context only exists to give the GPU some offset to load as the
     21 * current to invoke a save of the context we actually care about. In fact, the
     22 * code could likely be constructed, albeit in a more complicated fashion, to
     23 * never use the default context, though that limits the driver's ability to
     24 * swap out, and/or destroy other contexts.
     25 *
     26 * All other contexts are created as a request by the GPU client. These contexts
     27 * store GPU state, and thus allow GPU clients to not re-emit state (and
     28 * potentially query certain state) at any time. The kernel driver makes
     29 * certain that the appropriate commands are inserted.
     30 *
     31 * The context life cycle is semi-complicated in that context BOs may live
     32 * longer than the context itself because of the way the hardware, and object
     33 * tracking works. Below is a very crude representation of the state machine
     34 * describing the context life.
     35 *                                         refcount     pincount     active
     36 * S0: initial state                          0            0           0
     37 * S1: context created                        1            0           0
     38 * S2: context is currently running           2            1           X
     39 * S3: GPU referenced, but not current        2            0           1
     40 * S4: context is current, but destroyed      1            1           0
     41 * S5: like S3, but destroyed                 1            0           1
     42 *
     43 * The most common (but not all) transitions:
     44 * S0->S1: client creates a context
     45 * S1->S2: client submits execbuf with context
     46 * S2->S3: other clients submits execbuf with context
     47 * S3->S1: context object was retired
     48 * S3->S2: clients submits another execbuf
     49 * S2->S4: context destroy called with current context
     50 * S3->S5->S0: destroy path
     51 * S4->S5->S0: destroy path on current context
     52 *
     53 * There are two confusing terms used above:
     54 *  The "current context" means the context which is currently running on the
     55 *  GPU. The GPU has loaded its state already and has stored away the gtt
     56 *  offset of the BO. The GPU is not actively referencing the data at this
     57 *  offset, but it will on the next context switch. The only way to avoid this
     58 *  is to do a GPU reset.
     59 *
     60 *  An "active context' is one which was previously the "current context" and is
     61 *  on the active list waiting for the next context switch to occur. Until this
     62 *  happens, the object must remain at the same gtt offset. It is therefore
     63 *  possible to destroy a context, but it is still active.
     64 *
     65 */
     66
     67#include <linux/highmem.h>
     68#include <linux/log2.h>
     69#include <linux/nospec.h>
     70
     71#include <drm/drm_cache.h>
     72#include <drm/drm_syncobj.h>
     73
     74#include "gt/gen6_ppgtt.h"
     75#include "gt/intel_context.h"
     76#include "gt/intel_context_param.h"
     77#include "gt/intel_engine_heartbeat.h"
     78#include "gt/intel_engine_user.h"
     79#include "gt/intel_gpu_commands.h"
     80#include "gt/intel_ring.h"
     81
     82#include "pxp/intel_pxp.h"
     83
     84#include "i915_file_private.h"
     85#include "i915_gem_context.h"
     86#include "i915_trace.h"
     87#include "i915_user_extensions.h"
     88
     89#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
     90
     91static struct kmem_cache *slab_luts;
     92
     93struct i915_lut_handle *i915_lut_handle_alloc(void)
     94{
     95	return kmem_cache_alloc(slab_luts, GFP_KERNEL);
     96}
     97
     98void i915_lut_handle_free(struct i915_lut_handle *lut)
     99{
    100	return kmem_cache_free(slab_luts, lut);
    101}
    102
    103static void lut_close(struct i915_gem_context *ctx)
    104{
    105	struct radix_tree_iter iter;
    106	void __rcu **slot;
    107
    108	mutex_lock(&ctx->lut_mutex);
    109	rcu_read_lock();
    110	radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) {
    111		struct i915_vma *vma = rcu_dereference_raw(*slot);
    112		struct drm_i915_gem_object *obj = vma->obj;
    113		struct i915_lut_handle *lut;
    114
    115		if (!kref_get_unless_zero(&obj->base.refcount))
    116			continue;
    117
    118		spin_lock(&obj->lut_lock);
    119		list_for_each_entry(lut, &obj->lut_list, obj_link) {
    120			if (lut->ctx != ctx)
    121				continue;
    122
    123			if (lut->handle != iter.index)
    124				continue;
    125
    126			list_del(&lut->obj_link);
    127			break;
    128		}
    129		spin_unlock(&obj->lut_lock);
    130
    131		if (&lut->obj_link != &obj->lut_list) {
    132			i915_lut_handle_free(lut);
    133			radix_tree_iter_delete(&ctx->handles_vma, &iter, slot);
    134			i915_vma_close(vma);
    135			i915_gem_object_put(obj);
    136		}
    137
    138		i915_gem_object_put(obj);
    139	}
    140	rcu_read_unlock();
    141	mutex_unlock(&ctx->lut_mutex);
    142}
    143
    144static struct intel_context *
    145lookup_user_engine(struct i915_gem_context *ctx,
    146		   unsigned long flags,
    147		   const struct i915_engine_class_instance *ci)
    148#define LOOKUP_USER_INDEX BIT(0)
    149{
    150	int idx;
    151
    152	if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx))
    153		return ERR_PTR(-EINVAL);
    154
    155	if (!i915_gem_context_user_engines(ctx)) {
    156		struct intel_engine_cs *engine;
    157
    158		engine = intel_engine_lookup_user(ctx->i915,
    159						  ci->engine_class,
    160						  ci->engine_instance);
    161		if (!engine)
    162			return ERR_PTR(-EINVAL);
    163
    164		idx = engine->legacy_idx;
    165	} else {
    166		idx = ci->engine_instance;
    167	}
    168
    169	return i915_gem_context_get_engine(ctx, idx);
    170}
    171
    172static int validate_priority(struct drm_i915_private *i915,
    173			     const struct drm_i915_gem_context_param *args)
    174{
    175	s64 priority = args->value;
    176
    177	if (args->size)
    178		return -EINVAL;
    179
    180	if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
    181		return -ENODEV;
    182
    183	if (priority > I915_CONTEXT_MAX_USER_PRIORITY ||
    184	    priority < I915_CONTEXT_MIN_USER_PRIORITY)
    185		return -EINVAL;
    186
    187	if (priority > I915_CONTEXT_DEFAULT_PRIORITY &&
    188	    !capable(CAP_SYS_NICE))
    189		return -EPERM;
    190
    191	return 0;
    192}
    193
    194static void proto_context_close(struct drm_i915_private *i915,
    195				struct i915_gem_proto_context *pc)
    196{
    197	int i;
    198
    199	if (pc->pxp_wakeref)
    200		intel_runtime_pm_put(&i915->runtime_pm, pc->pxp_wakeref);
    201	if (pc->vm)
    202		i915_vm_put(pc->vm);
    203	if (pc->user_engines) {
    204		for (i = 0; i < pc->num_user_engines; i++)
    205			kfree(pc->user_engines[i].siblings);
    206		kfree(pc->user_engines);
    207	}
    208	kfree(pc);
    209}
    210
    211static int proto_context_set_persistence(struct drm_i915_private *i915,
    212					 struct i915_gem_proto_context *pc,
    213					 bool persist)
    214{
    215	if (persist) {
    216		/*
    217		 * Only contexts that are short-lived [that will expire or be
    218		 * reset] are allowed to survive past termination. We require
    219		 * hangcheck to ensure that the persistent requests are healthy.
    220		 */
    221		if (!i915->params.enable_hangcheck)
    222			return -EINVAL;
    223
    224		pc->user_flags |= BIT(UCONTEXT_PERSISTENCE);
    225	} else {
    226		/* To cancel a context we use "preempt-to-idle" */
    227		if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
    228			return -ENODEV;
    229
    230		/*
    231		 * If the cancel fails, we then need to reset, cleanly!
    232		 *
    233		 * If the per-engine reset fails, all hope is lost! We resort
    234		 * to a full GPU reset in that unlikely case, but realistically
    235		 * if the engine could not reset, the full reset does not fare
    236		 * much better. The damage has been done.
    237		 *
    238		 * However, if we cannot reset an engine by itself, we cannot
    239		 * cleanup a hanging persistent context without causing
    240		 * colateral damage, and we should not pretend we can by
    241		 * exposing the interface.
    242		 */
    243		if (!intel_has_reset_engine(to_gt(i915)))
    244			return -ENODEV;
    245
    246		pc->user_flags &= ~BIT(UCONTEXT_PERSISTENCE);
    247	}
    248
    249	return 0;
    250}
    251
    252static int proto_context_set_protected(struct drm_i915_private *i915,
    253				       struct i915_gem_proto_context *pc,
    254				       bool protected)
    255{
    256	int ret = 0;
    257
    258	if (!protected) {
    259		pc->uses_protected_content = false;
    260	} else if (!intel_pxp_is_enabled(&to_gt(i915)->pxp)) {
    261		ret = -ENODEV;
    262	} else if ((pc->user_flags & BIT(UCONTEXT_RECOVERABLE)) ||
    263		   !(pc->user_flags & BIT(UCONTEXT_BANNABLE))) {
    264		ret = -EPERM;
    265	} else {
    266		pc->uses_protected_content = true;
    267
    268		/*
    269		 * protected context usage requires the PXP session to be up,
    270		 * which in turn requires the device to be active.
    271		 */
    272		pc->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm);
    273
    274		if (!intel_pxp_is_active(&to_gt(i915)->pxp))
    275			ret = intel_pxp_start(&to_gt(i915)->pxp);
    276	}
    277
    278	return ret;
    279}
    280
    281static struct i915_gem_proto_context *
    282proto_context_create(struct drm_i915_private *i915, unsigned int flags)
    283{
    284	struct i915_gem_proto_context *pc, *err;
    285
    286	pc = kzalloc(sizeof(*pc), GFP_KERNEL);
    287	if (!pc)
    288		return ERR_PTR(-ENOMEM);
    289
    290	pc->num_user_engines = -1;
    291	pc->user_engines = NULL;
    292	pc->user_flags = BIT(UCONTEXT_BANNABLE) |
    293			 BIT(UCONTEXT_RECOVERABLE);
    294	if (i915->params.enable_hangcheck)
    295		pc->user_flags |= BIT(UCONTEXT_PERSISTENCE);
    296	pc->sched.priority = I915_PRIORITY_NORMAL;
    297
    298	if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) {
    299		if (!HAS_EXECLISTS(i915)) {
    300			err = ERR_PTR(-EINVAL);
    301			goto proto_close;
    302		}
    303		pc->single_timeline = true;
    304	}
    305
    306	return pc;
    307
    308proto_close:
    309	proto_context_close(i915, pc);
    310	return err;
    311}
    312
    313static int proto_context_register_locked(struct drm_i915_file_private *fpriv,
    314					 struct i915_gem_proto_context *pc,
    315					 u32 *id)
    316{
    317	int ret;
    318	void *old;
    319
    320	lockdep_assert_held(&fpriv->proto_context_lock);
    321
    322	ret = xa_alloc(&fpriv->context_xa, id, NULL, xa_limit_32b, GFP_KERNEL);
    323	if (ret)
    324		return ret;
    325
    326	old = xa_store(&fpriv->proto_context_xa, *id, pc, GFP_KERNEL);
    327	if (xa_is_err(old)) {
    328		xa_erase(&fpriv->context_xa, *id);
    329		return xa_err(old);
    330	}
    331	WARN_ON(old);
    332
    333	return 0;
    334}
    335
    336static int proto_context_register(struct drm_i915_file_private *fpriv,
    337				  struct i915_gem_proto_context *pc,
    338				  u32 *id)
    339{
    340	int ret;
    341
    342	mutex_lock(&fpriv->proto_context_lock);
    343	ret = proto_context_register_locked(fpriv, pc, id);
    344	mutex_unlock(&fpriv->proto_context_lock);
    345
    346	return ret;
    347}
    348
    349static struct i915_address_space *
    350i915_gem_vm_lookup(struct drm_i915_file_private *file_priv, u32 id)
    351{
    352	struct i915_address_space *vm;
    353
    354	xa_lock(&file_priv->vm_xa);
    355	vm = xa_load(&file_priv->vm_xa, id);
    356	if (vm)
    357		kref_get(&vm->ref);
    358	xa_unlock(&file_priv->vm_xa);
    359
    360	return vm;
    361}
    362
    363static int set_proto_ctx_vm(struct drm_i915_file_private *fpriv,
    364			    struct i915_gem_proto_context *pc,
    365			    const struct drm_i915_gem_context_param *args)
    366{
    367	struct drm_i915_private *i915 = fpriv->dev_priv;
    368	struct i915_address_space *vm;
    369
    370	if (args->size)
    371		return -EINVAL;
    372
    373	if (!HAS_FULL_PPGTT(i915))
    374		return -ENODEV;
    375
    376	if (upper_32_bits(args->value))
    377		return -ENOENT;
    378
    379	vm = i915_gem_vm_lookup(fpriv, args->value);
    380	if (!vm)
    381		return -ENOENT;
    382
    383	if (pc->vm)
    384		i915_vm_put(pc->vm);
    385	pc->vm = vm;
    386
    387	return 0;
    388}
    389
    390struct set_proto_ctx_engines {
    391	struct drm_i915_private *i915;
    392	unsigned num_engines;
    393	struct i915_gem_proto_engine *engines;
    394};
    395
    396static int
    397set_proto_ctx_engines_balance(struct i915_user_extension __user *base,
    398			      void *data)
    399{
    400	struct i915_context_engines_load_balance __user *ext =
    401		container_of_user(base, typeof(*ext), base);
    402	const struct set_proto_ctx_engines *set = data;
    403	struct drm_i915_private *i915 = set->i915;
    404	struct intel_engine_cs **siblings;
    405	u16 num_siblings, idx;
    406	unsigned int n;
    407	int err;
    408
    409	if (!HAS_EXECLISTS(i915))
    410		return -ENODEV;
    411
    412	if (get_user(idx, &ext->engine_index))
    413		return -EFAULT;
    414
    415	if (idx >= set->num_engines) {
    416		drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n",
    417			idx, set->num_engines);
    418		return -EINVAL;
    419	}
    420
    421	idx = array_index_nospec(idx, set->num_engines);
    422	if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_INVALID) {
    423		drm_dbg(&i915->drm,
    424			"Invalid placement[%d], already occupied\n", idx);
    425		return -EEXIST;
    426	}
    427
    428	if (get_user(num_siblings, &ext->num_siblings))
    429		return -EFAULT;
    430
    431	err = check_user_mbz(&ext->flags);
    432	if (err)
    433		return err;
    434
    435	err = check_user_mbz(&ext->mbz64);
    436	if (err)
    437		return err;
    438
    439	if (num_siblings == 0)
    440		return 0;
    441
    442	siblings = kmalloc_array(num_siblings, sizeof(*siblings), GFP_KERNEL);
    443	if (!siblings)
    444		return -ENOMEM;
    445
    446	for (n = 0; n < num_siblings; n++) {
    447		struct i915_engine_class_instance ci;
    448
    449		if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) {
    450			err = -EFAULT;
    451			goto err_siblings;
    452		}
    453
    454		siblings[n] = intel_engine_lookup_user(i915,
    455						       ci.engine_class,
    456						       ci.engine_instance);
    457		if (!siblings[n]) {
    458			drm_dbg(&i915->drm,
    459				"Invalid sibling[%d]: { class:%d, inst:%d }\n",
    460				n, ci.engine_class, ci.engine_instance);
    461			err = -EINVAL;
    462			goto err_siblings;
    463		}
    464	}
    465
    466	if (num_siblings == 1) {
    467		set->engines[idx].type = I915_GEM_ENGINE_TYPE_PHYSICAL;
    468		set->engines[idx].engine = siblings[0];
    469		kfree(siblings);
    470	} else {
    471		set->engines[idx].type = I915_GEM_ENGINE_TYPE_BALANCED;
    472		set->engines[idx].num_siblings = num_siblings;
    473		set->engines[idx].siblings = siblings;
    474	}
    475
    476	return 0;
    477
    478err_siblings:
    479	kfree(siblings);
    480
    481	return err;
    482}
    483
    484static int
    485set_proto_ctx_engines_bond(struct i915_user_extension __user *base, void *data)
    486{
    487	struct i915_context_engines_bond __user *ext =
    488		container_of_user(base, typeof(*ext), base);
    489	const struct set_proto_ctx_engines *set = data;
    490	struct drm_i915_private *i915 = set->i915;
    491	struct i915_engine_class_instance ci;
    492	struct intel_engine_cs *master;
    493	u16 idx, num_bonds;
    494	int err, n;
    495
    496	if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915) &&
    497	    !IS_ROCKETLAKE(i915) && !IS_ALDERLAKE_S(i915)) {
    498		drm_dbg(&i915->drm,
    499			"Bonding not supported on this platform\n");
    500		return -ENODEV;
    501	}
    502
    503	if (get_user(idx, &ext->virtual_index))
    504		return -EFAULT;
    505
    506	if (idx >= set->num_engines) {
    507		drm_dbg(&i915->drm,
    508			"Invalid index for virtual engine: %d >= %d\n",
    509			idx, set->num_engines);
    510		return -EINVAL;
    511	}
    512
    513	idx = array_index_nospec(idx, set->num_engines);
    514	if (set->engines[idx].type == I915_GEM_ENGINE_TYPE_INVALID) {
    515		drm_dbg(&i915->drm, "Invalid engine at %d\n", idx);
    516		return -EINVAL;
    517	}
    518
    519	if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_PHYSICAL) {
    520		drm_dbg(&i915->drm,
    521			"Bonding with virtual engines not allowed\n");
    522		return -EINVAL;
    523	}
    524
    525	err = check_user_mbz(&ext->flags);
    526	if (err)
    527		return err;
    528
    529	for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) {
    530		err = check_user_mbz(&ext->mbz64[n]);
    531		if (err)
    532			return err;
    533	}
    534
    535	if (copy_from_user(&ci, &ext->master, sizeof(ci)))
    536		return -EFAULT;
    537
    538	master = intel_engine_lookup_user(i915,
    539					  ci.engine_class,
    540					  ci.engine_instance);
    541	if (!master) {
    542		drm_dbg(&i915->drm,
    543			"Unrecognised master engine: { class:%u, instance:%u }\n",
    544			ci.engine_class, ci.engine_instance);
    545		return -EINVAL;
    546	}
    547
    548	if (intel_engine_uses_guc(master)) {
    549		DRM_DEBUG("bonding extension not supported with GuC submission");
    550		return -ENODEV;
    551	}
    552
    553	if (get_user(num_bonds, &ext->num_bonds))
    554		return -EFAULT;
    555
    556	for (n = 0; n < num_bonds; n++) {
    557		struct intel_engine_cs *bond;
    558
    559		if (copy_from_user(&ci, &ext->engines[n], sizeof(ci)))
    560			return -EFAULT;
    561
    562		bond = intel_engine_lookup_user(i915,
    563						ci.engine_class,
    564						ci.engine_instance);
    565		if (!bond) {
    566			drm_dbg(&i915->drm,
    567				"Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n",
    568				n, ci.engine_class, ci.engine_instance);
    569			return -EINVAL;
    570		}
    571	}
    572
    573	return 0;
    574}
    575
    576static int
    577set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base,
    578				      void *data)
    579{
    580	struct i915_context_engines_parallel_submit __user *ext =
    581		container_of_user(base, typeof(*ext), base);
    582	const struct set_proto_ctx_engines *set = data;
    583	struct drm_i915_private *i915 = set->i915;
    584	struct i915_engine_class_instance prev_engine;
    585	u64 flags;
    586	int err = 0, n, i, j;
    587	u16 slot, width, num_siblings;
    588	struct intel_engine_cs **siblings = NULL;
    589	intel_engine_mask_t prev_mask;
    590
    591	if (get_user(slot, &ext->engine_index))
    592		return -EFAULT;
    593
    594	if (get_user(width, &ext->width))
    595		return -EFAULT;
    596
    597	if (get_user(num_siblings, &ext->num_siblings))
    598		return -EFAULT;
    599
    600	if (!intel_uc_uses_guc_submission(&to_gt(i915)->uc) &&
    601	    num_siblings != 1) {
    602		drm_dbg(&i915->drm, "Only 1 sibling (%d) supported in non-GuC mode\n",
    603			num_siblings);
    604		return -EINVAL;
    605	}
    606
    607	if (slot >= set->num_engines) {
    608		drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n",
    609			slot, set->num_engines);
    610		return -EINVAL;
    611	}
    612
    613	if (set->engines[slot].type != I915_GEM_ENGINE_TYPE_INVALID) {
    614		drm_dbg(&i915->drm,
    615			"Invalid placement[%d], already occupied\n", slot);
    616		return -EINVAL;
    617	}
    618
    619	if (get_user(flags, &ext->flags))
    620		return -EFAULT;
    621
    622	if (flags) {
    623		drm_dbg(&i915->drm, "Unknown flags 0x%02llx", flags);
    624		return -EINVAL;
    625	}
    626
    627	for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) {
    628		err = check_user_mbz(&ext->mbz64[n]);
    629		if (err)
    630			return err;
    631	}
    632
    633	if (width < 2) {
    634		drm_dbg(&i915->drm, "Width (%d) < 2\n", width);
    635		return -EINVAL;
    636	}
    637
    638	if (num_siblings < 1) {
    639		drm_dbg(&i915->drm, "Number siblings (%d) < 1\n",
    640			num_siblings);
    641		return -EINVAL;
    642	}
    643
    644	siblings = kmalloc_array(num_siblings * width,
    645				 sizeof(*siblings),
    646				 GFP_KERNEL);
    647	if (!siblings)
    648		return -ENOMEM;
    649
    650	/* Create contexts / engines */
    651	for (i = 0; i < width; ++i) {
    652		intel_engine_mask_t current_mask = 0;
    653
    654		for (j = 0; j < num_siblings; ++j) {
    655			struct i915_engine_class_instance ci;
    656
    657			n = i * num_siblings + j;
    658			if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) {
    659				err = -EFAULT;
    660				goto out_err;
    661			}
    662
    663			siblings[n] =
    664				intel_engine_lookup_user(i915, ci.engine_class,
    665							 ci.engine_instance);
    666			if (!siblings[n]) {
    667				drm_dbg(&i915->drm,
    668					"Invalid sibling[%d]: { class:%d, inst:%d }\n",
    669					n, ci.engine_class, ci.engine_instance);
    670				err = -EINVAL;
    671				goto out_err;
    672			}
    673
    674			/*
    675			 * We don't support breadcrumb handshake on these
    676			 * classes
    677			 */
    678			if (siblings[n]->class == RENDER_CLASS ||
    679			    siblings[n]->class == COMPUTE_CLASS) {
    680				err = -EINVAL;
    681				goto out_err;
    682			}
    683
    684			if (n) {
    685				if (prev_engine.engine_class !=
    686				    ci.engine_class) {
    687					drm_dbg(&i915->drm,
    688						"Mismatched class %d, %d\n",
    689						prev_engine.engine_class,
    690						ci.engine_class);
    691					err = -EINVAL;
    692					goto out_err;
    693				}
    694			}
    695
    696			prev_engine = ci;
    697			current_mask |= siblings[n]->logical_mask;
    698		}
    699
    700		if (i > 0) {
    701			if (current_mask != prev_mask << 1) {
    702				drm_dbg(&i915->drm,
    703					"Non contiguous logical mask 0x%x, 0x%x\n",
    704					prev_mask, current_mask);
    705				err = -EINVAL;
    706				goto out_err;
    707			}
    708		}
    709		prev_mask = current_mask;
    710	}
    711
    712	set->engines[slot].type = I915_GEM_ENGINE_TYPE_PARALLEL;
    713	set->engines[slot].num_siblings = num_siblings;
    714	set->engines[slot].width = width;
    715	set->engines[slot].siblings = siblings;
    716
    717	return 0;
    718
    719out_err:
    720	kfree(siblings);
    721
    722	return err;
    723}
    724
    725static const i915_user_extension_fn set_proto_ctx_engines_extensions[] = {
    726	[I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_proto_ctx_engines_balance,
    727	[I915_CONTEXT_ENGINES_EXT_BOND] = set_proto_ctx_engines_bond,
    728	[I915_CONTEXT_ENGINES_EXT_PARALLEL_SUBMIT] =
    729		set_proto_ctx_engines_parallel_submit,
    730};
    731
    732static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv,
    733			         struct i915_gem_proto_context *pc,
    734			         const struct drm_i915_gem_context_param *args)
    735{
    736	struct drm_i915_private *i915 = fpriv->dev_priv;
    737	struct set_proto_ctx_engines set = { .i915 = i915 };
    738	struct i915_context_param_engines __user *user =
    739		u64_to_user_ptr(args->value);
    740	unsigned int n;
    741	u64 extensions;
    742	int err;
    743
    744	if (pc->num_user_engines >= 0) {
    745		drm_dbg(&i915->drm, "Cannot set engines twice");
    746		return -EINVAL;
    747	}
    748
    749	if (args->size < sizeof(*user) ||
    750	    !IS_ALIGNED(args->size - sizeof(*user), sizeof(*user->engines))) {
    751		drm_dbg(&i915->drm, "Invalid size for engine array: %d\n",
    752			args->size);
    753		return -EINVAL;
    754	}
    755
    756	set.num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines);
    757	/* RING_MASK has no shift so we can use it directly here */
    758	if (set.num_engines > I915_EXEC_RING_MASK + 1)
    759		return -EINVAL;
    760
    761	set.engines = kmalloc_array(set.num_engines, sizeof(*set.engines), GFP_KERNEL);
    762	if (!set.engines)
    763		return -ENOMEM;
    764
    765	for (n = 0; n < set.num_engines; n++) {
    766		struct i915_engine_class_instance ci;
    767		struct intel_engine_cs *engine;
    768
    769		if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) {
    770			kfree(set.engines);
    771			return -EFAULT;
    772		}
    773
    774		memset(&set.engines[n], 0, sizeof(set.engines[n]));
    775
    776		if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID &&
    777		    ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE)
    778			continue;
    779
    780		engine = intel_engine_lookup_user(i915,
    781						  ci.engine_class,
    782						  ci.engine_instance);
    783		if (!engine) {
    784			drm_dbg(&i915->drm,
    785				"Invalid engine[%d]: { class:%d, instance:%d }\n",
    786				n, ci.engine_class, ci.engine_instance);
    787			kfree(set.engines);
    788			return -ENOENT;
    789		}
    790
    791		set.engines[n].type = I915_GEM_ENGINE_TYPE_PHYSICAL;
    792		set.engines[n].engine = engine;
    793	}
    794
    795	err = -EFAULT;
    796	if (!get_user(extensions, &user->extensions))
    797		err = i915_user_extensions(u64_to_user_ptr(extensions),
    798					   set_proto_ctx_engines_extensions,
    799					   ARRAY_SIZE(set_proto_ctx_engines_extensions),
    800					   &set);
    801	if (err) {
    802		kfree(set.engines);
    803		return err;
    804	}
    805
    806	pc->num_user_engines = set.num_engines;
    807	pc->user_engines = set.engines;
    808
    809	return 0;
    810}
    811
    812static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv,
    813			      struct i915_gem_proto_context *pc,
    814			      struct drm_i915_gem_context_param *args)
    815{
    816	struct drm_i915_private *i915 = fpriv->dev_priv;
    817	struct drm_i915_gem_context_param_sseu user_sseu;
    818	struct intel_sseu *sseu;
    819	int ret;
    820
    821	if (args->size < sizeof(user_sseu))
    822		return -EINVAL;
    823
    824	if (GRAPHICS_VER(i915) != 11)
    825		return -ENODEV;
    826
    827	if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value),
    828			   sizeof(user_sseu)))
    829		return -EFAULT;
    830
    831	if (user_sseu.rsvd)
    832		return -EINVAL;
    833
    834	if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX))
    835		return -EINVAL;
    836
    837	if (!!(user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) != (pc->num_user_engines >= 0))
    838		return -EINVAL;
    839
    840	if (pc->num_user_engines >= 0) {
    841		int idx = user_sseu.engine.engine_instance;
    842		struct i915_gem_proto_engine *pe;
    843
    844		if (idx >= pc->num_user_engines)
    845			return -EINVAL;
    846
    847		pe = &pc->user_engines[idx];
    848
    849		/* Only render engine supports RPCS configuration. */
    850		if (pe->engine->class != RENDER_CLASS)
    851			return -EINVAL;
    852
    853		sseu = &pe->sseu;
    854	} else {
    855		/* Only render engine supports RPCS configuration. */
    856		if (user_sseu.engine.engine_class != I915_ENGINE_CLASS_RENDER)
    857			return -EINVAL;
    858
    859		/* There is only one render engine */
    860		if (user_sseu.engine.engine_instance != 0)
    861			return -EINVAL;
    862
    863		sseu = &pc->legacy_rcs_sseu;
    864	}
    865
    866	ret = i915_gem_user_to_context_sseu(to_gt(i915), &user_sseu, sseu);
    867	if (ret)
    868		return ret;
    869
    870	args->size = sizeof(user_sseu);
    871
    872	return 0;
    873}
    874
    875static int set_proto_ctx_param(struct drm_i915_file_private *fpriv,
    876			       struct i915_gem_proto_context *pc,
    877			       struct drm_i915_gem_context_param *args)
    878{
    879	int ret = 0;
    880
    881	switch (args->param) {
    882	case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
    883		if (args->size)
    884			ret = -EINVAL;
    885		else if (args->value)
    886			pc->user_flags |= BIT(UCONTEXT_NO_ERROR_CAPTURE);
    887		else
    888			pc->user_flags &= ~BIT(UCONTEXT_NO_ERROR_CAPTURE);
    889		break;
    890
    891	case I915_CONTEXT_PARAM_BANNABLE:
    892		if (args->size)
    893			ret = -EINVAL;
    894		else if (!capable(CAP_SYS_ADMIN) && !args->value)
    895			ret = -EPERM;
    896		else if (args->value)
    897			pc->user_flags |= BIT(UCONTEXT_BANNABLE);
    898		else if (pc->uses_protected_content)
    899			ret = -EPERM;
    900		else
    901			pc->user_flags &= ~BIT(UCONTEXT_BANNABLE);
    902		break;
    903
    904	case I915_CONTEXT_PARAM_RECOVERABLE:
    905		if (args->size)
    906			ret = -EINVAL;
    907		else if (!args->value)
    908			pc->user_flags &= ~BIT(UCONTEXT_RECOVERABLE);
    909		else if (pc->uses_protected_content)
    910			ret = -EPERM;
    911		else
    912			pc->user_flags |= BIT(UCONTEXT_RECOVERABLE);
    913		break;
    914
    915	case I915_CONTEXT_PARAM_PRIORITY:
    916		ret = validate_priority(fpriv->dev_priv, args);
    917		if (!ret)
    918			pc->sched.priority = args->value;
    919		break;
    920
    921	case I915_CONTEXT_PARAM_SSEU:
    922		ret = set_proto_ctx_sseu(fpriv, pc, args);
    923		break;
    924
    925	case I915_CONTEXT_PARAM_VM:
    926		ret = set_proto_ctx_vm(fpriv, pc, args);
    927		break;
    928
    929	case I915_CONTEXT_PARAM_ENGINES:
    930		ret = set_proto_ctx_engines(fpriv, pc, args);
    931		break;
    932
    933	case I915_CONTEXT_PARAM_PERSISTENCE:
    934		if (args->size)
    935			ret = -EINVAL;
    936		else
    937			ret = proto_context_set_persistence(fpriv->dev_priv, pc,
    938							    args->value);
    939		break;
    940
    941	case I915_CONTEXT_PARAM_PROTECTED_CONTENT:
    942		ret = proto_context_set_protected(fpriv->dev_priv, pc,
    943						  args->value);
    944		break;
    945
    946	case I915_CONTEXT_PARAM_NO_ZEROMAP:
    947	case I915_CONTEXT_PARAM_BAN_PERIOD:
    948	case I915_CONTEXT_PARAM_RINGSIZE:
    949	default:
    950		ret = -EINVAL;
    951		break;
    952	}
    953
    954	return ret;
    955}
    956
    957static int intel_context_set_gem(struct intel_context *ce,
    958				 struct i915_gem_context *ctx,
    959				 struct intel_sseu sseu)
    960{
    961	int ret = 0;
    962
    963	GEM_BUG_ON(rcu_access_pointer(ce->gem_context));
    964	RCU_INIT_POINTER(ce->gem_context, ctx);
    965
    966	GEM_BUG_ON(intel_context_is_pinned(ce));
    967	ce->ring_size = SZ_16K;
    968
    969	i915_vm_put(ce->vm);
    970	ce->vm = i915_gem_context_get_eb_vm(ctx);
    971
    972	if (ctx->sched.priority >= I915_PRIORITY_NORMAL &&
    973	    intel_engine_has_timeslices(ce->engine) &&
    974	    intel_engine_has_semaphores(ce->engine))
    975		__set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags);
    976
    977	if (CONFIG_DRM_I915_REQUEST_TIMEOUT &&
    978	    ctx->i915->params.request_timeout_ms) {
    979		unsigned int timeout_ms = ctx->i915->params.request_timeout_ms;
    980
    981		intel_context_set_watchdog_us(ce, (u64)timeout_ms * 1000);
    982	}
    983
    984	/* A valid SSEU has no zero fields */
    985	if (sseu.slice_mask && !WARN_ON(ce->engine->class != RENDER_CLASS))
    986		ret = intel_context_reconfigure_sseu(ce, sseu);
    987
    988	return ret;
    989}
    990
    991static void __unpin_engines(struct i915_gem_engines *e, unsigned int count)
    992{
    993	while (count--) {
    994		struct intel_context *ce = e->engines[count], *child;
    995
    996		if (!ce || !test_bit(CONTEXT_PERMA_PIN, &ce->flags))
    997			continue;
    998
    999		for_each_child(ce, child)
   1000			intel_context_unpin(child);
   1001		intel_context_unpin(ce);
   1002	}
   1003}
   1004
   1005static void unpin_engines(struct i915_gem_engines *e)
   1006{
   1007	__unpin_engines(e, e->num_engines);
   1008}
   1009
   1010static void __free_engines(struct i915_gem_engines *e, unsigned int count)
   1011{
   1012	while (count--) {
   1013		if (!e->engines[count])
   1014			continue;
   1015
   1016		intel_context_put(e->engines[count]);
   1017	}
   1018	kfree(e);
   1019}
   1020
   1021static void free_engines(struct i915_gem_engines *e)
   1022{
   1023	__free_engines(e, e->num_engines);
   1024}
   1025
   1026static void free_engines_rcu(struct rcu_head *rcu)
   1027{
   1028	struct i915_gem_engines *engines =
   1029		container_of(rcu, struct i915_gem_engines, rcu);
   1030
   1031	i915_sw_fence_fini(&engines->fence);
   1032	free_engines(engines);
   1033}
   1034
   1035static void accumulate_runtime(struct i915_drm_client *client,
   1036			       struct i915_gem_engines *engines)
   1037{
   1038	struct i915_gem_engines_iter it;
   1039	struct intel_context *ce;
   1040
   1041	if (!client)
   1042		return;
   1043
   1044	/* Transfer accumulated runtime to the parent GEM context. */
   1045	for_each_gem_engine(ce, engines, it) {
   1046		unsigned int class = ce->engine->uabi_class;
   1047
   1048		GEM_BUG_ON(class >= ARRAY_SIZE(client->past_runtime));
   1049		atomic64_add(intel_context_get_total_runtime_ns(ce),
   1050			     &client->past_runtime[class]);
   1051	}
   1052}
   1053
   1054static int
   1055engines_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
   1056{
   1057	struct i915_gem_engines *engines =
   1058		container_of(fence, typeof(*engines), fence);
   1059	struct i915_gem_context *ctx = engines->ctx;
   1060
   1061	switch (state) {
   1062	case FENCE_COMPLETE:
   1063		if (!list_empty(&engines->link)) {
   1064			unsigned long flags;
   1065
   1066			spin_lock_irqsave(&ctx->stale.lock, flags);
   1067			list_del(&engines->link);
   1068			spin_unlock_irqrestore(&ctx->stale.lock, flags);
   1069		}
   1070		accumulate_runtime(ctx->client, engines);
   1071		i915_gem_context_put(ctx);
   1072
   1073		break;
   1074
   1075	case FENCE_FREE:
   1076		init_rcu_head(&engines->rcu);
   1077		call_rcu(&engines->rcu, free_engines_rcu);
   1078		break;
   1079	}
   1080
   1081	return NOTIFY_DONE;
   1082}
   1083
   1084static struct i915_gem_engines *alloc_engines(unsigned int count)
   1085{
   1086	struct i915_gem_engines *e;
   1087
   1088	e = kzalloc(struct_size(e, engines, count), GFP_KERNEL);
   1089	if (!e)
   1090		return NULL;
   1091
   1092	i915_sw_fence_init(&e->fence, engines_notify);
   1093	return e;
   1094}
   1095
   1096static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx,
   1097						struct intel_sseu rcs_sseu)
   1098{
   1099	const struct intel_gt *gt = to_gt(ctx->i915);
   1100	struct intel_engine_cs *engine;
   1101	struct i915_gem_engines *e, *err;
   1102	enum intel_engine_id id;
   1103
   1104	e = alloc_engines(I915_NUM_ENGINES);
   1105	if (!e)
   1106		return ERR_PTR(-ENOMEM);
   1107
   1108	for_each_engine(engine, gt, id) {
   1109		struct intel_context *ce;
   1110		struct intel_sseu sseu = {};
   1111		int ret;
   1112
   1113		if (engine->legacy_idx == INVALID_ENGINE)
   1114			continue;
   1115
   1116		GEM_BUG_ON(engine->legacy_idx >= I915_NUM_ENGINES);
   1117		GEM_BUG_ON(e->engines[engine->legacy_idx]);
   1118
   1119		ce = intel_context_create(engine);
   1120		if (IS_ERR(ce)) {
   1121			err = ERR_CAST(ce);
   1122			goto free_engines;
   1123		}
   1124
   1125		e->engines[engine->legacy_idx] = ce;
   1126		e->num_engines = max(e->num_engines, engine->legacy_idx + 1);
   1127
   1128		if (engine->class == RENDER_CLASS)
   1129			sseu = rcs_sseu;
   1130
   1131		ret = intel_context_set_gem(ce, ctx, sseu);
   1132		if (ret) {
   1133			err = ERR_PTR(ret);
   1134			goto free_engines;
   1135		}
   1136
   1137	}
   1138
   1139	return e;
   1140
   1141free_engines:
   1142	free_engines(e);
   1143	return err;
   1144}
   1145
   1146static int perma_pin_contexts(struct intel_context *ce)
   1147{
   1148	struct intel_context *child;
   1149	int i = 0, j = 0, ret;
   1150
   1151	GEM_BUG_ON(!intel_context_is_parent(ce));
   1152
   1153	ret = intel_context_pin(ce);
   1154	if (unlikely(ret))
   1155		return ret;
   1156
   1157	for_each_child(ce, child) {
   1158		ret = intel_context_pin(child);
   1159		if (unlikely(ret))
   1160			goto unwind;
   1161		++i;
   1162	}
   1163
   1164	set_bit(CONTEXT_PERMA_PIN, &ce->flags);
   1165
   1166	return 0;
   1167
   1168unwind:
   1169	intel_context_unpin(ce);
   1170	for_each_child(ce, child) {
   1171		if (j++ < i)
   1172			intel_context_unpin(child);
   1173		else
   1174			break;
   1175	}
   1176
   1177	return ret;
   1178}
   1179
   1180static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx,
   1181					     unsigned int num_engines,
   1182					     struct i915_gem_proto_engine *pe)
   1183{
   1184	struct i915_gem_engines *e, *err;
   1185	unsigned int n;
   1186
   1187	e = alloc_engines(num_engines);
   1188	if (!e)
   1189		return ERR_PTR(-ENOMEM);
   1190	e->num_engines = num_engines;
   1191
   1192	for (n = 0; n < num_engines; n++) {
   1193		struct intel_context *ce, *child;
   1194		int ret;
   1195
   1196		switch (pe[n].type) {
   1197		case I915_GEM_ENGINE_TYPE_PHYSICAL:
   1198			ce = intel_context_create(pe[n].engine);
   1199			break;
   1200
   1201		case I915_GEM_ENGINE_TYPE_BALANCED:
   1202			ce = intel_engine_create_virtual(pe[n].siblings,
   1203							 pe[n].num_siblings, 0);
   1204			break;
   1205
   1206		case I915_GEM_ENGINE_TYPE_PARALLEL:
   1207			ce = intel_engine_create_parallel(pe[n].siblings,
   1208							  pe[n].num_siblings,
   1209							  pe[n].width);
   1210			break;
   1211
   1212		case I915_GEM_ENGINE_TYPE_INVALID:
   1213		default:
   1214			GEM_WARN_ON(pe[n].type != I915_GEM_ENGINE_TYPE_INVALID);
   1215			continue;
   1216		}
   1217
   1218		if (IS_ERR(ce)) {
   1219			err = ERR_CAST(ce);
   1220			goto free_engines;
   1221		}
   1222
   1223		e->engines[n] = ce;
   1224
   1225		ret = intel_context_set_gem(ce, ctx, pe->sseu);
   1226		if (ret) {
   1227			err = ERR_PTR(ret);
   1228			goto free_engines;
   1229		}
   1230		for_each_child(ce, child) {
   1231			ret = intel_context_set_gem(child, ctx, pe->sseu);
   1232			if (ret) {
   1233				err = ERR_PTR(ret);
   1234				goto free_engines;
   1235			}
   1236		}
   1237
   1238		/*
   1239		 * XXX: Must be done after calling intel_context_set_gem as that
   1240		 * function changes the ring size. The ring is allocated when
   1241		 * the context is pinned. If the ring size is changed after
   1242		 * allocation we have a mismatch of the ring size and will cause
   1243		 * the context to hang. Presumably with a bit of reordering we
   1244		 * could move the perma-pin step to the backend function
   1245		 * intel_engine_create_parallel.
   1246		 */
   1247		if (pe[n].type == I915_GEM_ENGINE_TYPE_PARALLEL) {
   1248			ret = perma_pin_contexts(ce);
   1249			if (ret) {
   1250				err = ERR_PTR(ret);
   1251				goto free_engines;
   1252			}
   1253		}
   1254	}
   1255
   1256	return e;
   1257
   1258free_engines:
   1259	free_engines(e);
   1260	return err;
   1261}
   1262
   1263static void i915_gem_context_release_work(struct work_struct *work)
   1264{
   1265	struct i915_gem_context *ctx = container_of(work, typeof(*ctx),
   1266						    release_work);
   1267	struct i915_address_space *vm;
   1268
   1269	trace_i915_context_free(ctx);
   1270	GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
   1271
   1272	if (ctx->syncobj)
   1273		drm_syncobj_put(ctx->syncobj);
   1274
   1275	vm = ctx->vm;
   1276	if (vm)
   1277		i915_vm_put(vm);
   1278
   1279	if (ctx->pxp_wakeref)
   1280		intel_runtime_pm_put(&ctx->i915->runtime_pm, ctx->pxp_wakeref);
   1281
   1282	if (ctx->client)
   1283		i915_drm_client_put(ctx->client);
   1284
   1285	mutex_destroy(&ctx->engines_mutex);
   1286	mutex_destroy(&ctx->lut_mutex);
   1287
   1288	put_pid(ctx->pid);
   1289	mutex_destroy(&ctx->mutex);
   1290
   1291	kfree_rcu(ctx, rcu);
   1292}
   1293
   1294void i915_gem_context_release(struct kref *ref)
   1295{
   1296	struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref);
   1297
   1298	queue_work(ctx->i915->wq, &ctx->release_work);
   1299}
   1300
   1301static inline struct i915_gem_engines *
   1302__context_engines_static(const struct i915_gem_context *ctx)
   1303{
   1304	return rcu_dereference_protected(ctx->engines, true);
   1305}
   1306
   1307static void __reset_context(struct i915_gem_context *ctx,
   1308			    struct intel_engine_cs *engine)
   1309{
   1310	intel_gt_handle_error(engine->gt, engine->mask, 0,
   1311			      "context closure in %s", ctx->name);
   1312}
   1313
   1314static bool __cancel_engine(struct intel_engine_cs *engine)
   1315{
   1316	/*
   1317	 * Send a "high priority pulse" down the engine to cause the
   1318	 * current request to be momentarily preempted. (If it fails to
   1319	 * be preempted, it will be reset). As we have marked our context
   1320	 * as banned, any incomplete request, including any running, will
   1321	 * be skipped following the preemption.
   1322	 *
   1323	 * If there is no hangchecking (one of the reasons why we try to
   1324	 * cancel the context) and no forced preemption, there may be no
   1325	 * means by which we reset the GPU and evict the persistent hog.
   1326	 * Ergo if we are unable to inject a preemptive pulse that can
   1327	 * kill the banned context, we fallback to doing a local reset
   1328	 * instead.
   1329	 */
   1330	return intel_engine_pulse(engine) == 0;
   1331}
   1332
   1333static struct intel_engine_cs *active_engine(struct intel_context *ce)
   1334{
   1335	struct intel_engine_cs *engine = NULL;
   1336	struct i915_request *rq;
   1337
   1338	if (intel_context_has_inflight(ce))
   1339		return intel_context_inflight(ce);
   1340
   1341	if (!ce->timeline)
   1342		return NULL;
   1343
   1344	/*
   1345	 * rq->link is only SLAB_TYPESAFE_BY_RCU, we need to hold a reference
   1346	 * to the request to prevent it being transferred to a new timeline
   1347	 * (and onto a new timeline->requests list).
   1348	 */
   1349	rcu_read_lock();
   1350	list_for_each_entry_reverse(rq, &ce->timeline->requests, link) {
   1351		bool found;
   1352
   1353		/* timeline is already completed upto this point? */
   1354		if (!i915_request_get_rcu(rq))
   1355			break;
   1356
   1357		/* Check with the backend if the request is inflight */
   1358		found = true;
   1359		if (likely(rcu_access_pointer(rq->timeline) == ce->timeline))
   1360			found = i915_request_active_engine(rq, &engine);
   1361
   1362		i915_request_put(rq);
   1363		if (found)
   1364			break;
   1365	}
   1366	rcu_read_unlock();
   1367
   1368	return engine;
   1369}
   1370
   1371static void kill_engines(struct i915_gem_engines *engines, bool ban)
   1372{
   1373	struct i915_gem_engines_iter it;
   1374	struct intel_context *ce;
   1375
   1376	/*
   1377	 * Map the user's engine back to the actual engines; one virtual
   1378	 * engine will be mapped to multiple engines, and using ctx->engine[]
   1379	 * the same engine may be have multiple instances in the user's map.
   1380	 * However, we only care about pending requests, so only include
   1381	 * engines on which there are incomplete requests.
   1382	 */
   1383	for_each_gem_engine(ce, engines, it) {
   1384		struct intel_engine_cs *engine;
   1385
   1386		if (ban && intel_context_ban(ce, NULL))
   1387			continue;
   1388
   1389		/*
   1390		 * Check the current active state of this context; if we
   1391		 * are currently executing on the GPU we need to evict
   1392		 * ourselves. On the other hand, if we haven't yet been
   1393		 * submitted to the GPU or if everything is complete,
   1394		 * we have nothing to do.
   1395		 */
   1396		engine = active_engine(ce);
   1397
   1398		/* First attempt to gracefully cancel the context */
   1399		if (engine && !__cancel_engine(engine) && ban)
   1400			/*
   1401			 * If we are unable to send a preemptive pulse to bump
   1402			 * the context from the GPU, we have to resort to a full
   1403			 * reset. We hope the collateral damage is worth it.
   1404			 */
   1405			__reset_context(engines->ctx, engine);
   1406	}
   1407}
   1408
   1409static void kill_context(struct i915_gem_context *ctx)
   1410{
   1411	bool ban = (!i915_gem_context_is_persistent(ctx) ||
   1412		    !ctx->i915->params.enable_hangcheck);
   1413	struct i915_gem_engines *pos, *next;
   1414
   1415	spin_lock_irq(&ctx->stale.lock);
   1416	GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
   1417	list_for_each_entry_safe(pos, next, &ctx->stale.engines, link) {
   1418		if (!i915_sw_fence_await(&pos->fence)) {
   1419			list_del_init(&pos->link);
   1420			continue;
   1421		}
   1422
   1423		spin_unlock_irq(&ctx->stale.lock);
   1424
   1425		kill_engines(pos, ban);
   1426
   1427		spin_lock_irq(&ctx->stale.lock);
   1428		GEM_BUG_ON(i915_sw_fence_signaled(&pos->fence));
   1429		list_safe_reset_next(pos, next, link);
   1430		list_del_init(&pos->link); /* decouple from FENCE_COMPLETE */
   1431
   1432		i915_sw_fence_complete(&pos->fence);
   1433	}
   1434	spin_unlock_irq(&ctx->stale.lock);
   1435}
   1436
   1437static void engines_idle_release(struct i915_gem_context *ctx,
   1438				 struct i915_gem_engines *engines)
   1439{
   1440	struct i915_gem_engines_iter it;
   1441	struct intel_context *ce;
   1442
   1443	INIT_LIST_HEAD(&engines->link);
   1444
   1445	engines->ctx = i915_gem_context_get(ctx);
   1446
   1447	for_each_gem_engine(ce, engines, it) {
   1448		int err;
   1449
   1450		/* serialises with execbuf */
   1451		set_bit(CONTEXT_CLOSED_BIT, &ce->flags);
   1452		if (!intel_context_pin_if_active(ce))
   1453			continue;
   1454
   1455		/* Wait until context is finally scheduled out and retired */
   1456		err = i915_sw_fence_await_active(&engines->fence,
   1457						 &ce->active,
   1458						 I915_ACTIVE_AWAIT_BARRIER);
   1459		intel_context_unpin(ce);
   1460		if (err)
   1461			goto kill;
   1462	}
   1463
   1464	spin_lock_irq(&ctx->stale.lock);
   1465	if (!i915_gem_context_is_closed(ctx))
   1466		list_add_tail(&engines->link, &ctx->stale.engines);
   1467	spin_unlock_irq(&ctx->stale.lock);
   1468
   1469kill:
   1470	if (list_empty(&engines->link)) /* raced, already closed */
   1471		kill_engines(engines, true);
   1472
   1473	i915_sw_fence_commit(&engines->fence);
   1474}
   1475
   1476static void set_closed_name(struct i915_gem_context *ctx)
   1477{
   1478	char *s;
   1479
   1480	/* Replace '[]' with '<>' to indicate closed in debug prints */
   1481
   1482	s = strrchr(ctx->name, '[');
   1483	if (!s)
   1484		return;
   1485
   1486	*s = '<';
   1487
   1488	s = strchr(s + 1, ']');
   1489	if (s)
   1490		*s = '>';
   1491}
   1492
   1493static void context_close(struct i915_gem_context *ctx)
   1494{
   1495	struct i915_drm_client *client;
   1496
   1497	/* Flush any concurrent set_engines() */
   1498	mutex_lock(&ctx->engines_mutex);
   1499	unpin_engines(__context_engines_static(ctx));
   1500	engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1));
   1501	i915_gem_context_set_closed(ctx);
   1502	mutex_unlock(&ctx->engines_mutex);
   1503
   1504	mutex_lock(&ctx->mutex);
   1505
   1506	set_closed_name(ctx);
   1507
   1508	/*
   1509	 * The LUT uses the VMA as a backpointer to unref the object,
   1510	 * so we need to clear the LUT before we close all the VMA (inside
   1511	 * the ppgtt).
   1512	 */
   1513	lut_close(ctx);
   1514
   1515	ctx->file_priv = ERR_PTR(-EBADF);
   1516
   1517	spin_lock(&ctx->i915->gem.contexts.lock);
   1518	list_del(&ctx->link);
   1519	spin_unlock(&ctx->i915->gem.contexts.lock);
   1520
   1521	client = ctx->client;
   1522	if (client) {
   1523		spin_lock(&client->ctx_lock);
   1524		list_del_rcu(&ctx->client_link);
   1525		spin_unlock(&client->ctx_lock);
   1526	}
   1527
   1528	mutex_unlock(&ctx->mutex);
   1529
   1530	/*
   1531	 * If the user has disabled hangchecking, we can not be sure that
   1532	 * the batches will ever complete after the context is closed,
   1533	 * keeping the context and all resources pinned forever. So in this
   1534	 * case we opt to forcibly kill off all remaining requests on
   1535	 * context close.
   1536	 */
   1537	kill_context(ctx);
   1538
   1539	i915_gem_context_put(ctx);
   1540}
   1541
   1542static int __context_set_persistence(struct i915_gem_context *ctx, bool state)
   1543{
   1544	if (i915_gem_context_is_persistent(ctx) == state)
   1545		return 0;
   1546
   1547	if (state) {
   1548		/*
   1549		 * Only contexts that are short-lived [that will expire or be
   1550		 * reset] are allowed to survive past termination. We require
   1551		 * hangcheck to ensure that the persistent requests are healthy.
   1552		 */
   1553		if (!ctx->i915->params.enable_hangcheck)
   1554			return -EINVAL;
   1555
   1556		i915_gem_context_set_persistence(ctx);
   1557	} else {
   1558		/* To cancel a context we use "preempt-to-idle" */
   1559		if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
   1560			return -ENODEV;
   1561
   1562		/*
   1563		 * If the cancel fails, we then need to reset, cleanly!
   1564		 *
   1565		 * If the per-engine reset fails, all hope is lost! We resort
   1566		 * to a full GPU reset in that unlikely case, but realistically
   1567		 * if the engine could not reset, the full reset does not fare
   1568		 * much better. The damage has been done.
   1569		 *
   1570		 * However, if we cannot reset an engine by itself, we cannot
   1571		 * cleanup a hanging persistent context without causing
   1572		 * colateral damage, and we should not pretend we can by
   1573		 * exposing the interface.
   1574		 */
   1575		if (!intel_has_reset_engine(to_gt(ctx->i915)))
   1576			return -ENODEV;
   1577
   1578		i915_gem_context_clear_persistence(ctx);
   1579	}
   1580
   1581	return 0;
   1582}
   1583
   1584static struct i915_gem_context *
   1585i915_gem_create_context(struct drm_i915_private *i915,
   1586			const struct i915_gem_proto_context *pc)
   1587{
   1588	struct i915_gem_context *ctx;
   1589	struct i915_address_space *vm = NULL;
   1590	struct i915_gem_engines *e;
   1591	int err;
   1592	int i;
   1593
   1594	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
   1595	if (!ctx)
   1596		return ERR_PTR(-ENOMEM);
   1597
   1598	kref_init(&ctx->ref);
   1599	ctx->i915 = i915;
   1600	ctx->sched = pc->sched;
   1601	mutex_init(&ctx->mutex);
   1602	INIT_LIST_HEAD(&ctx->link);
   1603	INIT_WORK(&ctx->release_work, i915_gem_context_release_work);
   1604
   1605	spin_lock_init(&ctx->stale.lock);
   1606	INIT_LIST_HEAD(&ctx->stale.engines);
   1607
   1608	if (pc->vm) {
   1609		vm = i915_vm_get(pc->vm);
   1610	} else if (HAS_FULL_PPGTT(i915)) {
   1611		struct i915_ppgtt *ppgtt;
   1612
   1613		ppgtt = i915_ppgtt_create(to_gt(i915), 0);
   1614		if (IS_ERR(ppgtt)) {
   1615			drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n",
   1616				PTR_ERR(ppgtt));
   1617			err = PTR_ERR(ppgtt);
   1618			goto err_ctx;
   1619		}
   1620		vm = &ppgtt->vm;
   1621	}
   1622	if (vm)
   1623		ctx->vm = vm;
   1624
   1625	mutex_init(&ctx->engines_mutex);
   1626	if (pc->num_user_engines >= 0) {
   1627		i915_gem_context_set_user_engines(ctx);
   1628		e = user_engines(ctx, pc->num_user_engines, pc->user_engines);
   1629	} else {
   1630		i915_gem_context_clear_user_engines(ctx);
   1631		e = default_engines(ctx, pc->legacy_rcs_sseu);
   1632	}
   1633	if (IS_ERR(e)) {
   1634		err = PTR_ERR(e);
   1635		goto err_vm;
   1636	}
   1637	RCU_INIT_POINTER(ctx->engines, e);
   1638
   1639	INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
   1640	mutex_init(&ctx->lut_mutex);
   1641
   1642	/* NB: Mark all slices as needing a remap so that when the context first
   1643	 * loads it will restore whatever remap state already exists. If there
   1644	 * is no remap info, it will be a NOP. */
   1645	ctx->remap_slice = ALL_L3_SLICES(i915);
   1646
   1647	ctx->user_flags = pc->user_flags;
   1648
   1649	for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++)
   1650		ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES;
   1651
   1652	if (pc->single_timeline) {
   1653		err = drm_syncobj_create(&ctx->syncobj,
   1654					 DRM_SYNCOBJ_CREATE_SIGNALED,
   1655					 NULL);
   1656		if (err)
   1657			goto err_engines;
   1658	}
   1659
   1660	if (pc->uses_protected_content) {
   1661		ctx->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm);
   1662		ctx->uses_protected_content = true;
   1663	}
   1664
   1665	trace_i915_context_create(ctx);
   1666
   1667	return ctx;
   1668
   1669err_engines:
   1670	free_engines(e);
   1671err_vm:
   1672	if (ctx->vm)
   1673		i915_vm_put(ctx->vm);
   1674err_ctx:
   1675	kfree(ctx);
   1676	return ERR_PTR(err);
   1677}
   1678
   1679static void init_contexts(struct i915_gem_contexts *gc)
   1680{
   1681	spin_lock_init(&gc->lock);
   1682	INIT_LIST_HEAD(&gc->list);
   1683}
   1684
   1685void i915_gem_init__contexts(struct drm_i915_private *i915)
   1686{
   1687	init_contexts(&i915->gem.contexts);
   1688}
   1689
   1690static void gem_context_register(struct i915_gem_context *ctx,
   1691				 struct drm_i915_file_private *fpriv,
   1692				 u32 id)
   1693{
   1694	struct drm_i915_private *i915 = ctx->i915;
   1695	void *old;
   1696
   1697	ctx->file_priv = fpriv;
   1698
   1699	ctx->pid = get_task_pid(current, PIDTYPE_PID);
   1700	ctx->client = i915_drm_client_get(fpriv->client);
   1701
   1702	snprintf(ctx->name, sizeof(ctx->name), "%s[%d]",
   1703		 current->comm, pid_nr(ctx->pid));
   1704
   1705	/* And finally expose ourselves to userspace via the idr */
   1706	old = xa_store(&fpriv->context_xa, id, ctx, GFP_KERNEL);
   1707	WARN_ON(old);
   1708
   1709	spin_lock(&ctx->client->ctx_lock);
   1710	list_add_tail_rcu(&ctx->client_link, &ctx->client->ctx_list);
   1711	spin_unlock(&ctx->client->ctx_lock);
   1712
   1713	spin_lock(&i915->gem.contexts.lock);
   1714	list_add_tail(&ctx->link, &i915->gem.contexts.list);
   1715	spin_unlock(&i915->gem.contexts.lock);
   1716}
   1717
   1718int i915_gem_context_open(struct drm_i915_private *i915,
   1719			  struct drm_file *file)
   1720{
   1721	struct drm_i915_file_private *file_priv = file->driver_priv;
   1722	struct i915_gem_proto_context *pc;
   1723	struct i915_gem_context *ctx;
   1724	int err;
   1725
   1726	mutex_init(&file_priv->proto_context_lock);
   1727	xa_init_flags(&file_priv->proto_context_xa, XA_FLAGS_ALLOC);
   1728
   1729	/* 0 reserved for the default context */
   1730	xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC1);
   1731
   1732	/* 0 reserved for invalid/unassigned ppgtt */
   1733	xa_init_flags(&file_priv->vm_xa, XA_FLAGS_ALLOC1);
   1734
   1735	pc = proto_context_create(i915, 0);
   1736	if (IS_ERR(pc)) {
   1737		err = PTR_ERR(pc);
   1738		goto err;
   1739	}
   1740
   1741	ctx = i915_gem_create_context(i915, pc);
   1742	proto_context_close(i915, pc);
   1743	if (IS_ERR(ctx)) {
   1744		err = PTR_ERR(ctx);
   1745		goto err;
   1746	}
   1747
   1748	gem_context_register(ctx, file_priv, 0);
   1749
   1750	return 0;
   1751
   1752err:
   1753	xa_destroy(&file_priv->vm_xa);
   1754	xa_destroy(&file_priv->context_xa);
   1755	xa_destroy(&file_priv->proto_context_xa);
   1756	mutex_destroy(&file_priv->proto_context_lock);
   1757	return err;
   1758}
   1759
   1760void i915_gem_context_close(struct drm_file *file)
   1761{
   1762	struct drm_i915_file_private *file_priv = file->driver_priv;
   1763	struct i915_gem_proto_context *pc;
   1764	struct i915_address_space *vm;
   1765	struct i915_gem_context *ctx;
   1766	unsigned long idx;
   1767
   1768	xa_for_each(&file_priv->proto_context_xa, idx, pc)
   1769		proto_context_close(file_priv->dev_priv, pc);
   1770	xa_destroy(&file_priv->proto_context_xa);
   1771	mutex_destroy(&file_priv->proto_context_lock);
   1772
   1773	xa_for_each(&file_priv->context_xa, idx, ctx)
   1774		context_close(ctx);
   1775	xa_destroy(&file_priv->context_xa);
   1776
   1777	xa_for_each(&file_priv->vm_xa, idx, vm)
   1778		i915_vm_put(vm);
   1779	xa_destroy(&file_priv->vm_xa);
   1780}
   1781
   1782int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data,
   1783			     struct drm_file *file)
   1784{
   1785	struct drm_i915_private *i915 = to_i915(dev);
   1786	struct drm_i915_gem_vm_control *args = data;
   1787	struct drm_i915_file_private *file_priv = file->driver_priv;
   1788	struct i915_ppgtt *ppgtt;
   1789	u32 id;
   1790	int err;
   1791
   1792	if (!HAS_FULL_PPGTT(i915))
   1793		return -ENODEV;
   1794
   1795	if (args->flags)
   1796		return -EINVAL;
   1797
   1798	ppgtt = i915_ppgtt_create(to_gt(i915), 0);
   1799	if (IS_ERR(ppgtt))
   1800		return PTR_ERR(ppgtt);
   1801
   1802	if (args->extensions) {
   1803		err = i915_user_extensions(u64_to_user_ptr(args->extensions),
   1804					   NULL, 0,
   1805					   ppgtt);
   1806		if (err)
   1807			goto err_put;
   1808	}
   1809
   1810	err = xa_alloc(&file_priv->vm_xa, &id, &ppgtt->vm,
   1811		       xa_limit_32b, GFP_KERNEL);
   1812	if (err)
   1813		goto err_put;
   1814
   1815	GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */
   1816	args->vm_id = id;
   1817	return 0;
   1818
   1819err_put:
   1820	i915_vm_put(&ppgtt->vm);
   1821	return err;
   1822}
   1823
   1824int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data,
   1825			      struct drm_file *file)
   1826{
   1827	struct drm_i915_file_private *file_priv = file->driver_priv;
   1828	struct drm_i915_gem_vm_control *args = data;
   1829	struct i915_address_space *vm;
   1830
   1831	if (args->flags)
   1832		return -EINVAL;
   1833
   1834	if (args->extensions)
   1835		return -EINVAL;
   1836
   1837	vm = xa_erase(&file_priv->vm_xa, args->vm_id);
   1838	if (!vm)
   1839		return -ENOENT;
   1840
   1841	i915_vm_put(vm);
   1842	return 0;
   1843}
   1844
   1845static int get_ppgtt(struct drm_i915_file_private *file_priv,
   1846		     struct i915_gem_context *ctx,
   1847		     struct drm_i915_gem_context_param *args)
   1848{
   1849	struct i915_address_space *vm;
   1850	int err;
   1851	u32 id;
   1852
   1853	if (!i915_gem_context_has_full_ppgtt(ctx))
   1854		return -ENODEV;
   1855
   1856	vm = ctx->vm;
   1857	GEM_BUG_ON(!vm);
   1858
   1859	err = xa_alloc(&file_priv->vm_xa, &id, vm, xa_limit_32b, GFP_KERNEL);
   1860	if (err)
   1861		return err;
   1862
   1863	i915_vm_get(vm);
   1864
   1865	GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */
   1866	args->value = id;
   1867	args->size = 0;
   1868
   1869	return err;
   1870}
   1871
   1872int
   1873i915_gem_user_to_context_sseu(struct intel_gt *gt,
   1874			      const struct drm_i915_gem_context_param_sseu *user,
   1875			      struct intel_sseu *context)
   1876{
   1877	const struct sseu_dev_info *device = &gt->info.sseu;
   1878	struct drm_i915_private *i915 = gt->i915;
   1879
   1880	/* No zeros in any field. */
   1881	if (!user->slice_mask || !user->subslice_mask ||
   1882	    !user->min_eus_per_subslice || !user->max_eus_per_subslice)
   1883		return -EINVAL;
   1884
   1885	/* Max > min. */
   1886	if (user->max_eus_per_subslice < user->min_eus_per_subslice)
   1887		return -EINVAL;
   1888
   1889	/*
   1890	 * Some future proofing on the types since the uAPI is wider than the
   1891	 * current internal implementation.
   1892	 */
   1893	if (overflows_type(user->slice_mask, context->slice_mask) ||
   1894	    overflows_type(user->subslice_mask, context->subslice_mask) ||
   1895	    overflows_type(user->min_eus_per_subslice,
   1896			   context->min_eus_per_subslice) ||
   1897	    overflows_type(user->max_eus_per_subslice,
   1898			   context->max_eus_per_subslice))
   1899		return -EINVAL;
   1900
   1901	/* Check validity against hardware. */
   1902	if (user->slice_mask & ~device->slice_mask)
   1903		return -EINVAL;
   1904
   1905	if (user->subslice_mask & ~device->subslice_mask[0])
   1906		return -EINVAL;
   1907
   1908	if (user->max_eus_per_subslice > device->max_eus_per_subslice)
   1909		return -EINVAL;
   1910
   1911	context->slice_mask = user->slice_mask;
   1912	context->subslice_mask = user->subslice_mask;
   1913	context->min_eus_per_subslice = user->min_eus_per_subslice;
   1914	context->max_eus_per_subslice = user->max_eus_per_subslice;
   1915
   1916	/* Part specific restrictions. */
   1917	if (GRAPHICS_VER(i915) == 11) {
   1918		unsigned int hw_s = hweight8(device->slice_mask);
   1919		unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]);
   1920		unsigned int req_s = hweight8(context->slice_mask);
   1921		unsigned int req_ss = hweight8(context->subslice_mask);
   1922
   1923		/*
   1924		 * Only full subslice enablement is possible if more than one
   1925		 * slice is turned on.
   1926		 */
   1927		if (req_s > 1 && req_ss != hw_ss_per_s)
   1928			return -EINVAL;
   1929
   1930		/*
   1931		 * If more than four (SScount bitfield limit) subslices are
   1932		 * requested then the number has to be even.
   1933		 */
   1934		if (req_ss > 4 && (req_ss & 1))
   1935			return -EINVAL;
   1936
   1937		/*
   1938		 * If only one slice is enabled and subslice count is below the
   1939		 * device full enablement, it must be at most half of the all
   1940		 * available subslices.
   1941		 */
   1942		if (req_s == 1 && req_ss < hw_ss_per_s &&
   1943		    req_ss > (hw_ss_per_s / 2))
   1944			return -EINVAL;
   1945
   1946		/* ABI restriction - VME use case only. */
   1947
   1948		/* All slices or one slice only. */
   1949		if (req_s != 1 && req_s != hw_s)
   1950			return -EINVAL;
   1951
   1952		/*
   1953		 * Half subslices or full enablement only when one slice is
   1954		 * enabled.
   1955		 */
   1956		if (req_s == 1 &&
   1957		    (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2)))
   1958			return -EINVAL;
   1959
   1960		/* No EU configuration changes. */
   1961		if ((user->min_eus_per_subslice !=
   1962		     device->max_eus_per_subslice) ||
   1963		    (user->max_eus_per_subslice !=
   1964		     device->max_eus_per_subslice))
   1965			return -EINVAL;
   1966	}
   1967
   1968	return 0;
   1969}
   1970
   1971static int set_sseu(struct i915_gem_context *ctx,
   1972		    struct drm_i915_gem_context_param *args)
   1973{
   1974	struct drm_i915_private *i915 = ctx->i915;
   1975	struct drm_i915_gem_context_param_sseu user_sseu;
   1976	struct intel_context *ce;
   1977	struct intel_sseu sseu;
   1978	unsigned long lookup;
   1979	int ret;
   1980
   1981	if (args->size < sizeof(user_sseu))
   1982		return -EINVAL;
   1983
   1984	if (GRAPHICS_VER(i915) != 11)
   1985		return -ENODEV;
   1986
   1987	if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value),
   1988			   sizeof(user_sseu)))
   1989		return -EFAULT;
   1990
   1991	if (user_sseu.rsvd)
   1992		return -EINVAL;
   1993
   1994	if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX))
   1995		return -EINVAL;
   1996
   1997	lookup = 0;
   1998	if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)
   1999		lookup |= LOOKUP_USER_INDEX;
   2000
   2001	ce = lookup_user_engine(ctx, lookup, &user_sseu.engine);
   2002	if (IS_ERR(ce))
   2003		return PTR_ERR(ce);
   2004
   2005	/* Only render engine supports RPCS configuration. */
   2006	if (ce->engine->class != RENDER_CLASS) {
   2007		ret = -ENODEV;
   2008		goto out_ce;
   2009	}
   2010
   2011	ret = i915_gem_user_to_context_sseu(ce->engine->gt, &user_sseu, &sseu);
   2012	if (ret)
   2013		goto out_ce;
   2014
   2015	ret = intel_context_reconfigure_sseu(ce, sseu);
   2016	if (ret)
   2017		goto out_ce;
   2018
   2019	args->size = sizeof(user_sseu);
   2020
   2021out_ce:
   2022	intel_context_put(ce);
   2023	return ret;
   2024}
   2025
   2026static int
   2027set_persistence(struct i915_gem_context *ctx,
   2028		const struct drm_i915_gem_context_param *args)
   2029{
   2030	if (args->size)
   2031		return -EINVAL;
   2032
   2033	return __context_set_persistence(ctx, args->value);
   2034}
   2035
   2036static int set_priority(struct i915_gem_context *ctx,
   2037			const struct drm_i915_gem_context_param *args)
   2038{
   2039	struct i915_gem_engines_iter it;
   2040	struct intel_context *ce;
   2041	int err;
   2042
   2043	err = validate_priority(ctx->i915, args);
   2044	if (err)
   2045		return err;
   2046
   2047	ctx->sched.priority = args->value;
   2048
   2049	for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
   2050		if (!intel_engine_has_timeslices(ce->engine))
   2051			continue;
   2052
   2053		if (ctx->sched.priority >= I915_PRIORITY_NORMAL &&
   2054		    intel_engine_has_semaphores(ce->engine))
   2055			intel_context_set_use_semaphores(ce);
   2056		else
   2057			intel_context_clear_use_semaphores(ce);
   2058	}
   2059	i915_gem_context_unlock_engines(ctx);
   2060
   2061	return 0;
   2062}
   2063
   2064static int get_protected(struct i915_gem_context *ctx,
   2065			 struct drm_i915_gem_context_param *args)
   2066{
   2067	args->size = 0;
   2068	args->value = i915_gem_context_uses_protected_content(ctx);
   2069
   2070	return 0;
   2071}
   2072
   2073static int ctx_setparam(struct drm_i915_file_private *fpriv,
   2074			struct i915_gem_context *ctx,
   2075			struct drm_i915_gem_context_param *args)
   2076{
   2077	int ret = 0;
   2078
   2079	switch (args->param) {
   2080	case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
   2081		if (args->size)
   2082			ret = -EINVAL;
   2083		else if (args->value)
   2084			i915_gem_context_set_no_error_capture(ctx);
   2085		else
   2086			i915_gem_context_clear_no_error_capture(ctx);
   2087		break;
   2088
   2089	case I915_CONTEXT_PARAM_BANNABLE:
   2090		if (args->size)
   2091			ret = -EINVAL;
   2092		else if (!capable(CAP_SYS_ADMIN) && !args->value)
   2093			ret = -EPERM;
   2094		else if (args->value)
   2095			i915_gem_context_set_bannable(ctx);
   2096		else if (i915_gem_context_uses_protected_content(ctx))
   2097			ret = -EPERM; /* can't clear this for protected contexts */
   2098		else
   2099			i915_gem_context_clear_bannable(ctx);
   2100		break;
   2101
   2102	case I915_CONTEXT_PARAM_RECOVERABLE:
   2103		if (args->size)
   2104			ret = -EINVAL;
   2105		else if (!args->value)
   2106			i915_gem_context_clear_recoverable(ctx);
   2107		else if (i915_gem_context_uses_protected_content(ctx))
   2108			ret = -EPERM; /* can't set this for protected contexts */
   2109		else
   2110			i915_gem_context_set_recoverable(ctx);
   2111		break;
   2112
   2113	case I915_CONTEXT_PARAM_PRIORITY:
   2114		ret = set_priority(ctx, args);
   2115		break;
   2116
   2117	case I915_CONTEXT_PARAM_SSEU:
   2118		ret = set_sseu(ctx, args);
   2119		break;
   2120
   2121	case I915_CONTEXT_PARAM_PERSISTENCE:
   2122		ret = set_persistence(ctx, args);
   2123		break;
   2124
   2125	case I915_CONTEXT_PARAM_PROTECTED_CONTENT:
   2126	case I915_CONTEXT_PARAM_NO_ZEROMAP:
   2127	case I915_CONTEXT_PARAM_BAN_PERIOD:
   2128	case I915_CONTEXT_PARAM_RINGSIZE:
   2129	case I915_CONTEXT_PARAM_VM:
   2130	case I915_CONTEXT_PARAM_ENGINES:
   2131	default:
   2132		ret = -EINVAL;
   2133		break;
   2134	}
   2135
   2136	return ret;
   2137}
   2138
   2139struct create_ext {
   2140	struct i915_gem_proto_context *pc;
   2141	struct drm_i915_file_private *fpriv;
   2142};
   2143
   2144static int create_setparam(struct i915_user_extension __user *ext, void *data)
   2145{
   2146	struct drm_i915_gem_context_create_ext_setparam local;
   2147	const struct create_ext *arg = data;
   2148
   2149	if (copy_from_user(&local, ext, sizeof(local)))
   2150		return -EFAULT;
   2151
   2152	if (local.param.ctx_id)
   2153		return -EINVAL;
   2154
   2155	return set_proto_ctx_param(arg->fpriv, arg->pc, &local.param);
   2156}
   2157
   2158static int invalid_ext(struct i915_user_extension __user *ext, void *data)
   2159{
   2160	return -EINVAL;
   2161}
   2162
   2163static const i915_user_extension_fn create_extensions[] = {
   2164	[I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam,
   2165	[I915_CONTEXT_CREATE_EXT_CLONE] = invalid_ext,
   2166};
   2167
   2168static bool client_is_banned(struct drm_i915_file_private *file_priv)
   2169{
   2170	return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED;
   2171}
   2172
   2173static inline struct i915_gem_context *
   2174__context_lookup(struct drm_i915_file_private *file_priv, u32 id)
   2175{
   2176	struct i915_gem_context *ctx;
   2177
   2178	rcu_read_lock();
   2179	ctx = xa_load(&file_priv->context_xa, id);
   2180	if (ctx && !kref_get_unless_zero(&ctx->ref))
   2181		ctx = NULL;
   2182	rcu_read_unlock();
   2183
   2184	return ctx;
   2185}
   2186
   2187static struct i915_gem_context *
   2188finalize_create_context_locked(struct drm_i915_file_private *file_priv,
   2189			       struct i915_gem_proto_context *pc, u32 id)
   2190{
   2191	struct i915_gem_context *ctx;
   2192	void *old;
   2193
   2194	lockdep_assert_held(&file_priv->proto_context_lock);
   2195
   2196	ctx = i915_gem_create_context(file_priv->dev_priv, pc);
   2197	if (IS_ERR(ctx))
   2198		return ctx;
   2199
   2200	gem_context_register(ctx, file_priv, id);
   2201
   2202	old = xa_erase(&file_priv->proto_context_xa, id);
   2203	GEM_BUG_ON(old != pc);
   2204	proto_context_close(file_priv->dev_priv, pc);
   2205
   2206	/* One for the xarray and one for the caller */
   2207	return i915_gem_context_get(ctx);
   2208}
   2209
   2210struct i915_gem_context *
   2211i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
   2212{
   2213	struct i915_gem_proto_context *pc;
   2214	struct i915_gem_context *ctx;
   2215
   2216	ctx = __context_lookup(file_priv, id);
   2217	if (ctx)
   2218		return ctx;
   2219
   2220	mutex_lock(&file_priv->proto_context_lock);
   2221	/* Try one more time under the lock */
   2222	ctx = __context_lookup(file_priv, id);
   2223	if (!ctx) {
   2224		pc = xa_load(&file_priv->proto_context_xa, id);
   2225		if (!pc)
   2226			ctx = ERR_PTR(-ENOENT);
   2227		else
   2228			ctx = finalize_create_context_locked(file_priv, pc, id);
   2229	}
   2230	mutex_unlock(&file_priv->proto_context_lock);
   2231
   2232	return ctx;
   2233}
   2234
   2235int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
   2236				  struct drm_file *file)
   2237{
   2238	struct drm_i915_private *i915 = to_i915(dev);
   2239	struct drm_i915_gem_context_create_ext *args = data;
   2240	struct create_ext ext_data;
   2241	int ret;
   2242	u32 id;
   2243
   2244	if (!DRIVER_CAPS(i915)->has_logical_contexts)
   2245		return -ENODEV;
   2246
   2247	if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN)
   2248		return -EINVAL;
   2249
   2250	ret = intel_gt_terminally_wedged(to_gt(i915));
   2251	if (ret)
   2252		return ret;
   2253
   2254	ext_data.fpriv = file->driver_priv;
   2255	if (client_is_banned(ext_data.fpriv)) {
   2256		drm_dbg(&i915->drm,
   2257			"client %s[%d] banned from creating ctx\n",
   2258			current->comm, task_pid_nr(current));
   2259		return -EIO;
   2260	}
   2261
   2262	ext_data.pc = proto_context_create(i915, args->flags);
   2263	if (IS_ERR(ext_data.pc))
   2264		return PTR_ERR(ext_data.pc);
   2265
   2266	if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) {
   2267		ret = i915_user_extensions(u64_to_user_ptr(args->extensions),
   2268					   create_extensions,
   2269					   ARRAY_SIZE(create_extensions),
   2270					   &ext_data);
   2271		if (ret)
   2272			goto err_pc;
   2273	}
   2274
   2275	if (GRAPHICS_VER(i915) > 12) {
   2276		struct i915_gem_context *ctx;
   2277
   2278		/* Get ourselves a context ID */
   2279		ret = xa_alloc(&ext_data.fpriv->context_xa, &id, NULL,
   2280			       xa_limit_32b, GFP_KERNEL);
   2281		if (ret)
   2282			goto err_pc;
   2283
   2284		ctx = i915_gem_create_context(i915, ext_data.pc);
   2285		if (IS_ERR(ctx)) {
   2286			ret = PTR_ERR(ctx);
   2287			goto err_pc;
   2288		}
   2289
   2290		proto_context_close(i915, ext_data.pc);
   2291		gem_context_register(ctx, ext_data.fpriv, id);
   2292	} else {
   2293		ret = proto_context_register(ext_data.fpriv, ext_data.pc, &id);
   2294		if (ret < 0)
   2295			goto err_pc;
   2296	}
   2297
   2298	args->ctx_id = id;
   2299	drm_dbg(&i915->drm, "HW context %d created\n", args->ctx_id);
   2300
   2301	return 0;
   2302
   2303err_pc:
   2304	proto_context_close(i915, ext_data.pc);
   2305	return ret;
   2306}
   2307
   2308int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
   2309				   struct drm_file *file)
   2310{
   2311	struct drm_i915_gem_context_destroy *args = data;
   2312	struct drm_i915_file_private *file_priv = file->driver_priv;
   2313	struct i915_gem_proto_context *pc;
   2314	struct i915_gem_context *ctx;
   2315
   2316	if (args->pad != 0)
   2317		return -EINVAL;
   2318
   2319	if (!args->ctx_id)
   2320		return -ENOENT;
   2321
   2322	/* We need to hold the proto-context lock here to prevent races
   2323	 * with finalize_create_context_locked().
   2324	 */
   2325	mutex_lock(&file_priv->proto_context_lock);
   2326	ctx = xa_erase(&file_priv->context_xa, args->ctx_id);
   2327	pc = xa_erase(&file_priv->proto_context_xa, args->ctx_id);
   2328	mutex_unlock(&file_priv->proto_context_lock);
   2329
   2330	if (!ctx && !pc)
   2331		return -ENOENT;
   2332	GEM_WARN_ON(ctx && pc);
   2333
   2334	if (pc)
   2335		proto_context_close(file_priv->dev_priv, pc);
   2336
   2337	if (ctx)
   2338		context_close(ctx);
   2339
   2340	return 0;
   2341}
   2342
   2343static int get_sseu(struct i915_gem_context *ctx,
   2344		    struct drm_i915_gem_context_param *args)
   2345{
   2346	struct drm_i915_gem_context_param_sseu user_sseu;
   2347	struct intel_context *ce;
   2348	unsigned long lookup;
   2349	int err;
   2350
   2351	if (args->size == 0)
   2352		goto out;
   2353	else if (args->size < sizeof(user_sseu))
   2354		return -EINVAL;
   2355
   2356	if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value),
   2357			   sizeof(user_sseu)))
   2358		return -EFAULT;
   2359
   2360	if (user_sseu.rsvd)
   2361		return -EINVAL;
   2362
   2363	if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX))
   2364		return -EINVAL;
   2365
   2366	lookup = 0;
   2367	if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)
   2368		lookup |= LOOKUP_USER_INDEX;
   2369
   2370	ce = lookup_user_engine(ctx, lookup, &user_sseu.engine);
   2371	if (IS_ERR(ce))
   2372		return PTR_ERR(ce);
   2373
   2374	err = intel_context_lock_pinned(ce); /* serialises with set_sseu */
   2375	if (err) {
   2376		intel_context_put(ce);
   2377		return err;
   2378	}
   2379
   2380	user_sseu.slice_mask = ce->sseu.slice_mask;
   2381	user_sseu.subslice_mask = ce->sseu.subslice_mask;
   2382	user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice;
   2383	user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice;
   2384
   2385	intel_context_unlock_pinned(ce);
   2386	intel_context_put(ce);
   2387
   2388	if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu,
   2389			 sizeof(user_sseu)))
   2390		return -EFAULT;
   2391
   2392out:
   2393	args->size = sizeof(user_sseu);
   2394
   2395	return 0;
   2396}
   2397
   2398int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
   2399				    struct drm_file *file)
   2400{
   2401	struct drm_i915_file_private *file_priv = file->driver_priv;
   2402	struct drm_i915_gem_context_param *args = data;
   2403	struct i915_gem_context *ctx;
   2404	struct i915_address_space *vm;
   2405	int ret = 0;
   2406
   2407	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
   2408	if (IS_ERR(ctx))
   2409		return PTR_ERR(ctx);
   2410
   2411	switch (args->param) {
   2412	case I915_CONTEXT_PARAM_GTT_SIZE:
   2413		args->size = 0;
   2414		vm = i915_gem_context_get_eb_vm(ctx);
   2415		args->value = vm->total;
   2416		i915_vm_put(vm);
   2417
   2418		break;
   2419
   2420	case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
   2421		args->size = 0;
   2422		args->value = i915_gem_context_no_error_capture(ctx);
   2423		break;
   2424
   2425	case I915_CONTEXT_PARAM_BANNABLE:
   2426		args->size = 0;
   2427		args->value = i915_gem_context_is_bannable(ctx);
   2428		break;
   2429
   2430	case I915_CONTEXT_PARAM_RECOVERABLE:
   2431		args->size = 0;
   2432		args->value = i915_gem_context_is_recoverable(ctx);
   2433		break;
   2434
   2435	case I915_CONTEXT_PARAM_PRIORITY:
   2436		args->size = 0;
   2437		args->value = ctx->sched.priority;
   2438		break;
   2439
   2440	case I915_CONTEXT_PARAM_SSEU:
   2441		ret = get_sseu(ctx, args);
   2442		break;
   2443
   2444	case I915_CONTEXT_PARAM_VM:
   2445		ret = get_ppgtt(file_priv, ctx, args);
   2446		break;
   2447
   2448	case I915_CONTEXT_PARAM_PERSISTENCE:
   2449		args->size = 0;
   2450		args->value = i915_gem_context_is_persistent(ctx);
   2451		break;
   2452
   2453	case I915_CONTEXT_PARAM_PROTECTED_CONTENT:
   2454		ret = get_protected(ctx, args);
   2455		break;
   2456
   2457	case I915_CONTEXT_PARAM_NO_ZEROMAP:
   2458	case I915_CONTEXT_PARAM_BAN_PERIOD:
   2459	case I915_CONTEXT_PARAM_ENGINES:
   2460	case I915_CONTEXT_PARAM_RINGSIZE:
   2461	default:
   2462		ret = -EINVAL;
   2463		break;
   2464	}
   2465
   2466	i915_gem_context_put(ctx);
   2467	return ret;
   2468}
   2469
   2470int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
   2471				    struct drm_file *file)
   2472{
   2473	struct drm_i915_file_private *file_priv = file->driver_priv;
   2474	struct drm_i915_gem_context_param *args = data;
   2475	struct i915_gem_proto_context *pc;
   2476	struct i915_gem_context *ctx;
   2477	int ret = 0;
   2478
   2479	mutex_lock(&file_priv->proto_context_lock);
   2480	ctx = __context_lookup(file_priv, args->ctx_id);
   2481	if (!ctx) {
   2482		pc = xa_load(&file_priv->proto_context_xa, args->ctx_id);
   2483		if (pc) {
   2484			/* Contexts should be finalized inside
   2485			 * GEM_CONTEXT_CREATE starting with graphics
   2486			 * version 13.
   2487			 */
   2488			WARN_ON(GRAPHICS_VER(file_priv->dev_priv) > 12);
   2489			ret = set_proto_ctx_param(file_priv, pc, args);
   2490		} else {
   2491			ret = -ENOENT;
   2492		}
   2493	}
   2494	mutex_unlock(&file_priv->proto_context_lock);
   2495
   2496	if (ctx) {
   2497		ret = ctx_setparam(file_priv, ctx, args);
   2498		i915_gem_context_put(ctx);
   2499	}
   2500
   2501	return ret;
   2502}
   2503
   2504int i915_gem_context_reset_stats_ioctl(struct drm_device *dev,
   2505				       void *data, struct drm_file *file)
   2506{
   2507	struct drm_i915_private *i915 = to_i915(dev);
   2508	struct drm_i915_reset_stats *args = data;
   2509	struct i915_gem_context *ctx;
   2510
   2511	if (args->flags || args->pad)
   2512		return -EINVAL;
   2513
   2514	ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id);
   2515	if (IS_ERR(ctx))
   2516		return PTR_ERR(ctx);
   2517
   2518	/*
   2519	 * We opt for unserialised reads here. This may result in tearing
   2520	 * in the extremely unlikely event of a GPU hang on this context
   2521	 * as we are querying them. If we need that extra layer of protection,
   2522	 * we should wrap the hangstats with a seqlock.
   2523	 */
   2524
   2525	if (capable(CAP_SYS_ADMIN))
   2526		args->reset_count = i915_reset_count(&i915->gpu_error);
   2527	else
   2528		args->reset_count = 0;
   2529
   2530	args->batch_active = atomic_read(&ctx->guilty_count);
   2531	args->batch_pending = atomic_read(&ctx->active_count);
   2532
   2533	i915_gem_context_put(ctx);
   2534	return 0;
   2535}
   2536
   2537/* GEM context-engines iterator: for_each_gem_engine() */
   2538struct intel_context *
   2539i915_gem_engines_iter_next(struct i915_gem_engines_iter *it)
   2540{
   2541	const struct i915_gem_engines *e = it->engines;
   2542	struct intel_context *ctx;
   2543
   2544	if (unlikely(!e))
   2545		return NULL;
   2546
   2547	do {
   2548		if (it->idx >= e->num_engines)
   2549			return NULL;
   2550
   2551		ctx = e->engines[it->idx++];
   2552	} while (!ctx);
   2553
   2554	return ctx;
   2555}
   2556
   2557#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
   2558#include "selftests/mock_context.c"
   2559#include "selftests/i915_gem_context.c"
   2560#endif
   2561
   2562void i915_gem_context_module_exit(void)
   2563{
   2564	kmem_cache_destroy(slab_luts);
   2565}
   2566
   2567int __init i915_gem_context_module_init(void)
   2568{
   2569	slab_luts = KMEM_CACHE(i915_lut_handle, 0);
   2570	if (!slab_luts)
   2571		return -ENOMEM;
   2572
   2573	return 0;
   2574}