cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

region_devs.c (31202B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
      4 */
      5#include <linux/scatterlist.h>
      6#include <linux/memregion.h>
      7#include <linux/highmem.h>
      8#include <linux/sched.h>
      9#include <linux/slab.h>
     10#include <linux/hash.h>
     11#include <linux/sort.h>
     12#include <linux/io.h>
     13#include <linux/nd.h>
     14#include "nd-core.h"
     15#include "nd.h"
     16
     17/*
     18 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
     19 * irrelevant.
     20 */
     21#include <linux/io-64-nonatomic-hi-lo.h>
     22
     23static DEFINE_PER_CPU(int, flush_idx);
     24
     25static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
     26		struct nd_region_data *ndrd)
     27{
     28	int i, j;
     29
     30	dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm),
     31			nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es");
     32	for (i = 0; i < (1 << ndrd->hints_shift); i++) {
     33		struct resource *res = &nvdimm->flush_wpq[i];
     34		unsigned long pfn = PHYS_PFN(res->start);
     35		void __iomem *flush_page;
     36
     37		/* check if flush hints share a page */
     38		for (j = 0; j < i; j++) {
     39			struct resource *res_j = &nvdimm->flush_wpq[j];
     40			unsigned long pfn_j = PHYS_PFN(res_j->start);
     41
     42			if (pfn == pfn_j)
     43				break;
     44		}
     45
     46		if (j < i)
     47			flush_page = (void __iomem *) ((unsigned long)
     48					ndrd_get_flush_wpq(ndrd, dimm, j)
     49					& PAGE_MASK);
     50		else
     51			flush_page = devm_nvdimm_ioremap(dev,
     52					PFN_PHYS(pfn), PAGE_SIZE);
     53		if (!flush_page)
     54			return -ENXIO;
     55		ndrd_set_flush_wpq(ndrd, dimm, i, flush_page
     56				+ (res->start & ~PAGE_MASK));
     57	}
     58
     59	return 0;
     60}
     61
     62int nd_region_activate(struct nd_region *nd_region)
     63{
     64	int i, j, num_flush = 0;
     65	struct nd_region_data *ndrd;
     66	struct device *dev = &nd_region->dev;
     67	size_t flush_data_size = sizeof(void *);
     68
     69	nvdimm_bus_lock(&nd_region->dev);
     70	for (i = 0; i < nd_region->ndr_mappings; i++) {
     71		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
     72		struct nvdimm *nvdimm = nd_mapping->nvdimm;
     73
     74		if (test_bit(NDD_SECURITY_OVERWRITE, &nvdimm->flags)) {
     75			nvdimm_bus_unlock(&nd_region->dev);
     76			return -EBUSY;
     77		}
     78
     79		/* at least one null hint slot per-dimm for the "no-hint" case */
     80		flush_data_size += sizeof(void *);
     81		num_flush = min_not_zero(num_flush, nvdimm->num_flush);
     82		if (!nvdimm->num_flush)
     83			continue;
     84		flush_data_size += nvdimm->num_flush * sizeof(void *);
     85	}
     86	nvdimm_bus_unlock(&nd_region->dev);
     87
     88	ndrd = devm_kzalloc(dev, sizeof(*ndrd) + flush_data_size, GFP_KERNEL);
     89	if (!ndrd)
     90		return -ENOMEM;
     91	dev_set_drvdata(dev, ndrd);
     92
     93	if (!num_flush)
     94		return 0;
     95
     96	ndrd->hints_shift = ilog2(num_flush);
     97	for (i = 0; i < nd_region->ndr_mappings; i++) {
     98		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
     99		struct nvdimm *nvdimm = nd_mapping->nvdimm;
    100		int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd);
    101
    102		if (rc)
    103			return rc;
    104	}
    105
    106	/*
    107	 * Clear out entries that are duplicates. This should prevent the
    108	 * extra flushings.
    109	 */
    110	for (i = 0; i < nd_region->ndr_mappings - 1; i++) {
    111		/* ignore if NULL already */
    112		if (!ndrd_get_flush_wpq(ndrd, i, 0))
    113			continue;
    114
    115		for (j = i + 1; j < nd_region->ndr_mappings; j++)
    116			if (ndrd_get_flush_wpq(ndrd, i, 0) ==
    117			    ndrd_get_flush_wpq(ndrd, j, 0))
    118				ndrd_set_flush_wpq(ndrd, j, 0, NULL);
    119	}
    120
    121	return 0;
    122}
    123
    124static void nd_region_release(struct device *dev)
    125{
    126	struct nd_region *nd_region = to_nd_region(dev);
    127	u16 i;
    128
    129	for (i = 0; i < nd_region->ndr_mappings; i++) {
    130		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
    131		struct nvdimm *nvdimm = nd_mapping->nvdimm;
    132
    133		put_device(&nvdimm->dev);
    134	}
    135	free_percpu(nd_region->lane);
    136	memregion_free(nd_region->id);
    137	kfree(nd_region);
    138}
    139
    140struct nd_region *to_nd_region(struct device *dev)
    141{
    142	struct nd_region *nd_region = container_of(dev, struct nd_region, dev);
    143
    144	WARN_ON(dev->type->release != nd_region_release);
    145	return nd_region;
    146}
    147EXPORT_SYMBOL_GPL(to_nd_region);
    148
    149struct device *nd_region_dev(struct nd_region *nd_region)
    150{
    151	if (!nd_region)
    152		return NULL;
    153	return &nd_region->dev;
    154}
    155EXPORT_SYMBOL_GPL(nd_region_dev);
    156
    157void *nd_region_provider_data(struct nd_region *nd_region)
    158{
    159	return nd_region->provider_data;
    160}
    161EXPORT_SYMBOL_GPL(nd_region_provider_data);
    162
    163/**
    164 * nd_region_to_nstype() - region to an integer namespace type
    165 * @nd_region: region-device to interrogate
    166 *
    167 * This is the 'nstype' attribute of a region as well, an input to the
    168 * MODALIAS for namespace devices, and bit number for a nvdimm_bus to match
    169 * namespace devices with namespace drivers.
    170 */
    171int nd_region_to_nstype(struct nd_region *nd_region)
    172{
    173	if (is_memory(&nd_region->dev)) {
    174		u16 i, label;
    175
    176		for (i = 0, label = 0; i < nd_region->ndr_mappings; i++) {
    177			struct nd_mapping *nd_mapping = &nd_region->mapping[i];
    178			struct nvdimm *nvdimm = nd_mapping->nvdimm;
    179
    180			if (test_bit(NDD_LABELING, &nvdimm->flags))
    181				label++;
    182		}
    183		if (label)
    184			return ND_DEVICE_NAMESPACE_PMEM;
    185		else
    186			return ND_DEVICE_NAMESPACE_IO;
    187	}
    188
    189	return 0;
    190}
    191EXPORT_SYMBOL(nd_region_to_nstype);
    192
    193static unsigned long long region_size(struct nd_region *nd_region)
    194{
    195	if (is_memory(&nd_region->dev)) {
    196		return nd_region->ndr_size;
    197	} else if (nd_region->ndr_mappings == 1) {
    198		struct nd_mapping *nd_mapping = &nd_region->mapping[0];
    199
    200		return nd_mapping->size;
    201	}
    202
    203	return 0;
    204}
    205
    206static ssize_t size_show(struct device *dev,
    207		struct device_attribute *attr, char *buf)
    208{
    209	struct nd_region *nd_region = to_nd_region(dev);
    210
    211	return sprintf(buf, "%llu\n", region_size(nd_region));
    212}
    213static DEVICE_ATTR_RO(size);
    214
    215static ssize_t deep_flush_show(struct device *dev,
    216		struct device_attribute *attr, char *buf)
    217{
    218	struct nd_region *nd_region = to_nd_region(dev);
    219
    220	/*
    221	 * NOTE: in the nvdimm_has_flush() error case this attribute is
    222	 * not visible.
    223	 */
    224	return sprintf(buf, "%d\n", nvdimm_has_flush(nd_region));
    225}
    226
    227static ssize_t deep_flush_store(struct device *dev, struct device_attribute *attr,
    228		const char *buf, size_t len)
    229{
    230	bool flush;
    231	int rc = strtobool(buf, &flush);
    232	struct nd_region *nd_region = to_nd_region(dev);
    233
    234	if (rc)
    235		return rc;
    236	if (!flush)
    237		return -EINVAL;
    238	rc = nvdimm_flush(nd_region, NULL);
    239	if (rc)
    240		return rc;
    241
    242	return len;
    243}
    244static DEVICE_ATTR_RW(deep_flush);
    245
    246static ssize_t mappings_show(struct device *dev,
    247		struct device_attribute *attr, char *buf)
    248{
    249	struct nd_region *nd_region = to_nd_region(dev);
    250
    251	return sprintf(buf, "%d\n", nd_region->ndr_mappings);
    252}
    253static DEVICE_ATTR_RO(mappings);
    254
    255static ssize_t nstype_show(struct device *dev,
    256		struct device_attribute *attr, char *buf)
    257{
    258	struct nd_region *nd_region = to_nd_region(dev);
    259
    260	return sprintf(buf, "%d\n", nd_region_to_nstype(nd_region));
    261}
    262static DEVICE_ATTR_RO(nstype);
    263
    264static ssize_t set_cookie_show(struct device *dev,
    265		struct device_attribute *attr, char *buf)
    266{
    267	struct nd_region *nd_region = to_nd_region(dev);
    268	struct nd_interleave_set *nd_set = nd_region->nd_set;
    269	ssize_t rc = 0;
    270
    271	if (is_memory(dev) && nd_set)
    272		/* pass, should be precluded by region_visible */;
    273	else
    274		return -ENXIO;
    275
    276	/*
    277	 * The cookie to show depends on which specification of the
    278	 * labels we are using. If there are not labels then default to
    279	 * the v1.1 namespace label cookie definition. To read all this
    280	 * data we need to wait for probing to settle.
    281	 */
    282	device_lock(dev);
    283	nvdimm_bus_lock(dev);
    284	wait_nvdimm_bus_probe_idle(dev);
    285	if (nd_region->ndr_mappings) {
    286		struct nd_mapping *nd_mapping = &nd_region->mapping[0];
    287		struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
    288
    289		if (ndd) {
    290			struct nd_namespace_index *nsindex;
    291
    292			nsindex = to_namespace_index(ndd, ndd->ns_current);
    293			rc = sprintf(buf, "%#llx\n",
    294					nd_region_interleave_set_cookie(nd_region,
    295						nsindex));
    296		}
    297	}
    298	nvdimm_bus_unlock(dev);
    299	device_unlock(dev);
    300
    301	if (rc)
    302		return rc;
    303	return sprintf(buf, "%#llx\n", nd_set->cookie1);
    304}
    305static DEVICE_ATTR_RO(set_cookie);
    306
    307resource_size_t nd_region_available_dpa(struct nd_region *nd_region)
    308{
    309	resource_size_t available;
    310	int i;
    311
    312	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));
    313
    314	available = 0;
    315	for (i = 0; i < nd_region->ndr_mappings; i++) {
    316		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
    317		struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
    318
    319		/* if a dimm is disabled the available capacity is zero */
    320		if (!ndd)
    321			return 0;
    322
    323		available += nd_pmem_available_dpa(nd_region, nd_mapping);
    324	}
    325
    326	return available;
    327}
    328
    329resource_size_t nd_region_allocatable_dpa(struct nd_region *nd_region)
    330{
    331	resource_size_t avail = 0;
    332	int i;
    333
    334	WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));
    335	for (i = 0; i < nd_region->ndr_mappings; i++) {
    336		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
    337
    338		avail = min_not_zero(avail, nd_pmem_max_contiguous_dpa(
    339						    nd_region, nd_mapping));
    340	}
    341	return avail * nd_region->ndr_mappings;
    342}
    343
    344static ssize_t available_size_show(struct device *dev,
    345		struct device_attribute *attr, char *buf)
    346{
    347	struct nd_region *nd_region = to_nd_region(dev);
    348	unsigned long long available = 0;
    349
    350	/*
    351	 * Flush in-flight updates and grab a snapshot of the available
    352	 * size.  Of course, this value is potentially invalidated the
    353	 * memory nvdimm_bus_lock() is dropped, but that's userspace's
    354	 * problem to not race itself.
    355	 */
    356	device_lock(dev);
    357	nvdimm_bus_lock(dev);
    358	wait_nvdimm_bus_probe_idle(dev);
    359	available = nd_region_available_dpa(nd_region);
    360	nvdimm_bus_unlock(dev);
    361	device_unlock(dev);
    362
    363	return sprintf(buf, "%llu\n", available);
    364}
    365static DEVICE_ATTR_RO(available_size);
    366
    367static ssize_t max_available_extent_show(struct device *dev,
    368		struct device_attribute *attr, char *buf)
    369{
    370	struct nd_region *nd_region = to_nd_region(dev);
    371	unsigned long long available = 0;
    372
    373	device_lock(dev);
    374	nvdimm_bus_lock(dev);
    375	wait_nvdimm_bus_probe_idle(dev);
    376	available = nd_region_allocatable_dpa(nd_region);
    377	nvdimm_bus_unlock(dev);
    378	device_unlock(dev);
    379
    380	return sprintf(buf, "%llu\n", available);
    381}
    382static DEVICE_ATTR_RO(max_available_extent);
    383
    384static ssize_t init_namespaces_show(struct device *dev,
    385		struct device_attribute *attr, char *buf)
    386{
    387	struct nd_region_data *ndrd = dev_get_drvdata(dev);
    388	ssize_t rc;
    389
    390	nvdimm_bus_lock(dev);
    391	if (ndrd)
    392		rc = sprintf(buf, "%d/%d\n", ndrd->ns_active, ndrd->ns_count);
    393	else
    394		rc = -ENXIO;
    395	nvdimm_bus_unlock(dev);
    396
    397	return rc;
    398}
    399static DEVICE_ATTR_RO(init_namespaces);
    400
    401static ssize_t namespace_seed_show(struct device *dev,
    402		struct device_attribute *attr, char *buf)
    403{
    404	struct nd_region *nd_region = to_nd_region(dev);
    405	ssize_t rc;
    406
    407	nvdimm_bus_lock(dev);
    408	if (nd_region->ns_seed)
    409		rc = sprintf(buf, "%s\n", dev_name(nd_region->ns_seed));
    410	else
    411		rc = sprintf(buf, "\n");
    412	nvdimm_bus_unlock(dev);
    413	return rc;
    414}
    415static DEVICE_ATTR_RO(namespace_seed);
    416
    417static ssize_t btt_seed_show(struct device *dev,
    418		struct device_attribute *attr, char *buf)
    419{
    420	struct nd_region *nd_region = to_nd_region(dev);
    421	ssize_t rc;
    422
    423	nvdimm_bus_lock(dev);
    424	if (nd_region->btt_seed)
    425		rc = sprintf(buf, "%s\n", dev_name(nd_region->btt_seed));
    426	else
    427		rc = sprintf(buf, "\n");
    428	nvdimm_bus_unlock(dev);
    429
    430	return rc;
    431}
    432static DEVICE_ATTR_RO(btt_seed);
    433
    434static ssize_t pfn_seed_show(struct device *dev,
    435		struct device_attribute *attr, char *buf)
    436{
    437	struct nd_region *nd_region = to_nd_region(dev);
    438	ssize_t rc;
    439
    440	nvdimm_bus_lock(dev);
    441	if (nd_region->pfn_seed)
    442		rc = sprintf(buf, "%s\n", dev_name(nd_region->pfn_seed));
    443	else
    444		rc = sprintf(buf, "\n");
    445	nvdimm_bus_unlock(dev);
    446
    447	return rc;
    448}
    449static DEVICE_ATTR_RO(pfn_seed);
    450
    451static ssize_t dax_seed_show(struct device *dev,
    452		struct device_attribute *attr, char *buf)
    453{
    454	struct nd_region *nd_region = to_nd_region(dev);
    455	ssize_t rc;
    456
    457	nvdimm_bus_lock(dev);
    458	if (nd_region->dax_seed)
    459		rc = sprintf(buf, "%s\n", dev_name(nd_region->dax_seed));
    460	else
    461		rc = sprintf(buf, "\n");
    462	nvdimm_bus_unlock(dev);
    463
    464	return rc;
    465}
    466static DEVICE_ATTR_RO(dax_seed);
    467
    468static ssize_t read_only_show(struct device *dev,
    469		struct device_attribute *attr, char *buf)
    470{
    471	struct nd_region *nd_region = to_nd_region(dev);
    472
    473	return sprintf(buf, "%d\n", nd_region->ro);
    474}
    475
    476static int revalidate_read_only(struct device *dev, void *data)
    477{
    478	nd_device_notify(dev, NVDIMM_REVALIDATE_REGION);
    479	return 0;
    480}
    481
    482static ssize_t read_only_store(struct device *dev,
    483		struct device_attribute *attr, const char *buf, size_t len)
    484{
    485	bool ro;
    486	int rc = strtobool(buf, &ro);
    487	struct nd_region *nd_region = to_nd_region(dev);
    488
    489	if (rc)
    490		return rc;
    491
    492	nd_region->ro = ro;
    493	device_for_each_child(dev, NULL, revalidate_read_only);
    494	return len;
    495}
    496static DEVICE_ATTR_RW(read_only);
    497
    498static ssize_t align_show(struct device *dev,
    499		struct device_attribute *attr, char *buf)
    500{
    501	struct nd_region *nd_region = to_nd_region(dev);
    502
    503	return sprintf(buf, "%#lx\n", nd_region->align);
    504}
    505
    506static ssize_t align_store(struct device *dev,
    507		struct device_attribute *attr, const char *buf, size_t len)
    508{
    509	struct nd_region *nd_region = to_nd_region(dev);
    510	unsigned long val, dpa;
    511	u32 remainder;
    512	int rc;
    513
    514	rc = kstrtoul(buf, 0, &val);
    515	if (rc)
    516		return rc;
    517
    518	if (!nd_region->ndr_mappings)
    519		return -ENXIO;
    520
    521	/*
    522	 * Ensure space-align is evenly divisible by the region
    523	 * interleave-width because the kernel typically has no facility
    524	 * to determine which DIMM(s), dimm-physical-addresses, would
    525	 * contribute to the tail capacity in system-physical-address
    526	 * space for the namespace.
    527	 */
    528	dpa = div_u64_rem(val, nd_region->ndr_mappings, &remainder);
    529	if (!is_power_of_2(dpa) || dpa < PAGE_SIZE
    530			|| val > region_size(nd_region) || remainder)
    531		return -EINVAL;
    532
    533	/*
    534	 * Given that space allocation consults this value multiple
    535	 * times ensure it does not change for the duration of the
    536	 * allocation.
    537	 */
    538	nvdimm_bus_lock(dev);
    539	nd_region->align = val;
    540	nvdimm_bus_unlock(dev);
    541
    542	return len;
    543}
    544static DEVICE_ATTR_RW(align);
    545
    546static ssize_t region_badblocks_show(struct device *dev,
    547		struct device_attribute *attr, char *buf)
    548{
    549	struct nd_region *nd_region = to_nd_region(dev);
    550	ssize_t rc;
    551
    552	device_lock(dev);
    553	if (dev->driver)
    554		rc = badblocks_show(&nd_region->bb, buf, 0);
    555	else
    556		rc = -ENXIO;
    557	device_unlock(dev);
    558
    559	return rc;
    560}
    561static DEVICE_ATTR(badblocks, 0444, region_badblocks_show, NULL);
    562
    563static ssize_t resource_show(struct device *dev,
    564		struct device_attribute *attr, char *buf)
    565{
    566	struct nd_region *nd_region = to_nd_region(dev);
    567
    568	return sprintf(buf, "%#llx\n", nd_region->ndr_start);
    569}
    570static DEVICE_ATTR_ADMIN_RO(resource);
    571
    572static ssize_t persistence_domain_show(struct device *dev,
    573		struct device_attribute *attr, char *buf)
    574{
    575	struct nd_region *nd_region = to_nd_region(dev);
    576
    577	if (test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags))
    578		return sprintf(buf, "cpu_cache\n");
    579	else if (test_bit(ND_REGION_PERSIST_MEMCTRL, &nd_region->flags))
    580		return sprintf(buf, "memory_controller\n");
    581	else
    582		return sprintf(buf, "\n");
    583}
    584static DEVICE_ATTR_RO(persistence_domain);
    585
    586static struct attribute *nd_region_attributes[] = {
    587	&dev_attr_size.attr,
    588	&dev_attr_align.attr,
    589	&dev_attr_nstype.attr,
    590	&dev_attr_mappings.attr,
    591	&dev_attr_btt_seed.attr,
    592	&dev_attr_pfn_seed.attr,
    593	&dev_attr_dax_seed.attr,
    594	&dev_attr_deep_flush.attr,
    595	&dev_attr_read_only.attr,
    596	&dev_attr_set_cookie.attr,
    597	&dev_attr_available_size.attr,
    598	&dev_attr_max_available_extent.attr,
    599	&dev_attr_namespace_seed.attr,
    600	&dev_attr_init_namespaces.attr,
    601	&dev_attr_badblocks.attr,
    602	&dev_attr_resource.attr,
    603	&dev_attr_persistence_domain.attr,
    604	NULL,
    605};
    606
    607static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n)
    608{
    609	struct device *dev = container_of(kobj, typeof(*dev), kobj);
    610	struct nd_region *nd_region = to_nd_region(dev);
    611	struct nd_interleave_set *nd_set = nd_region->nd_set;
    612	int type = nd_region_to_nstype(nd_region);
    613
    614	if (!is_memory(dev) && a == &dev_attr_pfn_seed.attr)
    615		return 0;
    616
    617	if (!is_memory(dev) && a == &dev_attr_dax_seed.attr)
    618		return 0;
    619
    620	if (!is_memory(dev) && a == &dev_attr_badblocks.attr)
    621		return 0;
    622
    623	if (a == &dev_attr_resource.attr && !is_memory(dev))
    624		return 0;
    625
    626	if (a == &dev_attr_deep_flush.attr) {
    627		int has_flush = nvdimm_has_flush(nd_region);
    628
    629		if (has_flush == 1)
    630			return a->mode;
    631		else if (has_flush == 0)
    632			return 0444;
    633		else
    634			return 0;
    635	}
    636
    637	if (a == &dev_attr_persistence_domain.attr) {
    638		if ((nd_region->flags & (BIT(ND_REGION_PERSIST_CACHE)
    639					| BIT(ND_REGION_PERSIST_MEMCTRL))) == 0)
    640			return 0;
    641		return a->mode;
    642	}
    643
    644	if (a == &dev_attr_align.attr)
    645		return a->mode;
    646
    647	if (a != &dev_attr_set_cookie.attr
    648			&& a != &dev_attr_available_size.attr)
    649		return a->mode;
    650
    651	if (type == ND_DEVICE_NAMESPACE_PMEM &&
    652	    a == &dev_attr_available_size.attr)
    653		return a->mode;
    654	else if (is_memory(dev) && nd_set)
    655		return a->mode;
    656
    657	return 0;
    658}
    659
    660static ssize_t mappingN(struct device *dev, char *buf, int n)
    661{
    662	struct nd_region *nd_region = to_nd_region(dev);
    663	struct nd_mapping *nd_mapping;
    664	struct nvdimm *nvdimm;
    665
    666	if (n >= nd_region->ndr_mappings)
    667		return -ENXIO;
    668	nd_mapping = &nd_region->mapping[n];
    669	nvdimm = nd_mapping->nvdimm;
    670
    671	return sprintf(buf, "%s,%llu,%llu,%d\n", dev_name(&nvdimm->dev),
    672			nd_mapping->start, nd_mapping->size,
    673			nd_mapping->position);
    674}
    675
    676#define REGION_MAPPING(idx) \
    677static ssize_t mapping##idx##_show(struct device *dev,		\
    678		struct device_attribute *attr, char *buf)	\
    679{								\
    680	return mappingN(dev, buf, idx);				\
    681}								\
    682static DEVICE_ATTR_RO(mapping##idx)
    683
    684/*
    685 * 32 should be enough for a while, even in the presence of socket
    686 * interleave a 32-way interleave set is a degenerate case.
    687 */
    688REGION_MAPPING(0);
    689REGION_MAPPING(1);
    690REGION_MAPPING(2);
    691REGION_MAPPING(3);
    692REGION_MAPPING(4);
    693REGION_MAPPING(5);
    694REGION_MAPPING(6);
    695REGION_MAPPING(7);
    696REGION_MAPPING(8);
    697REGION_MAPPING(9);
    698REGION_MAPPING(10);
    699REGION_MAPPING(11);
    700REGION_MAPPING(12);
    701REGION_MAPPING(13);
    702REGION_MAPPING(14);
    703REGION_MAPPING(15);
    704REGION_MAPPING(16);
    705REGION_MAPPING(17);
    706REGION_MAPPING(18);
    707REGION_MAPPING(19);
    708REGION_MAPPING(20);
    709REGION_MAPPING(21);
    710REGION_MAPPING(22);
    711REGION_MAPPING(23);
    712REGION_MAPPING(24);
    713REGION_MAPPING(25);
    714REGION_MAPPING(26);
    715REGION_MAPPING(27);
    716REGION_MAPPING(28);
    717REGION_MAPPING(29);
    718REGION_MAPPING(30);
    719REGION_MAPPING(31);
    720
    721static umode_t mapping_visible(struct kobject *kobj, struct attribute *a, int n)
    722{
    723	struct device *dev = container_of(kobj, struct device, kobj);
    724	struct nd_region *nd_region = to_nd_region(dev);
    725
    726	if (n < nd_region->ndr_mappings)
    727		return a->mode;
    728	return 0;
    729}
    730
    731static struct attribute *mapping_attributes[] = {
    732	&dev_attr_mapping0.attr,
    733	&dev_attr_mapping1.attr,
    734	&dev_attr_mapping2.attr,
    735	&dev_attr_mapping3.attr,
    736	&dev_attr_mapping4.attr,
    737	&dev_attr_mapping5.attr,
    738	&dev_attr_mapping6.attr,
    739	&dev_attr_mapping7.attr,
    740	&dev_attr_mapping8.attr,
    741	&dev_attr_mapping9.attr,
    742	&dev_attr_mapping10.attr,
    743	&dev_attr_mapping11.attr,
    744	&dev_attr_mapping12.attr,
    745	&dev_attr_mapping13.attr,
    746	&dev_attr_mapping14.attr,
    747	&dev_attr_mapping15.attr,
    748	&dev_attr_mapping16.attr,
    749	&dev_attr_mapping17.attr,
    750	&dev_attr_mapping18.attr,
    751	&dev_attr_mapping19.attr,
    752	&dev_attr_mapping20.attr,
    753	&dev_attr_mapping21.attr,
    754	&dev_attr_mapping22.attr,
    755	&dev_attr_mapping23.attr,
    756	&dev_attr_mapping24.attr,
    757	&dev_attr_mapping25.attr,
    758	&dev_attr_mapping26.attr,
    759	&dev_attr_mapping27.attr,
    760	&dev_attr_mapping28.attr,
    761	&dev_attr_mapping29.attr,
    762	&dev_attr_mapping30.attr,
    763	&dev_attr_mapping31.attr,
    764	NULL,
    765};
    766
    767static const struct attribute_group nd_mapping_attribute_group = {
    768	.is_visible = mapping_visible,
    769	.attrs = mapping_attributes,
    770};
    771
    772static const struct attribute_group nd_region_attribute_group = {
    773	.attrs = nd_region_attributes,
    774	.is_visible = region_visible,
    775};
    776
    777static const struct attribute_group *nd_region_attribute_groups[] = {
    778	&nd_device_attribute_group,
    779	&nd_region_attribute_group,
    780	&nd_numa_attribute_group,
    781	&nd_mapping_attribute_group,
    782	NULL,
    783};
    784
    785static const struct device_type nd_pmem_device_type = {
    786	.name = "nd_pmem",
    787	.release = nd_region_release,
    788	.groups = nd_region_attribute_groups,
    789};
    790
    791static const struct device_type nd_volatile_device_type = {
    792	.name = "nd_volatile",
    793	.release = nd_region_release,
    794	.groups = nd_region_attribute_groups,
    795};
    796
    797bool is_nd_pmem(struct device *dev)
    798{
    799	return dev ? dev->type == &nd_pmem_device_type : false;
    800}
    801
    802bool is_nd_volatile(struct device *dev)
    803{
    804	return dev ? dev->type == &nd_volatile_device_type : false;
    805}
    806
    807u64 nd_region_interleave_set_cookie(struct nd_region *nd_region,
    808		struct nd_namespace_index *nsindex)
    809{
    810	struct nd_interleave_set *nd_set = nd_region->nd_set;
    811
    812	if (!nd_set)
    813		return 0;
    814
    815	if (nsindex && __le16_to_cpu(nsindex->major) == 1
    816			&& __le16_to_cpu(nsindex->minor) == 1)
    817		return nd_set->cookie1;
    818	return nd_set->cookie2;
    819}
    820
    821u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region)
    822{
    823	struct nd_interleave_set *nd_set = nd_region->nd_set;
    824
    825	if (nd_set)
    826		return nd_set->altcookie;
    827	return 0;
    828}
    829
    830void nd_mapping_free_labels(struct nd_mapping *nd_mapping)
    831{
    832	struct nd_label_ent *label_ent, *e;
    833
    834	lockdep_assert_held(&nd_mapping->lock);
    835	list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) {
    836		list_del(&label_ent->list);
    837		kfree(label_ent);
    838	}
    839}
    840
    841/*
    842 * When a namespace is activated create new seeds for the next
    843 * namespace, or namespace-personality to be configured.
    844 */
    845void nd_region_advance_seeds(struct nd_region *nd_region, struct device *dev)
    846{
    847	nvdimm_bus_lock(dev);
    848	if (nd_region->ns_seed == dev) {
    849		nd_region_create_ns_seed(nd_region);
    850	} else if (is_nd_btt(dev)) {
    851		struct nd_btt *nd_btt = to_nd_btt(dev);
    852
    853		if (nd_region->btt_seed == dev)
    854			nd_region_create_btt_seed(nd_region);
    855		if (nd_region->ns_seed == &nd_btt->ndns->dev)
    856			nd_region_create_ns_seed(nd_region);
    857	} else if (is_nd_pfn(dev)) {
    858		struct nd_pfn *nd_pfn = to_nd_pfn(dev);
    859
    860		if (nd_region->pfn_seed == dev)
    861			nd_region_create_pfn_seed(nd_region);
    862		if (nd_region->ns_seed == &nd_pfn->ndns->dev)
    863			nd_region_create_ns_seed(nd_region);
    864	} else if (is_nd_dax(dev)) {
    865		struct nd_dax *nd_dax = to_nd_dax(dev);
    866
    867		if (nd_region->dax_seed == dev)
    868			nd_region_create_dax_seed(nd_region);
    869		if (nd_region->ns_seed == &nd_dax->nd_pfn.ndns->dev)
    870			nd_region_create_ns_seed(nd_region);
    871	}
    872	nvdimm_bus_unlock(dev);
    873}
    874
    875/**
    876 * nd_region_acquire_lane - allocate and lock a lane
    877 * @nd_region: region id and number of lanes possible
    878 *
    879 * A lane correlates to a BLK-data-window and/or a log slot in the BTT.
    880 * We optimize for the common case where there are 256 lanes, one
    881 * per-cpu.  For larger systems we need to lock to share lanes.  For now
    882 * this implementation assumes the cost of maintaining an allocator for
    883 * free lanes is on the order of the lock hold time, so it implements a
    884 * static lane = cpu % num_lanes mapping.
    885 *
    886 * In the case of a BTT instance on top of a BLK namespace a lane may be
    887 * acquired recursively.  We lock on the first instance.
    888 *
    889 * In the case of a BTT instance on top of PMEM, we only acquire a lane
    890 * for the BTT metadata updates.
    891 */
    892unsigned int nd_region_acquire_lane(struct nd_region *nd_region)
    893{
    894	unsigned int cpu, lane;
    895
    896	cpu = get_cpu();
    897	if (nd_region->num_lanes < nr_cpu_ids) {
    898		struct nd_percpu_lane *ndl_lock, *ndl_count;
    899
    900		lane = cpu % nd_region->num_lanes;
    901		ndl_count = per_cpu_ptr(nd_region->lane, cpu);
    902		ndl_lock = per_cpu_ptr(nd_region->lane, lane);
    903		if (ndl_count->count++ == 0)
    904			spin_lock(&ndl_lock->lock);
    905	} else
    906		lane = cpu;
    907
    908	return lane;
    909}
    910EXPORT_SYMBOL(nd_region_acquire_lane);
    911
    912void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane)
    913{
    914	if (nd_region->num_lanes < nr_cpu_ids) {
    915		unsigned int cpu = get_cpu();
    916		struct nd_percpu_lane *ndl_lock, *ndl_count;
    917
    918		ndl_count = per_cpu_ptr(nd_region->lane, cpu);
    919		ndl_lock = per_cpu_ptr(nd_region->lane, lane);
    920		if (--ndl_count->count == 0)
    921			spin_unlock(&ndl_lock->lock);
    922		put_cpu();
    923	}
    924	put_cpu();
    925}
    926EXPORT_SYMBOL(nd_region_release_lane);
    927
    928/*
    929 * PowerPC requires this alignment for memremap_pages(). All other archs
    930 * should be ok with SUBSECTION_SIZE (see memremap_compat_align()).
    931 */
    932#define MEMREMAP_COMPAT_ALIGN_MAX SZ_16M
    933
    934static unsigned long default_align(struct nd_region *nd_region)
    935{
    936	unsigned long align;
    937	u32 remainder;
    938	int mappings;
    939
    940	align = MEMREMAP_COMPAT_ALIGN_MAX;
    941	if (nd_region->ndr_size < MEMREMAP_COMPAT_ALIGN_MAX)
    942		align = PAGE_SIZE;
    943
    944	mappings = max_t(u16, 1, nd_region->ndr_mappings);
    945	div_u64_rem(align, mappings, &remainder);
    946	if (remainder)
    947		align *= mappings;
    948
    949	return align;
    950}
    951
    952static struct lock_class_key nvdimm_region_key;
    953
    954static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
    955		struct nd_region_desc *ndr_desc,
    956		const struct device_type *dev_type, const char *caller)
    957{
    958	struct nd_region *nd_region;
    959	struct device *dev;
    960	unsigned int i;
    961	int ro = 0;
    962
    963	for (i = 0; i < ndr_desc->num_mappings; i++) {
    964		struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
    965		struct nvdimm *nvdimm = mapping->nvdimm;
    966
    967		if ((mapping->start | mapping->size) % PAGE_SIZE) {
    968			dev_err(&nvdimm_bus->dev,
    969				"%s: %s mapping%d is not %ld aligned\n",
    970				caller, dev_name(&nvdimm->dev), i, PAGE_SIZE);
    971			return NULL;
    972		}
    973
    974		if (test_bit(NDD_UNARMED, &nvdimm->flags))
    975			ro = 1;
    976
    977	}
    978
    979	nd_region =
    980		kzalloc(struct_size(nd_region, mapping, ndr_desc->num_mappings),
    981			GFP_KERNEL);
    982
    983	if (!nd_region)
    984		return NULL;
    985	nd_region->id = memregion_alloc(GFP_KERNEL);
    986	if (nd_region->id < 0)
    987		goto err_id;
    988
    989	nd_region->lane = alloc_percpu(struct nd_percpu_lane);
    990	if (!nd_region->lane)
    991		goto err_percpu;
    992
    993        for (i = 0; i < nr_cpu_ids; i++) {
    994		struct nd_percpu_lane *ndl;
    995
    996		ndl = per_cpu_ptr(nd_region->lane, i);
    997		spin_lock_init(&ndl->lock);
    998		ndl->count = 0;
    999	}
   1000
   1001	for (i = 0; i < ndr_desc->num_mappings; i++) {
   1002		struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
   1003		struct nvdimm *nvdimm = mapping->nvdimm;
   1004
   1005		nd_region->mapping[i].nvdimm = nvdimm;
   1006		nd_region->mapping[i].start = mapping->start;
   1007		nd_region->mapping[i].size = mapping->size;
   1008		nd_region->mapping[i].position = mapping->position;
   1009		INIT_LIST_HEAD(&nd_region->mapping[i].labels);
   1010		mutex_init(&nd_region->mapping[i].lock);
   1011
   1012		get_device(&nvdimm->dev);
   1013	}
   1014	nd_region->ndr_mappings = ndr_desc->num_mappings;
   1015	nd_region->provider_data = ndr_desc->provider_data;
   1016	nd_region->nd_set = ndr_desc->nd_set;
   1017	nd_region->num_lanes = ndr_desc->num_lanes;
   1018	nd_region->flags = ndr_desc->flags;
   1019	nd_region->ro = ro;
   1020	nd_region->numa_node = ndr_desc->numa_node;
   1021	nd_region->target_node = ndr_desc->target_node;
   1022	ida_init(&nd_region->ns_ida);
   1023	ida_init(&nd_region->btt_ida);
   1024	ida_init(&nd_region->pfn_ida);
   1025	ida_init(&nd_region->dax_ida);
   1026	dev = &nd_region->dev;
   1027	dev_set_name(dev, "region%d", nd_region->id);
   1028	dev->parent = &nvdimm_bus->dev;
   1029	dev->type = dev_type;
   1030	dev->groups = ndr_desc->attr_groups;
   1031	dev->of_node = ndr_desc->of_node;
   1032	nd_region->ndr_size = resource_size(ndr_desc->res);
   1033	nd_region->ndr_start = ndr_desc->res->start;
   1034	nd_region->align = default_align(nd_region);
   1035	if (ndr_desc->flush)
   1036		nd_region->flush = ndr_desc->flush;
   1037	else
   1038		nd_region->flush = NULL;
   1039
   1040	device_initialize(dev);
   1041	lockdep_set_class(&dev->mutex, &nvdimm_region_key);
   1042	nd_device_register(dev);
   1043
   1044	return nd_region;
   1045
   1046 err_percpu:
   1047	memregion_free(nd_region->id);
   1048 err_id:
   1049	kfree(nd_region);
   1050	return NULL;
   1051}
   1052
   1053struct nd_region *nvdimm_pmem_region_create(struct nvdimm_bus *nvdimm_bus,
   1054		struct nd_region_desc *ndr_desc)
   1055{
   1056	ndr_desc->num_lanes = ND_MAX_LANES;
   1057	return nd_region_create(nvdimm_bus, ndr_desc, &nd_pmem_device_type,
   1058			__func__);
   1059}
   1060EXPORT_SYMBOL_GPL(nvdimm_pmem_region_create);
   1061
   1062struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus,
   1063		struct nd_region_desc *ndr_desc)
   1064{
   1065	ndr_desc->num_lanes = ND_MAX_LANES;
   1066	return nd_region_create(nvdimm_bus, ndr_desc, &nd_volatile_device_type,
   1067			__func__);
   1068}
   1069EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);
   1070
   1071int nvdimm_flush(struct nd_region *nd_region, struct bio *bio)
   1072{
   1073	int rc = 0;
   1074
   1075	if (!nd_region->flush)
   1076		rc = generic_nvdimm_flush(nd_region);
   1077	else {
   1078		if (nd_region->flush(nd_region, bio))
   1079			rc = -EIO;
   1080	}
   1081
   1082	return rc;
   1083}
   1084/**
   1085 * nvdimm_flush - flush any posted write queues between the cpu and pmem media
   1086 * @nd_region: interleaved pmem region
   1087 */
   1088int generic_nvdimm_flush(struct nd_region *nd_region)
   1089{
   1090	struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);
   1091	int i, idx;
   1092
   1093	/*
   1094	 * Try to encourage some diversity in flush hint addresses
   1095	 * across cpus assuming a limited number of flush hints.
   1096	 */
   1097	idx = this_cpu_read(flush_idx);
   1098	idx = this_cpu_add_return(flush_idx, hash_32(current->pid + idx, 8));
   1099
   1100	/*
   1101	 * The pmem_wmb() is needed to 'sfence' all
   1102	 * previous writes such that they are architecturally visible for
   1103	 * the platform buffer flush. Note that we've already arranged for pmem
   1104	 * writes to avoid the cache via memcpy_flushcache().  The final
   1105	 * wmb() ensures ordering for the NVDIMM flush write.
   1106	 */
   1107	pmem_wmb();
   1108	for (i = 0; i < nd_region->ndr_mappings; i++)
   1109		if (ndrd_get_flush_wpq(ndrd, i, 0))
   1110			writeq(1, ndrd_get_flush_wpq(ndrd, i, idx));
   1111	wmb();
   1112
   1113	return 0;
   1114}
   1115EXPORT_SYMBOL_GPL(nvdimm_flush);
   1116
   1117/**
   1118 * nvdimm_has_flush - determine write flushing requirements
   1119 * @nd_region: interleaved pmem region
   1120 *
   1121 * Returns 1 if writes require flushing
   1122 * Returns 0 if writes do not require flushing
   1123 * Returns -ENXIO if flushing capability can not be determined
   1124 */
   1125int nvdimm_has_flush(struct nd_region *nd_region)
   1126{
   1127	int i;
   1128
   1129	/* no nvdimm or pmem api == flushing capability unknown */
   1130	if (nd_region->ndr_mappings == 0
   1131			|| !IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API))
   1132		return -ENXIO;
   1133
   1134	/* Test if an explicit flush function is defined */
   1135	if (test_bit(ND_REGION_ASYNC, &nd_region->flags) && nd_region->flush)
   1136		return 1;
   1137
   1138	/* Test if any flush hints for the region are available */
   1139	for (i = 0; i < nd_region->ndr_mappings; i++) {
   1140		struct nd_mapping *nd_mapping = &nd_region->mapping[i];
   1141		struct nvdimm *nvdimm = nd_mapping->nvdimm;
   1142
   1143		/* flush hints present / available */
   1144		if (nvdimm->num_flush)
   1145			return 1;
   1146	}
   1147
   1148	/*
   1149	 * The platform defines dimm devices without hints nor explicit flush,
   1150	 * assume platform persistence mechanism like ADR
   1151	 */
   1152	return 0;
   1153}
   1154EXPORT_SYMBOL_GPL(nvdimm_has_flush);
   1155
   1156int nvdimm_has_cache(struct nd_region *nd_region)
   1157{
   1158	return is_nd_pmem(&nd_region->dev) &&
   1159		!test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags);
   1160}
   1161EXPORT_SYMBOL_GPL(nvdimm_has_cache);
   1162
   1163bool is_nvdimm_sync(struct nd_region *nd_region)
   1164{
   1165	if (is_nd_volatile(&nd_region->dev))
   1166		return true;
   1167
   1168	return is_nd_pmem(&nd_region->dev) &&
   1169		!test_bit(ND_REGION_ASYNC, &nd_region->flags);
   1170}
   1171EXPORT_SYMBOL_GPL(is_nvdimm_sync);
   1172
   1173struct conflict_context {
   1174	struct nd_region *nd_region;
   1175	resource_size_t start, size;
   1176};
   1177
   1178static int region_conflict(struct device *dev, void *data)
   1179{
   1180	struct nd_region *nd_region;
   1181	struct conflict_context *ctx = data;
   1182	resource_size_t res_end, region_end, region_start;
   1183
   1184	if (!is_memory(dev))
   1185		return 0;
   1186
   1187	nd_region = to_nd_region(dev);
   1188	if (nd_region == ctx->nd_region)
   1189		return 0;
   1190
   1191	res_end = ctx->start + ctx->size;
   1192	region_start = nd_region->ndr_start;
   1193	region_end = region_start + nd_region->ndr_size;
   1194	if (ctx->start >= region_start && ctx->start < region_end)
   1195		return -EBUSY;
   1196	if (res_end > region_start && res_end <= region_end)
   1197		return -EBUSY;
   1198	return 0;
   1199}
   1200
   1201int nd_region_conflict(struct nd_region *nd_region, resource_size_t start,
   1202		resource_size_t size)
   1203{
   1204	struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
   1205	struct conflict_context ctx = {
   1206		.nd_region = nd_region,
   1207		.start = start,
   1208		.size = size,
   1209	};
   1210
   1211	return device_for_each_child(&nvdimm_bus->dev, &ctx, region_conflict);
   1212}