cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

node.c (29120B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * Basic Node interface support
      4 */
      5
      6#include <linux/module.h>
      7#include <linux/init.h>
      8#include <linux/mm.h>
      9#include <linux/memory.h>
     10#include <linux/vmstat.h>
     11#include <linux/notifier.h>
     12#include <linux/node.h>
     13#include <linux/hugetlb.h>
     14#include <linux/compaction.h>
     15#include <linux/cpumask.h>
     16#include <linux/topology.h>
     17#include <linux/nodemask.h>
     18#include <linux/cpu.h>
     19#include <linux/device.h>
     20#include <linux/pm_runtime.h>
     21#include <linux/swap.h>
     22#include <linux/slab.h>
     23
     24static struct bus_type node_subsys = {
     25	.name = "node",
     26	.dev_name = "node",
     27};
     28
     29static inline ssize_t cpumap_read(struct file *file, struct kobject *kobj,
     30				  struct bin_attribute *attr, char *buf,
     31				  loff_t off, size_t count)
     32{
     33	struct device *dev = kobj_to_dev(kobj);
     34	struct node *node_dev = to_node(dev);
     35	cpumask_var_t mask;
     36	ssize_t n;
     37
     38	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
     39		return 0;
     40
     41	cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
     42	n = cpumap_print_bitmask_to_buf(buf, mask, off, count);
     43	free_cpumask_var(mask);
     44
     45	return n;
     46}
     47
     48static BIN_ATTR_RO(cpumap, 0);
     49
     50static inline ssize_t cpulist_read(struct file *file, struct kobject *kobj,
     51				   struct bin_attribute *attr, char *buf,
     52				   loff_t off, size_t count)
     53{
     54	struct device *dev = kobj_to_dev(kobj);
     55	struct node *node_dev = to_node(dev);
     56	cpumask_var_t mask;
     57	ssize_t n;
     58
     59	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
     60		return 0;
     61
     62	cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
     63	n = cpumap_print_list_to_buf(buf, mask, off, count);
     64	free_cpumask_var(mask);
     65
     66	return n;
     67}
     68
     69static BIN_ATTR_RO(cpulist, 0);
     70
     71/**
     72 * struct node_access_nodes - Access class device to hold user visible
     73 * 			      relationships to other nodes.
     74 * @dev:	Device for this memory access class
     75 * @list_node:	List element in the node's access list
     76 * @access:	The access class rank
     77 * @hmem_attrs: Heterogeneous memory performance attributes
     78 */
     79struct node_access_nodes {
     80	struct device		dev;
     81	struct list_head	list_node;
     82	unsigned int		access;
     83#ifdef CONFIG_HMEM_REPORTING
     84	struct node_hmem_attrs	hmem_attrs;
     85#endif
     86};
     87#define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
     88
     89static struct attribute *node_init_access_node_attrs[] = {
     90	NULL,
     91};
     92
     93static struct attribute *node_targ_access_node_attrs[] = {
     94	NULL,
     95};
     96
     97static const struct attribute_group initiators = {
     98	.name	= "initiators",
     99	.attrs	= node_init_access_node_attrs,
    100};
    101
    102static const struct attribute_group targets = {
    103	.name	= "targets",
    104	.attrs	= node_targ_access_node_attrs,
    105};
    106
    107static const struct attribute_group *node_access_node_groups[] = {
    108	&initiators,
    109	&targets,
    110	NULL,
    111};
    112
    113static void node_remove_accesses(struct node *node)
    114{
    115	struct node_access_nodes *c, *cnext;
    116
    117	list_for_each_entry_safe(c, cnext, &node->access_list, list_node) {
    118		list_del(&c->list_node);
    119		device_unregister(&c->dev);
    120	}
    121}
    122
    123static void node_access_release(struct device *dev)
    124{
    125	kfree(to_access_nodes(dev));
    126}
    127
    128static struct node_access_nodes *node_init_node_access(struct node *node,
    129						       unsigned int access)
    130{
    131	struct node_access_nodes *access_node;
    132	struct device *dev;
    133
    134	list_for_each_entry(access_node, &node->access_list, list_node)
    135		if (access_node->access == access)
    136			return access_node;
    137
    138	access_node = kzalloc(sizeof(*access_node), GFP_KERNEL);
    139	if (!access_node)
    140		return NULL;
    141
    142	access_node->access = access;
    143	dev = &access_node->dev;
    144	dev->parent = &node->dev;
    145	dev->release = node_access_release;
    146	dev->groups = node_access_node_groups;
    147	if (dev_set_name(dev, "access%u", access))
    148		goto free;
    149
    150	if (device_register(dev))
    151		goto free_name;
    152
    153	pm_runtime_no_callbacks(dev);
    154	list_add_tail(&access_node->list_node, &node->access_list);
    155	return access_node;
    156free_name:
    157	kfree_const(dev->kobj.name);
    158free:
    159	kfree(access_node);
    160	return NULL;
    161}
    162
    163#ifdef CONFIG_HMEM_REPORTING
    164#define ACCESS_ATTR(name)						\
    165static ssize_t name##_show(struct device *dev,				\
    166			   struct device_attribute *attr,		\
    167			   char *buf)					\
    168{									\
    169	return sysfs_emit(buf, "%u\n",					\
    170			  to_access_nodes(dev)->hmem_attrs.name);	\
    171}									\
    172static DEVICE_ATTR_RO(name)
    173
    174ACCESS_ATTR(read_bandwidth);
    175ACCESS_ATTR(read_latency);
    176ACCESS_ATTR(write_bandwidth);
    177ACCESS_ATTR(write_latency);
    178
    179static struct attribute *access_attrs[] = {
    180	&dev_attr_read_bandwidth.attr,
    181	&dev_attr_read_latency.attr,
    182	&dev_attr_write_bandwidth.attr,
    183	&dev_attr_write_latency.attr,
    184	NULL,
    185};
    186
    187/**
    188 * node_set_perf_attrs - Set the performance values for given access class
    189 * @nid: Node identifier to be set
    190 * @hmem_attrs: Heterogeneous memory performance attributes
    191 * @access: The access class the for the given attributes
    192 */
    193void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs,
    194			 unsigned int access)
    195{
    196	struct node_access_nodes *c;
    197	struct node *node;
    198	int i;
    199
    200	if (WARN_ON_ONCE(!node_online(nid)))
    201		return;
    202
    203	node = node_devices[nid];
    204	c = node_init_node_access(node, access);
    205	if (!c)
    206		return;
    207
    208	c->hmem_attrs = *hmem_attrs;
    209	for (i = 0; access_attrs[i] != NULL; i++) {
    210		if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i],
    211					    "initiators")) {
    212			pr_info("failed to add performance attribute to node %d\n",
    213				nid);
    214			break;
    215		}
    216	}
    217}
    218
    219/**
    220 * struct node_cache_info - Internal tracking for memory node caches
    221 * @dev:	Device represeting the cache level
    222 * @node:	List element for tracking in the node
    223 * @cache_attrs:Attributes for this cache level
    224 */
    225struct node_cache_info {
    226	struct device dev;
    227	struct list_head node;
    228	struct node_cache_attrs cache_attrs;
    229};
    230#define to_cache_info(device) container_of(device, struct node_cache_info, dev)
    231
    232#define CACHE_ATTR(name, fmt) 						\
    233static ssize_t name##_show(struct device *dev,				\
    234			   struct device_attribute *attr,		\
    235			   char *buf)					\
    236{									\
    237	return sysfs_emit(buf, fmt "\n",				\
    238			  to_cache_info(dev)->cache_attrs.name);	\
    239}									\
    240static DEVICE_ATTR_RO(name);
    241
    242CACHE_ATTR(size, "%llu")
    243CACHE_ATTR(line_size, "%u")
    244CACHE_ATTR(indexing, "%u")
    245CACHE_ATTR(write_policy, "%u")
    246
    247static struct attribute *cache_attrs[] = {
    248	&dev_attr_indexing.attr,
    249	&dev_attr_size.attr,
    250	&dev_attr_line_size.attr,
    251	&dev_attr_write_policy.attr,
    252	NULL,
    253};
    254ATTRIBUTE_GROUPS(cache);
    255
    256static void node_cache_release(struct device *dev)
    257{
    258	kfree(dev);
    259}
    260
    261static void node_cacheinfo_release(struct device *dev)
    262{
    263	struct node_cache_info *info = to_cache_info(dev);
    264	kfree(info);
    265}
    266
    267static void node_init_cache_dev(struct node *node)
    268{
    269	struct device *dev;
    270
    271	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
    272	if (!dev)
    273		return;
    274
    275	device_initialize(dev);
    276	dev->parent = &node->dev;
    277	dev->release = node_cache_release;
    278	if (dev_set_name(dev, "memory_side_cache"))
    279		goto put_device;
    280
    281	if (device_add(dev))
    282		goto put_device;
    283
    284	pm_runtime_no_callbacks(dev);
    285	node->cache_dev = dev;
    286	return;
    287put_device:
    288	put_device(dev);
    289}
    290
    291/**
    292 * node_add_cache() - add cache attribute to a memory node
    293 * @nid: Node identifier that has new cache attributes
    294 * @cache_attrs: Attributes for the cache being added
    295 */
    296void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs)
    297{
    298	struct node_cache_info *info;
    299	struct device *dev;
    300	struct node *node;
    301
    302	if (!node_online(nid) || !node_devices[nid])
    303		return;
    304
    305	node = node_devices[nid];
    306	list_for_each_entry(info, &node->cache_attrs, node) {
    307		if (info->cache_attrs.level == cache_attrs->level) {
    308			dev_warn(&node->dev,
    309				"attempt to add duplicate cache level:%d\n",
    310				cache_attrs->level);
    311			return;
    312		}
    313	}
    314
    315	if (!node->cache_dev)
    316		node_init_cache_dev(node);
    317	if (!node->cache_dev)
    318		return;
    319
    320	info = kzalloc(sizeof(*info), GFP_KERNEL);
    321	if (!info)
    322		return;
    323
    324	dev = &info->dev;
    325	device_initialize(dev);
    326	dev->parent = node->cache_dev;
    327	dev->release = node_cacheinfo_release;
    328	dev->groups = cache_groups;
    329	if (dev_set_name(dev, "index%d", cache_attrs->level))
    330		goto put_device;
    331
    332	info->cache_attrs = *cache_attrs;
    333	if (device_add(dev)) {
    334		dev_warn(&node->dev, "failed to add cache level:%d\n",
    335			 cache_attrs->level);
    336		goto put_device;
    337	}
    338	pm_runtime_no_callbacks(dev);
    339	list_add_tail(&info->node, &node->cache_attrs);
    340	return;
    341put_device:
    342	put_device(dev);
    343}
    344
    345static void node_remove_caches(struct node *node)
    346{
    347	struct node_cache_info *info, *next;
    348
    349	if (!node->cache_dev)
    350		return;
    351
    352	list_for_each_entry_safe(info, next, &node->cache_attrs, node) {
    353		list_del(&info->node);
    354		device_unregister(&info->dev);
    355	}
    356	device_unregister(node->cache_dev);
    357}
    358
    359static void node_init_caches(unsigned int nid)
    360{
    361	INIT_LIST_HEAD(&node_devices[nid]->cache_attrs);
    362}
    363#else
    364static void node_init_caches(unsigned int nid) { }
    365static void node_remove_caches(struct node *node) { }
    366#endif
    367
    368#define K(x) ((x) << (PAGE_SHIFT - 10))
    369static ssize_t node_read_meminfo(struct device *dev,
    370			struct device_attribute *attr, char *buf)
    371{
    372	int len = 0;
    373	int nid = dev->id;
    374	struct pglist_data *pgdat = NODE_DATA(nid);
    375	struct sysinfo i;
    376	unsigned long sreclaimable, sunreclaimable;
    377	unsigned long swapcached = 0;
    378
    379	si_meminfo_node(&i, nid);
    380	sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B);
    381	sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B);
    382#ifdef CONFIG_SWAP
    383	swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE);
    384#endif
    385	len = sysfs_emit_at(buf, len,
    386			    "Node %d MemTotal:       %8lu kB\n"
    387			    "Node %d MemFree:        %8lu kB\n"
    388			    "Node %d MemUsed:        %8lu kB\n"
    389			    "Node %d SwapCached:     %8lu kB\n"
    390			    "Node %d Active:         %8lu kB\n"
    391			    "Node %d Inactive:       %8lu kB\n"
    392			    "Node %d Active(anon):   %8lu kB\n"
    393			    "Node %d Inactive(anon): %8lu kB\n"
    394			    "Node %d Active(file):   %8lu kB\n"
    395			    "Node %d Inactive(file): %8lu kB\n"
    396			    "Node %d Unevictable:    %8lu kB\n"
    397			    "Node %d Mlocked:        %8lu kB\n",
    398			    nid, K(i.totalram),
    399			    nid, K(i.freeram),
    400			    nid, K(i.totalram - i.freeram),
    401			    nid, K(swapcached),
    402			    nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) +
    403				   node_page_state(pgdat, NR_ACTIVE_FILE)),
    404			    nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) +
    405				   node_page_state(pgdat, NR_INACTIVE_FILE)),
    406			    nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)),
    407			    nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)),
    408			    nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)),
    409			    nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)),
    410			    nid, K(node_page_state(pgdat, NR_UNEVICTABLE)),
    411			    nid, K(sum_zone_node_page_state(nid, NR_MLOCK)));
    412
    413#ifdef CONFIG_HIGHMEM
    414	len += sysfs_emit_at(buf, len,
    415			     "Node %d HighTotal:      %8lu kB\n"
    416			     "Node %d HighFree:       %8lu kB\n"
    417			     "Node %d LowTotal:       %8lu kB\n"
    418			     "Node %d LowFree:        %8lu kB\n",
    419			     nid, K(i.totalhigh),
    420			     nid, K(i.freehigh),
    421			     nid, K(i.totalram - i.totalhigh),
    422			     nid, K(i.freeram - i.freehigh));
    423#endif
    424	len += sysfs_emit_at(buf, len,
    425			     "Node %d Dirty:          %8lu kB\n"
    426			     "Node %d Writeback:      %8lu kB\n"
    427			     "Node %d FilePages:      %8lu kB\n"
    428			     "Node %d Mapped:         %8lu kB\n"
    429			     "Node %d AnonPages:      %8lu kB\n"
    430			     "Node %d Shmem:          %8lu kB\n"
    431			     "Node %d KernelStack:    %8lu kB\n"
    432#ifdef CONFIG_SHADOW_CALL_STACK
    433			     "Node %d ShadowCallStack:%8lu kB\n"
    434#endif
    435			     "Node %d PageTables:     %8lu kB\n"
    436			     "Node %d NFS_Unstable:   %8lu kB\n"
    437			     "Node %d Bounce:         %8lu kB\n"
    438			     "Node %d WritebackTmp:   %8lu kB\n"
    439			     "Node %d KReclaimable:   %8lu kB\n"
    440			     "Node %d Slab:           %8lu kB\n"
    441			     "Node %d SReclaimable:   %8lu kB\n"
    442			     "Node %d SUnreclaim:     %8lu kB\n"
    443#ifdef CONFIG_TRANSPARENT_HUGEPAGE
    444			     "Node %d AnonHugePages:  %8lu kB\n"
    445			     "Node %d ShmemHugePages: %8lu kB\n"
    446			     "Node %d ShmemPmdMapped: %8lu kB\n"
    447			     "Node %d FileHugePages: %8lu kB\n"
    448			     "Node %d FilePmdMapped: %8lu kB\n"
    449#endif
    450			     ,
    451			     nid, K(node_page_state(pgdat, NR_FILE_DIRTY)),
    452			     nid, K(node_page_state(pgdat, NR_WRITEBACK)),
    453			     nid, K(node_page_state(pgdat, NR_FILE_PAGES)),
    454			     nid, K(node_page_state(pgdat, NR_FILE_MAPPED)),
    455			     nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
    456			     nid, K(i.sharedram),
    457			     nid, node_page_state(pgdat, NR_KERNEL_STACK_KB),
    458#ifdef CONFIG_SHADOW_CALL_STACK
    459			     nid, node_page_state(pgdat, NR_KERNEL_SCS_KB),
    460#endif
    461			     nid, K(node_page_state(pgdat, NR_PAGETABLE)),
    462			     nid, 0UL,
    463			     nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
    464			     nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
    465			     nid, K(sreclaimable +
    466				    node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)),
    467			     nid, K(sreclaimable + sunreclaimable),
    468			     nid, K(sreclaimable),
    469			     nid, K(sunreclaimable)
    470#ifdef CONFIG_TRANSPARENT_HUGEPAGE
    471			     ,
    472			     nid, K(node_page_state(pgdat, NR_ANON_THPS)),
    473			     nid, K(node_page_state(pgdat, NR_SHMEM_THPS)),
    474			     nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)),
    475			     nid, K(node_page_state(pgdat, NR_FILE_THPS)),
    476			     nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED))
    477#endif
    478			    );
    479	len += hugetlb_report_node_meminfo(buf, len, nid);
    480	return len;
    481}
    482
    483#undef K
    484static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL);
    485
    486static ssize_t node_read_numastat(struct device *dev,
    487				  struct device_attribute *attr, char *buf)
    488{
    489	fold_vm_numa_events();
    490	return sysfs_emit(buf,
    491			  "numa_hit %lu\n"
    492			  "numa_miss %lu\n"
    493			  "numa_foreign %lu\n"
    494			  "interleave_hit %lu\n"
    495			  "local_node %lu\n"
    496			  "other_node %lu\n",
    497			  sum_zone_numa_event_state(dev->id, NUMA_HIT),
    498			  sum_zone_numa_event_state(dev->id, NUMA_MISS),
    499			  sum_zone_numa_event_state(dev->id, NUMA_FOREIGN),
    500			  sum_zone_numa_event_state(dev->id, NUMA_INTERLEAVE_HIT),
    501			  sum_zone_numa_event_state(dev->id, NUMA_LOCAL),
    502			  sum_zone_numa_event_state(dev->id, NUMA_OTHER));
    503}
    504static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL);
    505
    506static ssize_t node_read_vmstat(struct device *dev,
    507				struct device_attribute *attr, char *buf)
    508{
    509	int nid = dev->id;
    510	struct pglist_data *pgdat = NODE_DATA(nid);
    511	int i;
    512	int len = 0;
    513
    514	for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
    515		len += sysfs_emit_at(buf, len, "%s %lu\n",
    516				     zone_stat_name(i),
    517				     sum_zone_node_page_state(nid, i));
    518
    519#ifdef CONFIG_NUMA
    520	fold_vm_numa_events();
    521	for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
    522		len += sysfs_emit_at(buf, len, "%s %lu\n",
    523				     numa_stat_name(i),
    524				     sum_zone_numa_event_state(nid, i));
    525
    526#endif
    527	for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
    528		unsigned long pages = node_page_state_pages(pgdat, i);
    529
    530		if (vmstat_item_print_in_thp(i))
    531			pages /= HPAGE_PMD_NR;
    532		len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i),
    533				     pages);
    534	}
    535
    536	return len;
    537}
    538static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL);
    539
    540static ssize_t node_read_distance(struct device *dev,
    541				  struct device_attribute *attr, char *buf)
    542{
    543	int nid = dev->id;
    544	int len = 0;
    545	int i;
    546
    547	/*
    548	 * buf is currently PAGE_SIZE in length and each node needs 4 chars
    549	 * at the most (distance + space or newline).
    550	 */
    551	BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE);
    552
    553	for_each_online_node(i) {
    554		len += sysfs_emit_at(buf, len, "%s%d",
    555				     i ? " " : "", node_distance(nid, i));
    556	}
    557
    558	len += sysfs_emit_at(buf, len, "\n");
    559	return len;
    560}
    561static DEVICE_ATTR(distance, 0444, node_read_distance, NULL);
    562
    563static struct attribute *node_dev_attrs[] = {
    564	&dev_attr_meminfo.attr,
    565	&dev_attr_numastat.attr,
    566	&dev_attr_distance.attr,
    567	&dev_attr_vmstat.attr,
    568	NULL
    569};
    570
    571static struct bin_attribute *node_dev_bin_attrs[] = {
    572	&bin_attr_cpumap,
    573	&bin_attr_cpulist,
    574	NULL
    575};
    576
    577static const struct attribute_group node_dev_group = {
    578	.attrs = node_dev_attrs,
    579	.bin_attrs = node_dev_bin_attrs
    580};
    581
    582static const struct attribute_group *node_dev_groups[] = {
    583	&node_dev_group,
    584#ifdef CONFIG_HAVE_ARCH_NODE_DEV_GROUP
    585	&arch_node_dev_group,
    586#endif
    587	NULL
    588};
    589
    590#ifdef CONFIG_HUGETLBFS
    591/*
    592 * hugetlbfs per node attributes registration interface:
    593 * When/if hugetlb[fs] subsystem initializes [sometime after this module],
    594 * it will register its per node attributes for all online nodes with
    595 * memory.  It will also call register_hugetlbfs_with_node(), below, to
    596 * register its attribute registration functions with this node driver.
    597 * Once these hooks have been initialized, the node driver will call into
    598 * the hugetlb module to [un]register attributes for hot-plugged nodes.
    599 */
    600static node_registration_func_t __hugetlb_register_node;
    601static node_registration_func_t __hugetlb_unregister_node;
    602
    603static inline bool hugetlb_register_node(struct node *node)
    604{
    605	if (__hugetlb_register_node &&
    606			node_state(node->dev.id, N_MEMORY)) {
    607		__hugetlb_register_node(node);
    608		return true;
    609	}
    610	return false;
    611}
    612
    613static inline void hugetlb_unregister_node(struct node *node)
    614{
    615	if (__hugetlb_unregister_node)
    616		__hugetlb_unregister_node(node);
    617}
    618
    619void register_hugetlbfs_with_node(node_registration_func_t doregister,
    620				  node_registration_func_t unregister)
    621{
    622	__hugetlb_register_node   = doregister;
    623	__hugetlb_unregister_node = unregister;
    624}
    625#else
    626static inline void hugetlb_register_node(struct node *node) {}
    627
    628static inline void hugetlb_unregister_node(struct node *node) {}
    629#endif
    630
    631static void node_device_release(struct device *dev)
    632{
    633	struct node *node = to_node(dev);
    634
    635#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_HUGETLBFS)
    636	/*
    637	 * We schedule the work only when a memory section is
    638	 * onlined/offlined on this node. When we come here,
    639	 * all the memory on this node has been offlined,
    640	 * so we won't enqueue new work to this work.
    641	 *
    642	 * The work is using node->node_work, so we should
    643	 * flush work before freeing the memory.
    644	 */
    645	flush_work(&node->node_work);
    646#endif
    647	kfree(node);
    648}
    649
    650/*
    651 * register_node - Setup a sysfs device for a node.
    652 * @num - Node number to use when creating the device.
    653 *
    654 * Initialize and register the node device.
    655 */
    656static int register_node(struct node *node, int num)
    657{
    658	int error;
    659
    660	node->dev.id = num;
    661	node->dev.bus = &node_subsys;
    662	node->dev.release = node_device_release;
    663	node->dev.groups = node_dev_groups;
    664	error = device_register(&node->dev);
    665
    666	if (error)
    667		put_device(&node->dev);
    668	else {
    669		hugetlb_register_node(node);
    670
    671		compaction_register_node(node);
    672	}
    673	return error;
    674}
    675
    676/**
    677 * unregister_node - unregister a node device
    678 * @node: node going away
    679 *
    680 * Unregisters a node device @node.  All the devices on the node must be
    681 * unregistered before calling this function.
    682 */
    683void unregister_node(struct node *node)
    684{
    685	compaction_unregister_node(node);
    686	hugetlb_unregister_node(node);		/* no-op, if memoryless node */
    687	node_remove_accesses(node);
    688	node_remove_caches(node);
    689	device_unregister(&node->dev);
    690}
    691
    692struct node *node_devices[MAX_NUMNODES];
    693
    694/*
    695 * register cpu under node
    696 */
    697int register_cpu_under_node(unsigned int cpu, unsigned int nid)
    698{
    699	int ret;
    700	struct device *obj;
    701
    702	if (!node_online(nid))
    703		return 0;
    704
    705	obj = get_cpu_device(cpu);
    706	if (!obj)
    707		return 0;
    708
    709	ret = sysfs_create_link(&node_devices[nid]->dev.kobj,
    710				&obj->kobj,
    711				kobject_name(&obj->kobj));
    712	if (ret)
    713		return ret;
    714
    715	return sysfs_create_link(&obj->kobj,
    716				 &node_devices[nid]->dev.kobj,
    717				 kobject_name(&node_devices[nid]->dev.kobj));
    718}
    719
    720/**
    721 * register_memory_node_under_compute_node - link memory node to its compute
    722 *					     node for a given access class.
    723 * @mem_nid:	Memory node number
    724 * @cpu_nid:	Cpu  node number
    725 * @access:	Access class to register
    726 *
    727 * Description:
    728 * 	For use with platforms that may have separate memory and compute nodes.
    729 * 	This function will export node relationships linking which memory
    730 * 	initiator nodes can access memory targets at a given ranked access
    731 * 	class.
    732 */
    733int register_memory_node_under_compute_node(unsigned int mem_nid,
    734					    unsigned int cpu_nid,
    735					    unsigned int access)
    736{
    737	struct node *init_node, *targ_node;
    738	struct node_access_nodes *initiator, *target;
    739	int ret;
    740
    741	if (!node_online(cpu_nid) || !node_online(mem_nid))
    742		return -ENODEV;
    743
    744	init_node = node_devices[cpu_nid];
    745	targ_node = node_devices[mem_nid];
    746	initiator = node_init_node_access(init_node, access);
    747	target = node_init_node_access(targ_node, access);
    748	if (!initiator || !target)
    749		return -ENOMEM;
    750
    751	ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets",
    752				      &targ_node->dev.kobj,
    753				      dev_name(&targ_node->dev));
    754	if (ret)
    755		return ret;
    756
    757	ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators",
    758				      &init_node->dev.kobj,
    759				      dev_name(&init_node->dev));
    760	if (ret)
    761		goto err;
    762
    763	return 0;
    764 err:
    765	sysfs_remove_link_from_group(&initiator->dev.kobj, "targets",
    766				     dev_name(&targ_node->dev));
    767	return ret;
    768}
    769
    770int unregister_cpu_under_node(unsigned int cpu, unsigned int nid)
    771{
    772	struct device *obj;
    773
    774	if (!node_online(nid))
    775		return 0;
    776
    777	obj = get_cpu_device(cpu);
    778	if (!obj)
    779		return 0;
    780
    781	sysfs_remove_link(&node_devices[nid]->dev.kobj,
    782			  kobject_name(&obj->kobj));
    783	sysfs_remove_link(&obj->kobj,
    784			  kobject_name(&node_devices[nid]->dev.kobj));
    785
    786	return 0;
    787}
    788
    789#ifdef CONFIG_MEMORY_HOTPLUG
    790static int __ref get_nid_for_pfn(unsigned long pfn)
    791{
    792#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
    793	if (system_state < SYSTEM_RUNNING)
    794		return early_pfn_to_nid(pfn);
    795#endif
    796	return pfn_to_nid(pfn);
    797}
    798
    799static void do_register_memory_block_under_node(int nid,
    800						struct memory_block *mem_blk,
    801						enum meminit_context context)
    802{
    803	int ret;
    804
    805	memory_block_add_nid(mem_blk, nid, context);
    806
    807	ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
    808				       &mem_blk->dev.kobj,
    809				       kobject_name(&mem_blk->dev.kobj));
    810	if (ret && ret != -EEXIST)
    811		dev_err_ratelimited(&node_devices[nid]->dev,
    812				    "can't create link to %s in sysfs (%d)\n",
    813				    kobject_name(&mem_blk->dev.kobj), ret);
    814
    815	ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj,
    816				&node_devices[nid]->dev.kobj,
    817				kobject_name(&node_devices[nid]->dev.kobj));
    818	if (ret && ret != -EEXIST)
    819		dev_err_ratelimited(&mem_blk->dev,
    820				    "can't create link to %s in sysfs (%d)\n",
    821				    kobject_name(&node_devices[nid]->dev.kobj),
    822				    ret);
    823}
    824
    825/* register memory section under specified node if it spans that node */
    826static int register_mem_block_under_node_early(struct memory_block *mem_blk,
    827					       void *arg)
    828{
    829	unsigned long memory_block_pfns = memory_block_size_bytes() / PAGE_SIZE;
    830	unsigned long start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
    831	unsigned long end_pfn = start_pfn + memory_block_pfns - 1;
    832	int nid = *(int *)arg;
    833	unsigned long pfn;
    834
    835	for (pfn = start_pfn; pfn <= end_pfn; pfn++) {
    836		int page_nid;
    837
    838		/*
    839		 * memory block could have several absent sections from start.
    840		 * skip pfn range from absent section
    841		 */
    842		if (!pfn_in_present_section(pfn)) {
    843			pfn = round_down(pfn + PAGES_PER_SECTION,
    844					 PAGES_PER_SECTION) - 1;
    845			continue;
    846		}
    847
    848		/*
    849		 * We need to check if page belongs to nid only at the boot
    850		 * case because node's ranges can be interleaved.
    851		 */
    852		page_nid = get_nid_for_pfn(pfn);
    853		if (page_nid < 0)
    854			continue;
    855		if (page_nid != nid)
    856			continue;
    857
    858		do_register_memory_block_under_node(nid, mem_blk, MEMINIT_EARLY);
    859		return 0;
    860	}
    861	/* mem section does not span the specified node */
    862	return 0;
    863}
    864
    865/*
    866 * During hotplug we know that all pages in the memory block belong to the same
    867 * node.
    868 */
    869static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk,
    870						 void *arg)
    871{
    872	int nid = *(int *)arg;
    873
    874	do_register_memory_block_under_node(nid, mem_blk, MEMINIT_HOTPLUG);
    875	return 0;
    876}
    877
    878/*
    879 * Unregister a memory block device under the node it spans. Memory blocks
    880 * with multiple nodes cannot be offlined and therefore also never be removed.
    881 */
    882void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
    883{
    884	if (mem_blk->nid == NUMA_NO_NODE)
    885		return;
    886
    887	sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj,
    888			  kobject_name(&mem_blk->dev.kobj));
    889	sysfs_remove_link(&mem_blk->dev.kobj,
    890			  kobject_name(&node_devices[mem_blk->nid]->dev.kobj));
    891}
    892
    893void register_memory_blocks_under_node(int nid, unsigned long start_pfn,
    894				       unsigned long end_pfn,
    895				       enum meminit_context context)
    896{
    897	walk_memory_blocks_func_t func;
    898
    899	if (context == MEMINIT_HOTPLUG)
    900		func = register_mem_block_under_node_hotplug;
    901	else
    902		func = register_mem_block_under_node_early;
    903
    904	walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn),
    905			   (void *)&nid, func);
    906	return;
    907}
    908
    909#ifdef CONFIG_HUGETLBFS
    910/*
    911 * Handle per node hstate attribute [un]registration on transistions
    912 * to/from memoryless state.
    913 */
    914static void node_hugetlb_work(struct work_struct *work)
    915{
    916	struct node *node = container_of(work, struct node, node_work);
    917
    918	/*
    919	 * We only get here when a node transitions to/from memoryless state.
    920	 * We can detect which transition occurred by examining whether the
    921	 * node has memory now.  hugetlb_register_node() already check this
    922	 * so we try to register the attributes.  If that fails, then the
    923	 * node has transitioned to memoryless, try to unregister the
    924	 * attributes.
    925	 */
    926	if (!hugetlb_register_node(node))
    927		hugetlb_unregister_node(node);
    928}
    929
    930static void init_node_hugetlb_work(int nid)
    931{
    932	INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work);
    933}
    934
    935static int node_memory_callback(struct notifier_block *self,
    936				unsigned long action, void *arg)
    937{
    938	struct memory_notify *mnb = arg;
    939	int nid = mnb->status_change_nid;
    940
    941	switch (action) {
    942	case MEM_ONLINE:
    943	case MEM_OFFLINE:
    944		/*
    945		 * offload per node hstate [un]registration to a work thread
    946		 * when transitioning to/from memoryless state.
    947		 */
    948		if (nid != NUMA_NO_NODE)
    949			schedule_work(&node_devices[nid]->node_work);
    950		break;
    951
    952	case MEM_GOING_ONLINE:
    953	case MEM_GOING_OFFLINE:
    954	case MEM_CANCEL_ONLINE:
    955	case MEM_CANCEL_OFFLINE:
    956	default:
    957		break;
    958	}
    959
    960	return NOTIFY_OK;
    961}
    962#endif	/* CONFIG_HUGETLBFS */
    963#endif /* CONFIG_MEMORY_HOTPLUG */
    964
    965#if !defined(CONFIG_MEMORY_HOTPLUG) || !defined(CONFIG_HUGETLBFS)
    966static inline int node_memory_callback(struct notifier_block *self,
    967				unsigned long action, void *arg)
    968{
    969	return NOTIFY_OK;
    970}
    971
    972static void init_node_hugetlb_work(int nid) { }
    973
    974#endif
    975
    976int __register_one_node(int nid)
    977{
    978	int error;
    979	int cpu;
    980
    981	node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL);
    982	if (!node_devices[nid])
    983		return -ENOMEM;
    984
    985	error = register_node(node_devices[nid], nid);
    986
    987	/* link cpu under this node */
    988	for_each_present_cpu(cpu) {
    989		if (cpu_to_node(cpu) == nid)
    990			register_cpu_under_node(cpu, nid);
    991	}
    992
    993	INIT_LIST_HEAD(&node_devices[nid]->access_list);
    994	/* initialize work queue for memory hot plug */
    995	init_node_hugetlb_work(nid);
    996	node_init_caches(nid);
    997
    998	return error;
    999}
   1000
   1001void unregister_one_node(int nid)
   1002{
   1003	if (!node_devices[nid])
   1004		return;
   1005
   1006	unregister_node(node_devices[nid]);
   1007	node_devices[nid] = NULL;
   1008}
   1009
   1010/*
   1011 * node states attributes
   1012 */
   1013
   1014struct node_attr {
   1015	struct device_attribute attr;
   1016	enum node_states state;
   1017};
   1018
   1019static ssize_t show_node_state(struct device *dev,
   1020			       struct device_attribute *attr, char *buf)
   1021{
   1022	struct node_attr *na = container_of(attr, struct node_attr, attr);
   1023
   1024	return sysfs_emit(buf, "%*pbl\n",
   1025			  nodemask_pr_args(&node_states[na->state]));
   1026}
   1027
   1028#define _NODE_ATTR(name, state) \
   1029	{ __ATTR(name, 0444, show_node_state, NULL), state }
   1030
   1031static struct node_attr node_state_attr[] = {
   1032	[N_POSSIBLE] = _NODE_ATTR(possible, N_POSSIBLE),
   1033	[N_ONLINE] = _NODE_ATTR(online, N_ONLINE),
   1034	[N_NORMAL_MEMORY] = _NODE_ATTR(has_normal_memory, N_NORMAL_MEMORY),
   1035#ifdef CONFIG_HIGHMEM
   1036	[N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
   1037#endif
   1038	[N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
   1039	[N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
   1040	[N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
   1041					   N_GENERIC_INITIATOR),
   1042};
   1043
   1044static struct attribute *node_state_attrs[] = {
   1045	&node_state_attr[N_POSSIBLE].attr.attr,
   1046	&node_state_attr[N_ONLINE].attr.attr,
   1047	&node_state_attr[N_NORMAL_MEMORY].attr.attr,
   1048#ifdef CONFIG_HIGHMEM
   1049	&node_state_attr[N_HIGH_MEMORY].attr.attr,
   1050#endif
   1051	&node_state_attr[N_MEMORY].attr.attr,
   1052	&node_state_attr[N_CPU].attr.attr,
   1053	&node_state_attr[N_GENERIC_INITIATOR].attr.attr,
   1054	NULL
   1055};
   1056
   1057static const struct attribute_group memory_root_attr_group = {
   1058	.attrs = node_state_attrs,
   1059};
   1060
   1061static const struct attribute_group *cpu_root_attr_groups[] = {
   1062	&memory_root_attr_group,
   1063	NULL,
   1064};
   1065
   1066#define NODE_CALLBACK_PRI	2	/* lower than SLAB */
   1067void __init node_dev_init(void)
   1068{
   1069	static struct notifier_block node_memory_callback_nb = {
   1070		.notifier_call = node_memory_callback,
   1071		.priority = NODE_CALLBACK_PRI,
   1072	};
   1073	int ret, i;
   1074
   1075 	BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES);
   1076 	BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES);
   1077
   1078	ret = subsys_system_register(&node_subsys, cpu_root_attr_groups);
   1079	if (ret)
   1080		panic("%s() failed to register subsystem: %d\n", __func__, ret);
   1081
   1082	register_hotmemory_notifier(&node_memory_callback_nb);
   1083
   1084	/*
   1085	 * Create all node devices, which will properly link the node
   1086	 * to applicable memory block devices and already created cpu devices.
   1087	 */
   1088	for_each_online_node(i) {
   1089		ret = register_one_node(i);
   1090		if (ret)
   1091			panic("%s() failed to add node: %d\n", __func__, ret);
   1092	}
   1093}