cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

xdp_redirect_cpu_user.c (15291B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/* Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
      3 */
      4static const char *__doc__ =
      5"XDP CPU redirect tool, using BPF_MAP_TYPE_CPUMAP\n"
      6"Usage: xdp_redirect_cpu -d <IFINDEX|IFNAME> -c 0 ... -c N\n"
      7"Valid specification for CPUMAP BPF program:\n"
      8"  --mprog-name/-e pass (use built-in XDP_PASS program)\n"
      9"  --mprog-name/-e drop (use built-in XDP_DROP program)\n"
     10"  --redirect-device/-r <ifindex|ifname> (use built-in DEVMAP redirect program)\n"
     11"  Custom CPUMAP BPF program:\n"
     12"    --mprog-filename/-f <filename> --mprog-name/-e <program>\n"
     13"    Optionally, also pass --redirect-map/-m and --redirect-device/-r together\n"
     14"    to configure DEVMAP in BPF object <filename>\n";
     15
     16#include <errno.h>
     17#include <signal.h>
     18#include <stdio.h>
     19#include <stdlib.h>
     20#include <stdbool.h>
     21#include <string.h>
     22#include <unistd.h>
     23#include <locale.h>
     24#include <sys/sysinfo.h>
     25#include <getopt.h>
     26#include <net/if.h>
     27#include <time.h>
     28#include <linux/limits.h>
     29#include <arpa/inet.h>
     30#include <linux/if_link.h>
     31#include <bpf/bpf.h>
     32#include <bpf/libbpf.h>
     33#include "bpf_util.h"
     34#include "xdp_sample_user.h"
     35#include "xdp_redirect_cpu.skel.h"
     36
     37static int map_fd;
     38static int avail_fd;
     39static int count_fd;
     40
     41static int mask = SAMPLE_RX_CNT | SAMPLE_REDIRECT_ERR_MAP_CNT |
     42		  SAMPLE_CPUMAP_ENQUEUE_CNT | SAMPLE_CPUMAP_KTHREAD_CNT |
     43		  SAMPLE_EXCEPTION_CNT;
     44
     45DEFINE_SAMPLE_INIT(xdp_redirect_cpu);
     46
     47static const struct option long_options[] = {
     48	{ "help", no_argument, NULL, 'h' },
     49	{ "dev", required_argument, NULL, 'd' },
     50	{ "skb-mode", no_argument, NULL, 'S' },
     51	{ "progname", required_argument, NULL, 'p' },
     52	{ "qsize", required_argument, NULL, 'q' },
     53	{ "cpu", required_argument, NULL, 'c' },
     54	{ "stress-mode", no_argument, NULL, 'x' },
     55	{ "force", no_argument, NULL, 'F' },
     56	{ "interval", required_argument, NULL, 'i' },
     57	{ "verbose", no_argument, NULL, 'v' },
     58	{ "stats", no_argument, NULL, 's' },
     59	{ "mprog-name", required_argument, NULL, 'e' },
     60	{ "mprog-filename", required_argument, NULL, 'f' },
     61	{ "redirect-device", required_argument, NULL, 'r' },
     62	{ "redirect-map", required_argument, NULL, 'm' },
     63	{}
     64};
     65
     66static void print_avail_progs(struct bpf_object *obj)
     67{
     68	struct bpf_program *pos;
     69
     70	printf(" Programs to be used for -p/--progname:\n");
     71	bpf_object__for_each_program(pos, obj) {
     72		if (bpf_program__type(pos) == BPF_PROG_TYPE_XDP) {
     73			if (!strncmp(bpf_program__name(pos), "xdp_prognum",
     74				     sizeof("xdp_prognum") - 1))
     75				printf(" %s\n", bpf_program__name(pos));
     76		}
     77	}
     78}
     79
     80static void usage(char *argv[], const struct option *long_options,
     81		  const char *doc, int mask, bool error, struct bpf_object *obj)
     82{
     83	sample_usage(argv, long_options, doc, mask, error);
     84	print_avail_progs(obj);
     85}
     86
     87static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value,
     88			    __u32 avail_idx, bool new)
     89{
     90	__u32 curr_cpus_count = 0;
     91	__u32 key = 0;
     92	int ret;
     93
     94	/* Add a CPU entry to cpumap, as this allocate a cpu entry in
     95	 * the kernel for the cpu.
     96	 */
     97	ret = bpf_map_update_elem(map_fd, &cpu, value, 0);
     98	if (ret < 0) {
     99		fprintf(stderr, "Create CPU entry failed: %s\n", strerror(errno));
    100		return ret;
    101	}
    102
    103	/* Inform bpf_prog's that a new CPU is available to select
    104	 * from via some control maps.
    105	 */
    106	ret = bpf_map_update_elem(avail_fd, &avail_idx, &cpu, 0);
    107	if (ret < 0) {
    108		fprintf(stderr, "Add to avail CPUs failed: %s\n", strerror(errno));
    109		return ret;
    110	}
    111
    112	/* When not replacing/updating existing entry, bump the count */
    113	ret = bpf_map_lookup_elem(count_fd, &key, &curr_cpus_count);
    114	if (ret < 0) {
    115		fprintf(stderr, "Failed reading curr cpus_count: %s\n",
    116			strerror(errno));
    117		return ret;
    118	}
    119	if (new) {
    120		curr_cpus_count++;
    121		ret = bpf_map_update_elem(count_fd, &key,
    122					  &curr_cpus_count, 0);
    123		if (ret < 0) {
    124			fprintf(stderr, "Failed write curr cpus_count: %s\n",
    125				strerror(errno));
    126			return ret;
    127		}
    128	}
    129
    130	printf("%s CPU: %u as idx: %u qsize: %d cpumap_prog_fd: %d (cpus_count: %u)\n",
    131	       new ? "Add new" : "Replace", cpu, avail_idx,
    132	       value->qsize, value->bpf_prog.fd, curr_cpus_count);
    133
    134	return 0;
    135}
    136
    137/* CPUs are zero-indexed. Thus, add a special sentinel default value
    138 * in map cpus_available to mark CPU index'es not configured
    139 */
    140static int mark_cpus_unavailable(void)
    141{
    142	int ret, i, n_cpus = libbpf_num_possible_cpus();
    143	__u32 invalid_cpu = n_cpus;
    144
    145	for (i = 0; i < n_cpus; i++) {
    146		ret = bpf_map_update_elem(avail_fd, &i,
    147					  &invalid_cpu, 0);
    148		if (ret < 0) {
    149			fprintf(stderr, "Failed marking CPU unavailable: %s\n",
    150				strerror(errno));
    151			return ret;
    152		}
    153	}
    154
    155	return 0;
    156}
    157
    158/* Stress cpumap management code by concurrently changing underlying cpumap */
    159static void stress_cpumap(void *ctx)
    160{
    161	struct bpf_cpumap_val *value = ctx;
    162
    163	/* Changing qsize will cause kernel to free and alloc a new
    164	 * bpf_cpu_map_entry, with an associated/complicated tear-down
    165	 * procedure.
    166	 */
    167	value->qsize = 1024;
    168	create_cpu_entry(1, value, 0, false);
    169	value->qsize = 8;
    170	create_cpu_entry(1, value, 0, false);
    171	value->qsize = 16000;
    172	create_cpu_entry(1, value, 0, false);
    173}
    174
    175static int set_cpumap_prog(struct xdp_redirect_cpu *skel,
    176			   const char *redir_interface, const char *redir_map,
    177			   const char *mprog_filename, const char *mprog_name)
    178{
    179	if (mprog_filename) {
    180		struct bpf_program *prog;
    181		struct bpf_object *obj;
    182		int ret;
    183
    184		if (!mprog_name) {
    185			fprintf(stderr, "BPF program not specified for file %s\n",
    186				mprog_filename);
    187			goto end;
    188		}
    189		if ((redir_interface && !redir_map) || (!redir_interface && redir_map)) {
    190			fprintf(stderr, "--redirect-%s specified but --redirect-%s not specified\n",
    191				redir_interface ? "device" : "map", redir_interface ? "map" : "device");
    192			goto end;
    193		}
    194
    195		/* Custom BPF program */
    196		obj = bpf_object__open_file(mprog_filename, NULL);
    197		if (!obj) {
    198			ret = -errno;
    199			fprintf(stderr, "Failed to bpf_prog_load_xattr: %s\n",
    200				strerror(errno));
    201			return ret;
    202		}
    203
    204		ret = bpf_object__load(obj);
    205		if (ret < 0) {
    206			ret = -errno;
    207			fprintf(stderr, "Failed to bpf_object__load: %s\n",
    208				strerror(errno));
    209			return ret;
    210		}
    211
    212		if (redir_map) {
    213			int err, redir_map_fd, ifindex_out, key = 0;
    214
    215			redir_map_fd = bpf_object__find_map_fd_by_name(obj, redir_map);
    216			if (redir_map_fd < 0) {
    217				fprintf(stderr, "Failed to bpf_object__find_map_fd_by_name: %s\n",
    218					strerror(errno));
    219				return redir_map_fd;
    220			}
    221
    222			ifindex_out = if_nametoindex(redir_interface);
    223			if (!ifindex_out)
    224				ifindex_out = strtoul(redir_interface, NULL, 0);
    225			if (!ifindex_out) {
    226				fprintf(stderr, "Bad interface name or index\n");
    227				return -EINVAL;
    228			}
    229
    230			err = bpf_map_update_elem(redir_map_fd, &key, &ifindex_out, 0);
    231			if (err < 0)
    232				return err;
    233		}
    234
    235		prog = bpf_object__find_program_by_name(obj, mprog_name);
    236		if (!prog) {
    237			ret = -errno;
    238			fprintf(stderr, "Failed to bpf_object__find_program_by_name: %s\n",
    239				strerror(errno));
    240			return ret;
    241		}
    242
    243		return bpf_program__fd(prog);
    244	} else {
    245		if (mprog_name) {
    246			if (redir_interface || redir_map) {
    247				fprintf(stderr, "Need to specify --mprog-filename/-f\n");
    248				goto end;
    249			}
    250			if (!strcmp(mprog_name, "pass") || !strcmp(mprog_name, "drop")) {
    251				/* Use built-in pass/drop programs */
    252				return *mprog_name == 'p' ? bpf_program__fd(skel->progs.xdp_redirect_cpu_pass)
    253					: bpf_program__fd(skel->progs.xdp_redirect_cpu_drop);
    254			} else {
    255				fprintf(stderr, "Unknown name \"%s\" for built-in BPF program\n",
    256					mprog_name);
    257				goto end;
    258			}
    259		} else {
    260			if (redir_map) {
    261				fprintf(stderr, "Need to specify --mprog-filename, --mprog-name and"
    262					" --redirect-device with --redirect-map\n");
    263				goto end;
    264			}
    265			if (redir_interface) {
    266				/* Use built-in devmap redirect */
    267				struct bpf_devmap_val val = {};
    268				int ifindex_out, err;
    269				__u32 key = 0;
    270
    271				if (!redir_interface)
    272					return 0;
    273
    274				ifindex_out = if_nametoindex(redir_interface);
    275				if (!ifindex_out)
    276					ifindex_out = strtoul(redir_interface, NULL, 0);
    277				if (!ifindex_out) {
    278					fprintf(stderr, "Bad interface name or index\n");
    279					return -EINVAL;
    280				}
    281
    282				if (get_mac_addr(ifindex_out, skel->bss->tx_mac_addr) < 0) {
    283					printf("Get interface %d mac failed\n", ifindex_out);
    284					return -EINVAL;
    285				}
    286
    287				val.ifindex = ifindex_out;
    288				val.bpf_prog.fd = bpf_program__fd(skel->progs.xdp_redirect_egress_prog);
    289				err = bpf_map_update_elem(bpf_map__fd(skel->maps.tx_port), &key, &val, 0);
    290				if (err < 0)
    291					return -errno;
    292
    293				return bpf_program__fd(skel->progs.xdp_redirect_cpu_devmap);
    294			}
    295		}
    296	}
    297
    298	/* Disabled */
    299	return 0;
    300end:
    301	fprintf(stderr, "Invalid options for CPUMAP BPF program\n");
    302	return -EINVAL;
    303}
    304
    305int main(int argc, char **argv)
    306{
    307	const char *redir_interface = NULL, *redir_map = NULL;
    308	const char *mprog_filename = NULL, *mprog_name = NULL;
    309	struct xdp_redirect_cpu *skel;
    310	struct bpf_map_info info = {};
    311	struct bpf_cpumap_val value;
    312	__u32 infosz = sizeof(info);
    313	int ret = EXIT_FAIL_OPTION;
    314	unsigned long interval = 2;
    315	bool stress_mode = false;
    316	struct bpf_program *prog;
    317	const char *prog_name;
    318	bool generic = false;
    319	bool force = false;
    320	int added_cpus = 0;
    321	bool error = true;
    322	int longindex = 0;
    323	int add_cpu = -1;
    324	int ifindex = -1;
    325	int *cpu, i, opt;
    326	__u32 qsize;
    327	int n_cpus;
    328
    329	n_cpus = libbpf_num_possible_cpus();
    330
    331	/* Notice: Choosing the queue size is very important when CPU is
    332	 * configured with power-saving states.
    333	 *
    334	 * If deepest state take 133 usec to wakeup from (133/10^6). When link
    335	 * speed is 10Gbit/s ((10*10^9/8) in bytes/sec). How many bytes can
    336	 * arrive with in 133 usec at this speed: (10*10^9/8)*(133/10^6) =
    337	 * 166250 bytes. With MTU size packets this is 110 packets, and with
    338	 * minimum Ethernet (MAC-preamble + intergap) 84 bytes is 1979 packets.
    339	 *
    340	 * Setting default cpumap queue to 2048 as worst-case (small packet)
    341	 * should be +64 packet due kthread wakeup call (due to xdp_do_flush)
    342	 * worst-case is 2043 packets.
    343	 *
    344	 * Sysadm can configured system to avoid deep-sleep via:
    345	 *   tuned-adm profile network-latency
    346	 */
    347	qsize = 2048;
    348
    349	skel = xdp_redirect_cpu__open();
    350	if (!skel) {
    351		fprintf(stderr, "Failed to xdp_redirect_cpu__open: %s\n",
    352			strerror(errno));
    353		ret = EXIT_FAIL_BPF;
    354		goto end;
    355	}
    356
    357	ret = sample_init_pre_load(skel);
    358	if (ret < 0) {
    359		fprintf(stderr, "Failed to sample_init_pre_load: %s\n", strerror(-ret));
    360		ret = EXIT_FAIL_BPF;
    361		goto end_destroy;
    362	}
    363
    364	if (bpf_map__set_max_entries(skel->maps.cpu_map, n_cpus) < 0) {
    365		fprintf(stderr, "Failed to set max entries for cpu_map map: %s",
    366			strerror(errno));
    367		ret = EXIT_FAIL_BPF;
    368		goto end_destroy;
    369	}
    370
    371	if (bpf_map__set_max_entries(skel->maps.cpus_available, n_cpus) < 0) {
    372		fprintf(stderr, "Failed to set max entries for cpus_available map: %s",
    373			strerror(errno));
    374		ret = EXIT_FAIL_BPF;
    375		goto end_destroy;
    376	}
    377
    378	cpu = calloc(n_cpus, sizeof(int));
    379	if (!cpu) {
    380		fprintf(stderr, "Failed to allocate cpu array\n");
    381		goto end_destroy;
    382	}
    383
    384	prog = skel->progs.xdp_prognum5_lb_hash_ip_pairs;
    385	while ((opt = getopt_long(argc, argv, "d:si:Sxp:f:e:r:m:c:q:Fvh",
    386				  long_options, &longindex)) != -1) {
    387		switch (opt) {
    388		case 'd':
    389			if (strlen(optarg) >= IF_NAMESIZE) {
    390				fprintf(stderr, "-d/--dev name too long\n");
    391				usage(argv, long_options, __doc__, mask, true, skel->obj);
    392				goto end_cpu;
    393			}
    394			ifindex = if_nametoindex(optarg);
    395			if (!ifindex)
    396				ifindex = strtoul(optarg, NULL, 0);
    397			if (!ifindex) {
    398				fprintf(stderr, "Bad interface index or name (%d): %s\n",
    399					errno, strerror(errno));
    400				usage(argv, long_options, __doc__, mask, true, skel->obj);
    401				goto end_cpu;
    402			}
    403			break;
    404		case 's':
    405			mask |= SAMPLE_REDIRECT_MAP_CNT;
    406			break;
    407		case 'i':
    408			interval = strtoul(optarg, NULL, 0);
    409			break;
    410		case 'S':
    411			generic = true;
    412			break;
    413		case 'x':
    414			stress_mode = true;
    415			break;
    416		case 'p':
    417			/* Selecting eBPF prog to load */
    418			prog_name = optarg;
    419			prog = bpf_object__find_program_by_name(skel->obj,
    420								prog_name);
    421			if (!prog) {
    422				fprintf(stderr,
    423					"Failed to find program %s specified by"
    424					" option -p/--progname\n",
    425					prog_name);
    426				print_avail_progs(skel->obj);
    427				goto end_cpu;
    428			}
    429			break;
    430		case 'f':
    431			mprog_filename = optarg;
    432			break;
    433		case 'e':
    434			mprog_name = optarg;
    435			break;
    436		case 'r':
    437			redir_interface = optarg;
    438			mask |= SAMPLE_DEVMAP_XMIT_CNT_MULTI;
    439			break;
    440		case 'm':
    441			redir_map = optarg;
    442			break;
    443		case 'c':
    444			/* Add multiple CPUs */
    445			add_cpu = strtoul(optarg, NULL, 0);
    446			if (add_cpu >= n_cpus) {
    447				fprintf(stderr,
    448				"--cpu nr too large for cpumap err (%d):%s\n",
    449					errno, strerror(errno));
    450				usage(argv, long_options, __doc__, mask, true, skel->obj);
    451				goto end_cpu;
    452			}
    453			cpu[added_cpus++] = add_cpu;
    454			break;
    455		case 'q':
    456			qsize = strtoul(optarg, NULL, 0);
    457			break;
    458		case 'F':
    459			force = true;
    460			break;
    461		case 'v':
    462			sample_switch_mode();
    463			break;
    464		case 'h':
    465			error = false;
    466		default:
    467			usage(argv, long_options, __doc__, mask, error, skel->obj);
    468			goto end_cpu;
    469		}
    470	}
    471
    472	ret = EXIT_FAIL_OPTION;
    473	if (ifindex == -1) {
    474		fprintf(stderr, "Required option --dev missing\n");
    475		usage(argv, long_options, __doc__, mask, true, skel->obj);
    476		goto end_cpu;
    477	}
    478
    479	if (add_cpu == -1) {
    480		fprintf(stderr, "Required option --cpu missing\n"
    481				"Specify multiple --cpu option to add more\n");
    482		usage(argv, long_options, __doc__, mask, true, skel->obj);
    483		goto end_cpu;
    484	}
    485
    486	skel->rodata->from_match[0] = ifindex;
    487	if (redir_interface)
    488		skel->rodata->to_match[0] = if_nametoindex(redir_interface);
    489
    490	ret = xdp_redirect_cpu__load(skel);
    491	if (ret < 0) {
    492		fprintf(stderr, "Failed to xdp_redirect_cpu__load: %s\n",
    493			strerror(errno));
    494		goto end_cpu;
    495	}
    496
    497	ret = bpf_obj_get_info_by_fd(bpf_map__fd(skel->maps.cpu_map), &info, &infosz);
    498	if (ret < 0) {
    499		fprintf(stderr, "Failed bpf_obj_get_info_by_fd for cpumap: %s\n",
    500			strerror(errno));
    501		goto end_cpu;
    502	}
    503
    504	skel->bss->cpumap_map_id = info.id;
    505
    506	map_fd = bpf_map__fd(skel->maps.cpu_map);
    507	avail_fd = bpf_map__fd(skel->maps.cpus_available);
    508	count_fd = bpf_map__fd(skel->maps.cpus_count);
    509
    510	ret = mark_cpus_unavailable();
    511	if (ret < 0) {
    512		fprintf(stderr, "Unable to mark CPUs as unavailable\n");
    513		goto end_cpu;
    514	}
    515
    516	ret = sample_init(skel, mask);
    517	if (ret < 0) {
    518		fprintf(stderr, "Failed to initialize sample: %s\n", strerror(-ret));
    519		ret = EXIT_FAIL;
    520		goto end_cpu;
    521	}
    522
    523	value.bpf_prog.fd = set_cpumap_prog(skel, redir_interface, redir_map,
    524					    mprog_filename, mprog_name);
    525	if (value.bpf_prog.fd < 0) {
    526		fprintf(stderr, "Failed to set CPUMAP BPF program: %s\n",
    527			strerror(-value.bpf_prog.fd));
    528		usage(argv, long_options, __doc__, mask, true, skel->obj);
    529		ret = EXIT_FAIL_BPF;
    530		goto end_cpu;
    531	}
    532	value.qsize = qsize;
    533
    534	for (i = 0; i < added_cpus; i++) {
    535		if (create_cpu_entry(cpu[i], &value, i, true) < 0) {
    536			fprintf(stderr, "Cannot proceed, exiting\n");
    537			usage(argv, long_options, __doc__, mask, true, skel->obj);
    538			goto end_cpu;
    539		}
    540	}
    541
    542	ret = EXIT_FAIL_XDP;
    543	if (sample_install_xdp(prog, ifindex, generic, force) < 0)
    544		goto end_cpu;
    545
    546	ret = sample_run(interval, stress_mode ? stress_cpumap : NULL, &value);
    547	if (ret < 0) {
    548		fprintf(stderr, "Failed during sample run: %s\n", strerror(-ret));
    549		ret = EXIT_FAIL;
    550		goto end_cpu;
    551	}
    552	ret = EXIT_OK;
    553end_cpu:
    554	free(cpu);
    555end_destroy:
    556	xdp_redirect_cpu__destroy(skel);
    557end:
    558	sample_exit(ret);
    559}