From e232333be69ee9ccd4281032af0d2416940cb98d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 8 Nov 2021 11:42:20 +0000 Subject: scripts/sorttable: Unify arm64 & x86 sort functions The format of the arm64 and x86 exception table entries is essentially the same as of commits: 46d28947d9876fc0 ("x86/extable: Rework the exception table mechanics") d6e2cc5647753825 ("arm64: extable: add `type` and `data` fields") Both use a 12-byte entry consisting of two 32-bit relative offsets and 32 bits of (absolute) data, and their sort functions are identical aside from commentary, with arm64 saying: /* Don't touch the fixup type or data */ ... and x86 saying: /* Don't touch the fixup type */ Unify the two behind a common sort_relative_table_with_data() function, retaining the arm64 commentary. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Cc: Ard Biesheuvel Cc: Borislav Petkov Cc: Catalin Marinas Cc: Thomas Gleixner Cc: Will Deacon Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Palmer Dabbelt --- scripts/sorttable.c | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) (limited to 'scripts/sorttable.c') diff --git a/scripts/sorttable.c b/scripts/sorttable.c index b7c2ad71f9cf..ca9db62bf766 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -231,7 +231,7 @@ static void sort_relative_table(char *extab_image, int image_size) } } -static void arm64_sort_relative_table(char *extab_image, int image_size) +static void sort_relative_table_with_data(char *extab_image, int image_size) { int i = 0; @@ -259,34 +259,6 @@ static void arm64_sort_relative_table(char *extab_image, int image_size) } } -static void x86_sort_relative_table(char *extab_image, int image_size) -{ - int i = 0; - - while (i < image_size) { - uint32_t *loc = (uint32_t *)(extab_image + i); - - w(r(loc) + i, loc); - w(r(loc + 1) + i + 4, loc + 1); - /* Don't touch the fixup type */ - - i += sizeof(uint32_t) * 3; - } - - qsort(extab_image, image_size / 12, 12, compare_relative_table); - - i = 0; - while (i < image_size) { - uint32_t *loc = (uint32_t *)(extab_image + i); - - w(r(loc) - i, loc); - w(r(loc + 1) - (i + 4), loc + 1); - /* Don't touch the fixup type */ - - i += sizeof(uint32_t) * 3; - } -} - static void s390_sort_relative_table(char *extab_image, int image_size) { int i; @@ -364,15 +336,13 @@ static int do_file(char const *const fname, void *addr) switch (r2(&ehdr->e_machine)) { case EM_386: + case EM_AARCH64: case EM_X86_64: - custom_sort = x86_sort_relative_table; + custom_sort = sort_relative_table_with_data; break; case EM_S390: custom_sort = s390_sort_relative_table; break; - case EM_AARCH64: - custom_sort = arm64_sort_relative_table; - break; case EM_PARISC: case EM_PPC: case EM_PPC64: -- cgit v1.2.3-71-gd317 From bb1f85d6046f0db757ac52ed60a5eba5df394819 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:22:51 +0800 Subject: riscv: switch to relative exception tables Similar as other architectures such as arm64, x86 and so on, use offsets relative to the exception table entry values rather than absolute addresses for both the exception locationand the fixup. However, RISCV label difference will actually produce two relocations, a pair of R_RISCV_ADD32 and R_RISCV_SUB32. Take below simple code for example: $ cat test.S .section .text 1: nop .section __ex_table,"a" .balign 4 .long (1b - .) .previous $ riscv64-linux-gnu-gcc -c test.S $ riscv64-linux-gnu-readelf -r test.o Relocation section '.rela__ex_table' at offset 0x100 contains 2 entries: Offset Info Type Sym. Value Sym. Name + Addend 000000000000 000600000023 R_RISCV_ADD32 0000000000000000 .L1^B1 + 0 000000000000 000500000027 R_RISCV_SUB32 0000000000000000 .L0 + 0 The modpost will complain the R_RISCV_SUB32 relocation, so we need to patch modpost.c to skip this relocation for .rela__ex_table section. After this patch, the __ex_table section size of defconfig vmlinux is reduced from 7072 Bytes to 3536 Bytes. Signed-off-by: Jisheng Zhang Reviewed-by: Kefeng Wang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/Kbuild | 1 - arch/riscv/include/asm/extable.h | 25 +++++++++++++++++++++++++ arch/riscv/include/asm/uaccess.h | 4 ++-- arch/riscv/lib/uaccess.S | 4 ++-- arch/riscv/mm/extable.c | 2 +- scripts/mod/modpost.c | 15 +++++++++++++++ scripts/sorttable.c | 2 +- 7 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 arch/riscv/include/asm/extable.h (limited to 'scripts/sorttable.c') diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index 445ccc97305a..57b86fd9916c 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += early_ioremap.h -generic-y += extable.h generic-y += flat.h generic-y += kvm_para.h generic-y += user.h diff --git a/arch/riscv/include/asm/extable.h b/arch/riscv/include/asm/extable.h new file mode 100644 index 000000000000..84760392fc69 --- /dev/null +++ b/arch/riscv/include/asm/extable.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_EXTABLE_H +#define _ASM_RISCV_EXTABLE_H + +/* + * The exception table consists of pairs of relative offsets: the first + * is the relative offset to an instruction that is allowed to fault, + * and the second is the relative offset at which the program should + * continue. No registers are modified, so it is entirely up to the + * continuation code to figure out what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + int insn, fixup; +}; + +#define ARCH_HAS_RELATIVE_EXTABLE + +int fixup_exception(struct pt_regs *regs); +#endif diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 714cd311d9f1..0f2c5b9d2e8f 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -12,8 +12,8 @@ #define _ASM_EXTABLE(from, to) \ " .pushsection __ex_table, \"a\"\n" \ - " .balign " RISCV_SZPTR " \n" \ - " " RISCV_PTR "(" #from "), (" #to ")\n" \ + " .balign 4\n" \ + " .long (" #from " - .), (" #to " - .)\n" \ " .popsection\n" /* diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 63bc691cff91..55f80f84e23f 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -7,8 +7,8 @@ 100: \op \reg, \addr .section __ex_table,"a" - .balign RISCV_SZPTR - RISCV_PTR 100b, \lbl + .balign 4 + .long (100b - .), (\lbl - .) .previous .endm diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index ddb7d3b99e89..d8d239c2c1bd 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -28,6 +28,6 @@ int fixup_exception(struct pt_regs *regs) return rv_bpf_fixup_exception(fixup, regs); #endif - regs->epc = fixup->fixup; + regs->epc = (unsigned long)&fixup->fixup + fixup->fixup; return 1; } diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index cb8ab7d91d30..6bfa33217914 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1830,6 +1830,14 @@ static int addend_mips_rel(struct elf_info *elf, Elf_Shdr *sechdr, Elf_Rela *r) return 0; } +#ifndef EM_RISCV +#define EM_RISCV 243 +#endif + +#ifndef R_RISCV_SUB32 +#define R_RISCV_SUB32 39 +#endif + static void section_rela(const char *modname, struct elf_info *elf, Elf_Shdr *sechdr) { @@ -1866,6 +1874,13 @@ static void section_rela(const char *modname, struct elf_info *elf, r_sym = ELF_R_SYM(r.r_info); #endif r.r_addend = TO_NATIVE(rela->r_addend); + switch (elf->hdr->e_machine) { + case EM_RISCV: + if (!strcmp("__ex_table", fromsec) && + ELF_R_TYPE(r.r_info) == R_RISCV_SUB32) + continue; + break; + } sym = elf->symtab_start + r_sym; /* Skip special sections */ if (is_shndx_special(sym->st_shndx)) diff --git a/scripts/sorttable.c b/scripts/sorttable.c index ca9db62bf766..f4a8255036b5 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -346,6 +346,7 @@ static int do_file(char const *const fname, void *addr) case EM_PARISC: case EM_PPC: case EM_PPC64: + case EM_RISCV: custom_sort = sort_relative_table; break; case EM_ARCOMPACT: @@ -353,7 +354,6 @@ static int do_file(char const *const fname, void *addr) case EM_ARM: case EM_MICROBLAZE: case EM_MIPS: - case EM_RISCV: case EM_XTENSA: break; default: -- cgit v1.2.3-71-gd317 From 2bf847db0c7437c28b10fba2981b9a7db4b4e0e2 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:26:05 +0800 Subject: riscv: extable: add `type` and `data` fields This is a riscv port of commit d6e2cc564775 ("arm64: extable: add `type` and `data` fields"). Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/asm-extable.h | 25 +++++++++++++++++-------- arch/riscv/include/asm/extable.h | 17 ++++++++++++++--- arch/riscv/kernel/vmlinux.lds.S | 2 +- arch/riscv/mm/extable.c | 25 +++++++++++++++++++++---- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- scripts/sorttable.c | 2 +- 6 files changed, 57 insertions(+), 19 deletions(-) (limited to 'scripts/sorttable.c') diff --git a/arch/riscv/include/asm/asm-extable.h b/arch/riscv/include/asm/asm-extable.h index b790c02dbdda..1b1f4ffd8d37 100644 --- a/arch/riscv/include/asm/asm-extable.h +++ b/arch/riscv/include/asm/asm-extable.h @@ -2,31 +2,40 @@ #ifndef __ASM_ASM_EXTABLE_H #define __ASM_ASM_EXTABLE_H +#define EX_TYPE_NONE 0 +#define EX_TYPE_FIXUP 1 +#define EX_TYPE_BPF 2 + #ifdef __ASSEMBLY__ -#define __ASM_EXTABLE_RAW(insn, fixup) \ - .pushsection __ex_table, "a"; \ - .balign 4; \ - .long ((insn) - .); \ - .long ((fixup) - .); \ +#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ + .pushsection __ex_table, "a"; \ + .balign 4; \ + .long ((insn) - .); \ + .long ((fixup) - .); \ + .short (type); \ + .short (data); \ .popsection; .macro _asm_extable, insn, fixup - __ASM_EXTABLE_RAW(\insn, \fixup) + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0) .endm #else /* __ASSEMBLY__ */ #include -#define __ASM_EXTABLE_RAW(insn, fixup) \ +#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ ".pushsection __ex_table, \"a\"\n" \ ".balign 4\n" \ ".long ((" insn ") - .)\n" \ ".long ((" fixup ") - .)\n" \ + ".short (" type ")\n" \ + ".short (" data ")\n" \ ".popsection\n" -#define _ASM_EXTABLE(insn, fixup) __ASM_EXTABLE_RAW(#insn, #fixup) +#define _ASM_EXTABLE(insn, fixup) \ + __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0") #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/extable.h b/arch/riscv/include/asm/extable.h index e4374dde02b4..512012d193dc 100644 --- a/arch/riscv/include/asm/extable.h +++ b/arch/riscv/include/asm/extable.h @@ -17,18 +17,29 @@ struct exception_table_entry { int insn, fixup; + short type, data; }; #define ARCH_HAS_RELATIVE_EXTABLE +#define swap_ex_entry_fixup(a, b, tmp, delta) \ +do { \ + (a)->fixup = (b)->fixup + (delta); \ + (b)->fixup = (tmp).fixup - (delta); \ + (a)->type = (b)->type; \ + (b)->type = (tmp).type; \ + (a)->data = (b)->data; \ + (b)->data = (tmp).data; \ +} while (0) + bool fixup_exception(struct pt_regs *regs); #if defined(CONFIG_BPF_JIT) && defined(CONFIG_ARCH_RV64I) -bool rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs); +bool ex_handler_bpf(const struct exception_table_entry *ex, struct pt_regs *regs); #else static inline bool -rv_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs) +ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs) { return false; } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 5104f3a871e3..0e5ae851929e 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define RO_EXCEPTION_TABLE_ALIGN 16 +#define RO_EXCEPTION_TABLE_ALIGN 4 #ifdef CONFIG_XIP_KERNEL #include "vmlinux-xip.lds.S" diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index 3c561f1d0115..91e52c4bb33a 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -10,6 +10,20 @@ #include #include #include +#include + +static inline unsigned long +get_ex_fixup(const struct exception_table_entry *ex) +{ + return ((unsigned long)&ex->fixup + ex->fixup); +} + +static bool ex_handler_fixup(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + regs->epc = get_ex_fixup(ex); + return true; +} bool fixup_exception(struct pt_regs *regs) { @@ -19,9 +33,12 @@ bool fixup_exception(struct pt_regs *regs) if (!ex) return false; - if (regs->epc >= BPF_JIT_REGION_START && regs->epc < BPF_JIT_REGION_END) - return rv_bpf_fixup_exception(ex, regs); + switch (ex->type) { + case EX_TYPE_FIXUP: + return ex_handler_fixup(ex, regs); + case EX_TYPE_BPF: + return ex_handler_bpf(ex, regs); + } - regs->epc = (unsigned long)&ex->fixup + ex->fixup; - return true; + BUG(); } diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 7714081cbb64..69bab7e28f91 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -459,8 +459,8 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) -bool rv_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs) +bool ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs) { off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); @@ -514,6 +514,7 @@ static int add_exception_handler(const struct bpf_insn *insn, ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); + ex->type = EX_TYPE_BPF; ctx->nexentries++; return 0; diff --git a/scripts/sorttable.c b/scripts/sorttable.c index f4a8255036b5..82b162b3941b 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -337,6 +337,7 @@ static int do_file(char const *const fname, void *addr) switch (r2(&ehdr->e_machine)) { case EM_386: case EM_AARCH64: + case EM_RISCV: case EM_X86_64: custom_sort = sort_relative_table_with_data; break; @@ -346,7 +347,6 @@ static int do_file(char const *const fname, void *addr) case EM_PARISC: case EM_PPC: case EM_PPC64: - case EM_RISCV: custom_sort = sort_relative_table; break; case EM_ARCOMPACT: -- cgit v1.2.3-71-gd317 From 72b3942a173c387b27860ba1069636726e208777 Mon Sep 17 00:00:00 2001 From: Yinan Liu Date: Sun, 12 Dec 2021 19:33:58 +0800 Subject: scripts: ftrace - move the sort-processing in ftrace_init When the kernel starts, the initialization of ftrace takes up a portion of the time (approximately 6~8ms) to sort mcount addresses. We can save this time by moving mcount-sorting to compile time. Link: https://lkml.kernel.org/r/20211212113358.34208-2-yinan@linux.alibaba.com Signed-off-by: Yinan Liu Reported-by: kernel test robot Reported-by: kernel test robot Signed-off-by: Steven Rostedt --- kernel/trace/ftrace.c | 11 ++++- scripts/Makefile | 6 ++- scripts/link-vmlinux.sh | 6 +-- scripts/sorttable.c | 2 + scripts/sorttable.h | 120 +++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 137 insertions(+), 8 deletions(-) (limited to 'scripts/sorttable.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 30bc880c3849..9ca63df6553a 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6406,8 +6406,15 @@ static int ftrace_process_locs(struct module *mod, if (!count) return 0; - sort(start, count, sizeof(*start), - ftrace_cmp_ips, NULL); + /* + * Sorting mcount in vmlinux at build time depend on + * CONFIG_BUILDTIME_TABLE_SORT, while mcount loc in + * modules can not be sorted at build time. + */ + if (!IS_ENABLED(CONFIG_BUILDTIME_TABLE_SORT) || mod) { + sort(start, count, sizeof(*start), + ftrace_cmp_ips, NULL); + } start_pg = ftrace_allocate_pages(count); if (!start_pg) diff --git a/scripts/Makefile b/scripts/Makefile index 9adb6d247818..b082d2f93357 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -17,6 +17,7 @@ hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert hostprogs-always-$(CONFIG_SYSTEM_REVOCATION_LIST) += extract-cert HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include +HOSTLDLIBS_sorttable = -lpthread HOSTCFLAGS_asn1_compiler.o = -I$(srctree)/include HOSTCFLAGS_sign-file.o = $(CRYPTO_CFLAGS) HOSTLDLIBS_sign-file = $(CRYPTO_LIBS) @@ -29,7 +30,10 @@ ARCH := x86 endif HOSTCFLAGS_sorttable.o += -I$(srctree)/tools/arch/x86/include HOSTCFLAGS_sorttable.o += -DUNWINDER_ORC_ENABLED -HOSTLDLIBS_sorttable = -lpthread +endif + +ifdef CONFIG_DYNAMIC_FTRACE +HOSTCFLAGS_sorttable.o += -DMCOUNT_SORT_ENABLED endif # The following programs are only built on demand diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index 5cdd9bc5c385..dd9955f45774 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -400,6 +400,9 @@ if [ -n "${CONFIG_DEBUG_INFO_BTF}" -a -n "${CONFIG_BPF}" ]; then ${RESOLVE_BTFIDS} vmlinux fi +info SYSMAP System.map +mksysmap vmlinux System.map + if [ -n "${CONFIG_BUILDTIME_TABLE_SORT}" ]; then info SORTTAB vmlinux if ! sorttable vmlinux; then @@ -408,9 +411,6 @@ if [ -n "${CONFIG_BUILDTIME_TABLE_SORT}" ]; then fi fi -info SYSMAP System.map -mksysmap vmlinux System.map - # step a (see comment above) if [ -n "${CONFIG_KALLSYMS}" ]; then mksysmap ${kallsyms_vmlinux} .tmp_System.map diff --git a/scripts/sorttable.c b/scripts/sorttable.c index b7c2ad71f9cf..70bdc787ddfb 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include diff --git a/scripts/sorttable.h b/scripts/sorttable.h index 7b9745cf8c70..1e8b77928fa4 100644 --- a/scripts/sorttable.h +++ b/scripts/sorttable.h @@ -19,6 +19,9 @@ #undef extable_ent_size #undef compare_extable +#undef get_mcount_loc +#undef sort_mcount_loc +#undef elf_mcount_loc #undef do_sort #undef Elf_Addr #undef Elf_Ehdr @@ -41,6 +44,9 @@ #ifdef SORTTABLE_64 # define extable_ent_size 16 # define compare_extable compare_extable_64 +# define get_mcount_loc get_mcount_loc_64 +# define sort_mcount_loc sort_mcount_loc_64 +# define elf_mcount_loc elf_mcount_loc_64 # define do_sort do_sort_64 # define Elf_Addr Elf64_Addr # define Elf_Ehdr Elf64_Ehdr @@ -62,6 +68,9 @@ #else # define extable_ent_size 8 # define compare_extable compare_extable_32 +# define get_mcount_loc get_mcount_loc_32 +# define sort_mcount_loc sort_mcount_loc_32 +# define elf_mcount_loc elf_mcount_loc_32 # define do_sort do_sort_32 # define Elf_Addr Elf32_Addr # define Elf_Ehdr Elf32_Ehdr @@ -84,8 +93,6 @@ #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) /* ORC unwinder only support X86_64 */ -#include -#include #include #define ERRSTR_MAXSZ 256 @@ -191,7 +198,64 @@ static int compare_extable(const void *a, const void *b) return 1; return 0; } +#ifdef MCOUNT_SORT_ENABLED +struct elf_mcount_loc { + Elf_Ehdr *ehdr; + Elf_Shdr *init_data_sec; + uint_t start_mcount_loc; + uint_t stop_mcount_loc; +}; + +/* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ +static void *sort_mcount_loc(void *arg) +{ + struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; + uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr) + + _r(&(emloc->init_data_sec)->sh_offset); + uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; + unsigned char *start_loc = (void *)emloc->ehdr + offset; + + qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable); + return NULL; +} + +/* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ +static void get_mcount_loc(uint_t *_start, uint_t *_stop) +{ + FILE *file_start, *file_stop; + char start_buff[20]; + char stop_buff[20]; + int len = 0; + + file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r"); + if (!file_start) { + fprintf(stderr, "get start_mcount_loc error!"); + return; + } + + file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r"); + if (!file_stop) { + fprintf(stderr, "get stop_mcount_loc error!"); + pclose(file_start); + return; + } + + while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) { + len = strlen(start_buff); + start_buff[len - 1] = '\0'; + } + *_start = strtoul(start_buff, NULL, 16); + + while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) { + len = strlen(stop_buff); + stop_buff[len - 1] = '\0'; + } + *_stop = strtoul(stop_buff, NULL, 16); + pclose(file_start); + pclose(file_stop); +} +#endif static int do_sort(Elf_Ehdr *ehdr, char const *const fname, table_sort_t custom_sort) @@ -217,6 +281,12 @@ static int do_sort(Elf_Ehdr *ehdr, int idx; unsigned int shnum; unsigned int shstrndx; +#ifdef MCOUNT_SORT_ENABLED + struct elf_mcount_loc mstruct; + uint_t _start_mcount_loc = 0; + uint_t _stop_mcount_loc = 0; + pthread_t mcount_sort_thread; +#endif #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) unsigned int orc_ip_size = 0; unsigned int orc_size = 0; @@ -253,6 +323,17 @@ static int do_sort(Elf_Ehdr *ehdr, symtab_shndx = (Elf32_Word *)((const char *)ehdr + _r(&s->sh_offset)); +#ifdef MCOUNT_SORT_ENABLED + /* locate the .init.data section in vmlinux */ + if (!strcmp(secstrings + idx, ".init.data")) { + get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc); + mstruct.ehdr = ehdr; + mstruct.init_data_sec = s; + mstruct.start_mcount_loc = _start_mcount_loc; + mstruct.stop_mcount_loc = _stop_mcount_loc; + } +#endif + #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) /* locate the ORC unwind tables */ if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { @@ -294,6 +375,23 @@ static int do_sort(Elf_Ehdr *ehdr, goto out; } #endif + +#ifdef MCOUNT_SORT_ENABLED + if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) { + fprintf(stderr, + "incomplete mcount's sort in file: %s\n", + fname); + goto out; + } + + /* create thread to sort mcount_loc concurrently */ + if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { + fprintf(stderr, + "pthread_create mcount_sort_thread failed '%s': %s\n", + strerror(errno), fname); + goto out; + } +#endif if (!extab_sec) { fprintf(stderr, "no __ex_table in file: %s\n", fname); goto out; @@ -376,5 +474,23 @@ out: } } #endif + +#ifdef MCOUNT_SORT_ENABLED + if (mcount_sort_thread) { + void *retval = NULL; + /* wait for mcount sort done */ + rc = pthread_join(mcount_sort_thread, &retval); + if (rc) { + fprintf(stderr, + "pthread_join failed '%s': %s\n", + strerror(errno), fname); + } else if (retval) { + rc = -1; + fprintf(stderr, + "failed to sort mcount '%s': %s\n", + (char *)retval, fname); + } + } +#endif return rc; } -- cgit v1.2.3-71-gd317