From 262f1586c375746315f042ba3b687398165d8a89 Mon Sep 17 00:00:00 2001 From: zhoukang Date: Mon, 6 Jun 2022 12:57:08 +0000 Subject: [PATCH] elf: load elf files with hugepages When loading elf files, we read contents of the file into a preallocated hugepage instead of mapping with dl_map_segments. Using hugepage can help us saving TLB entries. Normal libraries are not aligned to 2MB, so we should ignore the permissions in elf header and give the hugepage area rwx permissions. Prerequisites: reserve enough hugepages. (default size is 2MB*128, see huge_length in elf/dl-map-segments.h) Signed-off-by: Liu Zixian Signed-off-by: Yang Yanchao Signed-off-by: zhoukang --- 0001-elf-load-elf-files-with-hugepages.patch | 142 +++++++++++++++++++ glibc.spec | 6 +- 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 0001-elf-load-elf-files-with-hugepages.patch diff --git a/0001-elf-load-elf-files-with-hugepages.patch b/0001-elf-load-elf-files-with-hugepages.patch new file mode 100644 index 0000000..88af8bd --- /dev/null +++ b/0001-elf-load-elf-files-with-hugepages.patch @@ -0,0 +1,142 @@ +From 3a605576fb6c7398da6f45c7e3177fd0fd4bc680 Mon Sep 17 00:00:00 2001 +From: zhoukang +Date: Sat, 21 May 2022 09:20:12 +0000 +Subject: [PATCH] elf: load elf files with hugepages + +When loading elf files, we read contents of the file into +a preallocated hugepage instead of mapping with dl_map_segments. +Using hugepage can help us saving TLB entries. + +Normal libraries are not aligned to 2MB, so we should ignore the +permissions in elf header and give the hugepage area rwx permissions. + +Prerequisites: reserve enough hugepages. +(default size is 2MB*128, see huge_length in elf/dl-map-segments.h) + +Signed-off-by: Liu Zixian +Signed-off-by: Yang Yanchao +Signed-off-by: zhoukang +--- + elf/dl-load.c | 5 +++++ + elf/dl-map-segments.h | 41 ++++++++++++++++++++++++++++++++++++++ + elf/dl-reloc.c | 8 ++++++++ + elf/rtld.c | 1 + + sysdeps/generic/ldsodefs.h | 2 ++ + 5 files changed, 57 insertions(+) + +diff --git a/elf/dl-load.c b/elf/dl-load.c +index 710582b5..14f23ccb 100644 +--- a/elf/dl-load.c ++++ b/elf/dl-load.c +@@ -1307,6 +1307,11 @@ _dl_map_object_from_fd (const char *name, const char *origname, int fd, + goto fallback; + } + } ++ else if (GLRO(dl_debug_mask) & DL_HUGEPAGE_MERGE_FLAG) ++ { ++ errstring = _dl_map_segments_huge (l, fd, header, type, loadcmds, nloadcmds, ++ maplength, has_holes, loader); ++ } + else + { + fallback: +diff --git a/elf/dl-map-segments.h b/elf/dl-map-segments.h +index ae8dfb25..fb379378 100644 +--- a/elf/dl-map-segments.h ++++ b/elf/dl-map-segments.h +@@ -18,6 +18,47 @@ + + #include + ++#ifdef HUGEPAGE_SHARED_LIB ++#include ++ ++#define MAP_HUGE_SHIFT 26 ++ ++ElfW(Addr) huge_start, huge_end; ++size_t huge_length = 0x200000 * 128; /* reserve enough space */ ++ ++static __always_inline const char * ++_dl_map_segments_huge (struct link_map *l, int fd, ++ const ElfW(Ehdr) *header, int type, ++ const struct loadcmd loadcmds[], size_t nloadcmds, ++ const size_t maplength, bool has_holes, ++ struct link_map *loader) ++{ ++ const struct loadcmd *c = loadcmds; ++ ++ if (huge_start == (ElfW(Addr))NULL) { ++ /* init huge page */ ++ huge_start = (ElfW(Addr))__mmap(0, huge_length, ++ PROT_READ|PROT_WRITE|PROT_EXEC, ++ MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|(21 << MAP_HUGE_SHIFT), -1, 0); ++ huge_end = huge_start; ++ } ++ l->l_map_start = huge_end; ++ huge_end += ALIGN_UP (loadcmds[nloadcmds - 1].allocend, GLRO(dl_pagesize)) - loadcmds[0].mapstart; ++ l->l_map_end = huge_end; ++ l->l_contiguous = !has_holes; ++ l->l_addr = l->l_map_start - c->mapstart; ++ while (c < &loadcmds[nloadcmds]) { ++ /* use read instead of mmap */ ++ lseek(fd, c->mapoff, SEEK_SET); ++ __read(fd, (void*)(c->mapstart + l->l_addr), c->dataend - c->mapstart); ++ _dl_postprocess_loadcmd(l, header, c); ++ ++c; ++ } ++ ++ return NULL; ++} ++#endif ++ + /* This implementation assumes (as does the corresponding implementation + of _dl_unmap_segments, in dl-unmap-segments.h) that shared objects + are always laid out with all segments contiguous (or with gaps +diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c +index e13a672a..82ad89fc 100644 +--- a/elf/dl-reloc.c ++++ b/elf/dl-reloc.c +@@ -317,6 +317,14 @@ _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[], + textrels = textrels->next; + } + ++#ifdef HUGEPAGE_SHARED_LIB ++ if (GLRO(dl_debug_mask) & DL_HUGEPAGE_MERGE_FLAG) ++ { ++ /* do not protect data in MERGE mode */ ++ return; ++ } ++#endif ++ + /* In case we can protect the data now that the relocations are + done, do it. */ + if (l->l_relro_size != 0) +diff --git a/elf/rtld.c b/elf/rtld.c +index 39ee5739..9067e0a9 100644 +--- a/elf/rtld.c ++++ b/elf/rtld.c +@@ -2515,6 +2515,7 @@ process_dl_hugepage (const char *dl_hugepage) + } hpopts[] = + { + {DL_HUGEPAGE_LARGE_IN, DL_HUGEPAGE_LIB_LARGE_IN_FLAG}, ++ {DL_HUGEPAGE_MERGE, DL_HUGEPAGE_MERGE_FLAG}, + }; + #define nhpopts (sizeof (hpopts) / sizeof (hpopts[0])) + +diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h +index fbd6bfb1..142f3a38 100644 +--- a/sysdeps/generic/ldsodefs.h ++++ b/sysdeps/generic/ldsodefs.h +@@ -552,7 +552,9 @@ struct rtld_global_ro + #ifdef HUGEPAGE_SHARED_LIB + #define DL_HUGEPAGE_PROBE_FLAG (1 << 31) + #define DL_HUGEPAGE_LIB_LARGE_IN_FLAG (1 << 30) ++#define DL_HUGEPAGE_MERGE_FLAG (1 << 29) + #define DL_HUGEPAGE_LARGE_IN 1 ++#define DL_HUGEPAGE_MERGE 2 + #endif + /* OS version. */ + EXTERN unsigned int _dl_osversion; +-- +2.31.1 + diff --git a/glibc.spec b/glibc.spec index f856adb..f45e88b 100644 --- a/glibc.spec +++ b/glibc.spec @@ -66,7 +66,7 @@ ############################################################################## Name: glibc Version: 2.34 -Release: 85 +Release: 86 Summary: The GNU libc libraries License: %{all_license} URL: http://www.gnu.org/software/glibc/ @@ -247,6 +247,7 @@ Patch9018: 0002-elf-ld.so-add-testcase-for-ld.so-load-shared-object-.patch Patch9019: 0003-elf-ld.so-use-special-mmap-for-hugepage-to-get-symbo.patch Patch9020: malloc-use-__get_nprocs-replace-__get_nprocs_sched.patch Patch9021: use-mlock-to-determine-hugepage-RLIMIT_MEMLOCK-soft-.patch +Patch9022: 0001-elf-load-elf-files-with-hugepages.patch Provides: ldconfig rtld(GNU_HASH) bundled(gnulib) @@ -1401,6 +1402,9 @@ fi %endif %changelog +* Mon Jun 6 2022 zhoukang - 2.34-86 +- add dynamic load lib use merge hugepage + * Wed Jun 1 2022 Qingqing Li - 2.34-85 - use locale-archive to prevent basic command performance regression -- Gitee