From 5983cde7021d65101643e49a9eabdc4f81cf3d88 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Thu, 12 Jan 2023 23:18:17 +0000 Subject: [PATCH 1/2] tsan: fix broken aarch64_39/42 mappings and expand them The aarch64 39- and 42-bit mappings were broken: mappings to meta and shadow were not fully invertible. This CL introduces a working set of mappings, and also increases the size of some app regions: * aarch64, 39-bit (2^39 == 512GB): - Low: (Old) 4GB -> (New) 20GB - Mid: 4GB -> 20GB - Heap: 4GB -> 12GB - High: 8GB -> 12GB * aarch64, 42-bit (2^42 == 4TB): - Low: 64GB -> 128GB - Mid: 4GB -> 88GB - Heap: 64GB -> 192GB - High: 64GB Additionally, this CL improves the code comments for all the linux aarch64 mappings. Differential Revision: https://reviews.llvm.org/D141640 --- compiler-rt/lib/tsan/rtl/tsan_platform.h | 95 ++++++++++++++---------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h index cad99f1a0bd2..d6dd2a902d37 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -147,69 +147,82 @@ struct MappingAppleAarch64 { /* C/C++ on linux/aarch64 (39-bit VMA) -0000 0010 00 - 0100 0000 00: main binary -0100 0000 00 - 0400 0000 00: - -0400 0000 00 - 1000 0000 00: shadow memory -2000 0000 00 - 3100 0000 00: - -3100 0000 00 - 3400 0000 00: metainfo -3400 0000 00 - 5500 0000 00: - -5500 0000 00 - 5600 0000 00: main binary (PIE) +0000 0010 00 - 0500 0000 00: main binary (20 GB) +0100 0000 00 - 2000 0000 00: - +2000 0000 00 - 4000 0000 00: shadow memory (128 GB) +4000 0000 00 - 4800 0000 00: metainfo (32 GB) +4800 0000 00 - 5500 0000 00: - +5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB) 5600 0000 00 - 7c00 0000 00: - -7c00 0000 00 - 7d00 0000 00: heap -7d00 0000 00 - 7fff ffff ff: modules and main thread stack +7a00 0000 00 - 7d00 0000 00: heap (12 GB) +7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB) */ struct MappingAarch64_39 { static const uptr kLoAppMemBeg = 0x0000001000ull; - static const uptr kLoAppMemEnd = 0x0100000000ull; - static const uptr kShadowBeg = 0x0400000000ull; - static const uptr kShadowEnd = 0x1000000000ull; - static const uptr kMetaShadowBeg = 0x3100000000ull; - static const uptr kMetaShadowEnd = 0x3400000000ull; + static const uptr kLoAppMemEnd = 0x0500000000ull; + static const uptr kShadowBeg = 0x2000000000ull; + static const uptr kShadowEnd = 0x4000000000ull; + static const uptr kMetaShadowBeg = 0x4000000000ull; + static const uptr kMetaShadowEnd = 0x4800000000ull; static const uptr kMidAppMemBeg = 0x5500000000ull; - static const uptr kMidAppMemEnd = 0x5600000000ull; - static const uptr kHeapMemBeg = 0x7c00000000ull; + static const uptr kMidAppMemEnd = 0x5a00000000ull; + static const uptr kHeapMemBeg = 0x7a00000000ull; static const uptr kHeapMemEnd = 0x7d00000000ull; - static const uptr kHiAppMemBeg = 0x7e00000000ull; + static const uptr kHiAppMemBeg = 0x7d00000000ull; static const uptr kHiAppMemEnd = 0x7fffffffffull; - static const uptr kShadowMsk = 0x7800000000ull; - static const uptr kShadowXor = 0x0200000000ull; - static const uptr kShadowAdd = 0x0000000000ull; + static const uptr kShadowMsk = 0x7000000000ull; + static const uptr kShadowXor = 0x1000000000ull; + static const uptr kShadowAdd = 0x0000000000ull; static const uptr kVdsoBeg = 0x7f00000000ull; }; /* C/C++ on linux/aarch64 (42-bit VMA) -00000 0010 00 - 01000 0000 00: main binary -01000 0000 00 - 08000 0000 00: - -08000 0000 00 - 10000 0000 00: shadow memory -10000 0000 00 - 26000 0000 00: - -26000 0000 00 - 28000 0000 00: metainfo -28000 0000 00 - 2aa00 0000 00: - -2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) -2ab00 0000 00 - 3e000 0000 00: - -3e000 0000 00 - 3f000 0000 00: heap -3f000 0000 00 - 3ffff ffff ff: modules and main thread stack +00000 0010 00 - 02000 0000 00: main binary (128 GB) +02000 0000 00 - 08000 0000 00: - +10000 0000 00 - 20000 0000 00: shadow memory (1024 GB) +20000 0000 00 - 24000 0000 00: metainfo (256 GB) +24000 0000 00 - 2aa00 0000 00: - +2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB) +2c000 0000 00 - 3c000 0000 00: - +3c000 0000 00 - 3f000 0000 00: heap (192 GB) +3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB) */ struct MappingAarch64_42 { - static const uptr kBroken = kBrokenReverseMapping; static const uptr kLoAppMemBeg = 0x00000001000ull; - static const uptr kLoAppMemEnd = 0x01000000000ull; - static const uptr kShadowBeg = 0x08000000000ull; - static const uptr kShadowEnd = 0x10000000000ull; - static const uptr kMetaShadowBeg = 0x26000000000ull; - static const uptr kMetaShadowEnd = 0x28000000000ull; + static const uptr kLoAppMemEnd = 0x02000000000ull; + static const uptr kShadowBeg = 0x10000000000ull; + static const uptr kShadowEnd = 0x20000000000ull; + static const uptr kMetaShadowBeg = 0x20000000000ull; + static const uptr kMetaShadowEnd = 0x24000000000ull; static const uptr kMidAppMemBeg = 0x2aa00000000ull; - static const uptr kMidAppMemEnd = 0x2ab00000000ull; - static const uptr kHeapMemBeg = 0x3e000000000ull; + static const uptr kMidAppMemEnd = 0x2c000000000ull; + static const uptr kHeapMemBeg = 0x3c000000000ull; static const uptr kHeapMemEnd = 0x3f000000000ull; static const uptr kHiAppMemBeg = 0x3f000000000ull; static const uptr kHiAppMemEnd = 0x3ffffffffffull; - static const uptr kShadowMsk = 0x3c000000000ull; - static const uptr kShadowXor = 0x04000000000ull; - static const uptr kShadowAdd = 0x00000000000ull; + static const uptr kShadowMsk = 0x38000000000ull; + static const uptr kShadowXor = 0x08000000000ull; + static const uptr kShadowAdd = 0x00000000000ull; static const uptr kVdsoBeg = 0x37f00000000ull; }; +/* +C/C++ on linux/aarch64 (48-bit VMA) +0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB) +0a00 0000 1000 - 1554 0000 0000: - +1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB) +5400 0000 1000 - 8000 0000 0000: - +8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB) +a000 0000 1000 - aaaa 0000 0000: - +aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB) +ac00 0000 1000 - fc00 0000 0000: - +fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB) + +N.B. the shadow memory region has a strange start address, because it +contains the shadows for the mid, high and low app regions (in this +unusual order). +*/ struct MappingAarch64_48 { static const uptr kLoAppMemBeg = 0x0000000001000ull; static const uptr kLoAppMemEnd = 0x0000200000000ull; -- Gitee From 14dc2ef4ec672f51b85394b5a79e0d516033de9e Mon Sep 17 00:00:00 2001 From: Sukhikh Alexander Date: Fri, 13 Sep 2024 14:56:30 +0300 Subject: [PATCH 2/2] [compiler-rt] Introduce new AARCH64_TSAN_REQUIRE_BIGGER_HEAP define, which will allow to use another memory layout for tsan. The new layout allow to use up to 64GB heap, but will probably be less performant Issue: ? Signed-off-by: Sukhikh Alexander --- compiler-rt/lib/tsan/rtl/tsan_platform.h | 227 +++++++++++++++++++++-- 1 file changed, 216 insertions(+), 11 deletions(-) diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h index d6dd2a902d37..b2d6900eaa03 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -18,9 +18,12 @@ # error "Only 64-bit is supported" #endif +#include #include "sanitizer_common/sanitizer_common.h" #include "tsan_defs.h" +#define AARCH64_TSAN_REQUIRE_BIGGER_HEAP + namespace __tsan { enum { @@ -145,6 +148,7 @@ struct MappingAppleAarch64 { static const uptr kMidAppMemEnd = 0; }; +#ifndef AARCH64_TSAN_REQUIRE_BIGGER_HEAP /* C/C++ on linux/aarch64 (39-bit VMA) 0000 0010 00 - 0500 0000 00: main binary (20 GB) @@ -153,7 +157,7 @@ C/C++ on linux/aarch64 (39-bit VMA) 4000 0000 00 - 4800 0000 00: metainfo (32 GB) 4800 0000 00 - 5500 0000 00: - 5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB) -5600 0000 00 - 7c00 0000 00: - +5600 0000 00 - 7a00 0000 00: - 7a00 0000 00 - 7d00 0000 00: heap (12 GB) 7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB) */ @@ -175,6 +179,71 @@ struct MappingAarch64_39 { static const uptr kShadowAdd = 0x0000000000ull; static const uptr kVdsoBeg = 0x7f00000000ull; }; +#else +/* +Sometimes 12 GB heap is not enough, so for this particular case let's +create another mapping, which will require more calculations and +more shadow memory, also it will make tsan less performant, +but will give us more heap space. + +C/C++ on linux/aarch64 (39-bit VMA) +0000 0010 00 - 0500 0000 00: main binary (20 GB) +0500 0000 00 - 0600 0000 00: - +0600 0000 00 - 4000 0000 00: shadow memory (232 GB) + / ------------------------------------------------------------ \ + 0600 0000 00 - 1000 0000 00: 2x Low App (40 GB) + 1000 0000 00 - 1a00 0000 00: 2x Mid App (40 GB) + 1a00 0000 00 - 3a00 0000 00: 2x Heap (128 GB) + 3a00 0000 00 - 4000 0000 00: 2x High App (24 GB) + \ ------------------------------------------------------------ / +4000 0000 00 - 5000 0000 00: metainfo (64 GB) + / ------------------------------------------------------------ \ + 4000 0000 00 - 4280 0000 00: 0.5x Low App (10 GB) + 4280 0000 00 - 4500 0000 00: 0.5x Mid App (10 GB) + 4500 0000 00 - 4d00 0000 00: 0.5x Heap (32 GB) + 4d00 0000 00 - 5000 0000 00: 0.5x High App (6+ GB) + \ ------------------------------------------------------------ / +5000 0000 00 - 5500 0000 00: - +5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB) +5600 0000 00 - 6d00 0000 00: - +6d00 0000 00 - 7d00 0000 00: heap (64 GB) +7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB) +*/ +struct MappingAarch64_39 { + static const uptr kLoAppMemBeg = 0x0000001000ull; + static const uptr kLoAppMemEnd = 0x0500000000ull; + static const uptr kShadowBeg = 0x0600000000ull; + static const uptr kLoShadowBeg = 0x0600000000ull; + static const uptr kLoShadowEnd = 0x1000000000ull; + static const uptr kMidShadowBeg = 0x1000000000ull; + static const uptr kMidShadowEnd = 0x1a00000000ull; + static const uptr kHeapShadowBeg = 0x1a00000000ull; + static const uptr kHeapShadowEnd = 0x3a00000000ull; + static const uptr kHighShadowBeg = 0x3a00000000ull; + static const uptr kHighShadowEnd = 0x4000000000ull; + static const uptr kShadowEnd = 0x4000000000ull; + static const uptr kMetaShadowBeg = 0x4000000000ull; + static const uptr kLoMetaShadowBeg = 0x4000000000ull; + static const uptr kLoMetaShadowEnd = 0x4280000000ull; + static const uptr kMidMetaShadowBeg = 0x4280000000ull; + static const uptr kMidMetaShadowEnd = 0x4500000000ull; + static const uptr kHeapMetaShadowBeg = 0x4500000000ull; + static const uptr kHeapMetaShadowEnd = 0x4d00000000ull; + static const uptr kHighMetaShadowBeg = 0x4d00000000ull; + static const uptr kHighMetaShadowEnd = 0x5000000000ull; + static const uptr kMetaShadowEnd = 0x5000000000ull; + static const uptr kMidAppMemBeg = 0x5500000000ull; + static const uptr kMidAppMemEnd = 0x5a00000000ull; + static const uptr kHeapMemBeg = 0x6d00000000ull; + static const uptr kHeapMemEnd = 0x7d00000000ull; + static const uptr kHiAppMemBeg = 0x7d00000000ull; + static const uptr kHiAppMemEnd = 0x7fffffffffull; + static const uptr kShadowMsk = 0x7000000000ull; // These values are not + static const uptr kShadowXor = 0x1000000000ull; // used, but not removed + static const uptr kShadowAdd = 0x0000000000ull; // for compatibility + static const uptr kVdsoBeg = 0x7f00000000ull; +}; +#endif /* C/C++ on linux/aarch64 (42-bit VMA) @@ -772,13 +841,54 @@ uptr MetaShadowBeg(void) { return SelectMapping(kMetaShadowBeg); } ALWAYS_INLINE uptr MetaShadowEnd(void) { return SelectMapping(kMetaShadowEnd); } +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP +ALWAYS_INLINE +uptr LinearMappingWiden(uptr x, uptr fromL, uptr fromR, uptr toL, uptr toR) { + assert(fromL != fromR); + assert(toL != toR); + uptr fromDiff = (x - fromL); + // uptr toDiff = (toR - toL); + return (fromDiff << 1) + toL; +} + +ALWAYS_INLINE +uptr LinearMappingShrink(uptr x, uptr fromL, uptr fromR, uptr toL, uptr toR) { + assert(fromL != fromR); + assert(toL != toR); + uptr fromDiff = (x - fromL); + // uptr toDiff = (toR - toL); + return (fromDiff >> 1) + toL; +} +#endif + struct IsAppMemImpl { + + template + static bool IsHeapMem(uptr mem) { + return mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd; + } + + template + static bool IsLoAppMem(uptr mem) { + return mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd; + } + + template + static bool IsMidAppMem(uptr mem) { + return mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd; + } + + template + static bool IsHiAppMem(uptr mem) { + return mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd; + } + template static bool Apply(uptr mem) { - return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || - (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || - (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || - (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); + return IsHeapMem(mem) || + IsLoAppMem(mem) || + IsMidAppMem(mem) || + IsHiAppMem(mem); } }; @@ -809,15 +919,45 @@ bool IsMetaMem(const u32 *p) { return SelectMapping(reinterpret_cast(p)); } -struct MemToShadowImpl { +namespace { + template - static uptr Apply(uptr x) { + static uptr MemToShadowImplApply(uptr x) { DCHECK(IsAppMemImpl::Apply(x)); return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^ Mapping::kShadowXor) * kShadowMultiplier + Mapping::kShadowAdd; } + +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP + using M = MappingAarch64_39; + template <> + uptr MemToShadowImplApply(uptr x) { + DCHECK(IsAppMemImpl::Apply(x)); + #define CHECK_AND_RETURN_SHADOW(x, memType, shadowType) \ + if (IsAppMemImpl::Is##memType(x)) {\ + return LinearMappingWiden(x & ~(kShadowCell - 1),\ + M::k ## memType ## Beg,\ + M::k ## memType ## End,\ + M::k ## shadowType ## Beg,\ + M::k ## shadowType ## End);\ + } else {} + CHECK_AND_RETURN_SHADOW(x, HeapMem, HeapShadow); + CHECK_AND_RETURN_SHADOW(x, LoAppMem, LoShadow); + CHECK_AND_RETURN_SHADOW(x, MidAppMem, MidShadow); + CHECK_AND_RETURN_SHADOW(x, HiAppMem, HighShadow); + #undef CHECK_AND_RETURN_SHADOW + __builtin_unreachable(); + } +#endif +} + +struct MemToShadowImpl { + template + static uptr Apply(uptr x) { + return MemToShadowImplApply(x); + } }; ALWAYS_INLINE @@ -825,22 +965,53 @@ RawShadow *MemToShadow(uptr x) { return reinterpret_cast(SelectMapping(x)); } -struct MemToMetaImpl { +namespace { + template - static u32 *Apply(uptr x) { + u32 *MemToMetaImplApply(uptr x) { DCHECK(IsAppMemImpl::Apply(x)); return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) / kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg); } + +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP + using M = MappingAarch64_39; + template <> + u32 *MemToMetaImplApply(uptr x) { + DCHECK(IsAppMemImpl::Apply(x)); + #define CHECK_AND_RETURN_META(x, memType, shadowType) \ + if(IsAppMemImpl::Is##memType(x)) {\ + return (u32 *)LinearMappingShrink(x & ~(kMetaShadowCell - 1),\ + M::k ## memType ## Beg,\ + M::k ## memType ## End,\ + M::k ## shadowType ## Beg,\ + M::k ## shadowType ## End);\ + } else {} + CHECK_AND_RETURN_META(x, HeapMem, HeapMetaShadow); + CHECK_AND_RETURN_META(x, LoAppMem, LoMetaShadow); + CHECK_AND_RETURN_META(x, MidAppMem, MidMetaShadow); + CHECK_AND_RETURN_META(x, HiAppMem, HighMetaShadow); + #undef CHECK_AND_RETURN_META + __builtin_unreachable(); + } +#endif +} + +struct MemToMetaImpl { + template + static u32 *Apply(uptr x) { + return MemToMetaImplApply(x); + } }; ALWAYS_INLINE u32 *MemToMeta(uptr x) { return SelectMapping(x); } -struct ShadowToMemImpl { + +namespace { template - static uptr Apply(uptr sp) { + uptr ShadowToMemImplApply(uptr sp) { if (!IsShadowMemImpl::Apply(sp)) return 0; // The shadow mapping is non-linear and we've lost some bits, so we don't @@ -861,6 +1032,40 @@ struct ShadowToMemImpl { } return p | Mapping::kShadowMsk; } + +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP + using M = MappingAarch64_39; + template <> + uptr ShadowToMemImplApply(uptr sp) { + if (!IsShadowMemImpl::Apply(sp)) + return 0; + + #define CHECK_AND_MAP_TO_APP(x, shadowType, memType) \ + if (M::k##shadowType##Beg <= sp &&\ + sp < M::k##shadowType##End) {\ + return LinearMappingShrink(x, \ + M::k ## shadowType ## Beg,\ + M::k ## shadowType ## End,\ + M::k ## memType ## Beg,\ + M::k ## memType ## End);\ + } else {} + CHECK_AND_MAP_TO_APP(sp, HeapShadow, HeapMem); + CHECK_AND_MAP_TO_APP(sp, LoShadow, LoAppMem); + CHECK_AND_MAP_TO_APP(sp, MidShadow, MidAppMem); + CHECK_AND_MAP_TO_APP(sp, HighShadow, HiAppMem); + #undef CHECK_AND_MAP_TO_APP + __builtin_unreachable(); + } +#endif +} + +struct ShadowToMemImpl { + + template + static uptr Apply(uptr sp) { + return ShadowToMemImplApply(sp); + } + }; ALWAYS_INLINE -- Gitee