diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h index cad99f1a0bd2635b121307e05ebc04eb02c0f38f..b2d6900eaa033b107dc69afaa292bfbfd75478f8 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -18,9 +18,12 @@ # error "Only 64-bit is supported" #endif +#include #include "sanitizer_common/sanitizer_common.h" #include "tsan_defs.h" +#define AARCH64_TSAN_REQUIRE_BIGGER_HEAP + namespace __tsan { enum { @@ -145,71 +148,150 @@ struct MappingAppleAarch64 { static const uptr kMidAppMemEnd = 0; }; +#ifndef AARCH64_TSAN_REQUIRE_BIGGER_HEAP /* C/C++ on linux/aarch64 (39-bit VMA) -0000 0010 00 - 0100 0000 00: main binary -0100 0000 00 - 0400 0000 00: - -0400 0000 00 - 1000 0000 00: shadow memory -2000 0000 00 - 3100 0000 00: - -3100 0000 00 - 3400 0000 00: metainfo -3400 0000 00 - 5500 0000 00: - -5500 0000 00 - 5600 0000 00: main binary (PIE) -5600 0000 00 - 7c00 0000 00: - -7c00 0000 00 - 7d00 0000 00: heap -7d00 0000 00 - 7fff ffff ff: modules and main thread stack +0000 0010 00 - 0500 0000 00: main binary (20 GB) +0100 0000 00 - 2000 0000 00: - +2000 0000 00 - 4000 0000 00: shadow memory (128 GB) +4000 0000 00 - 4800 0000 00: metainfo (32 GB) +4800 0000 00 - 5500 0000 00: - +5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB) +5600 0000 00 - 7a00 0000 00: - +7a00 0000 00 - 7d00 0000 00: heap (12 GB) +7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB) */ struct MappingAarch64_39 { static const uptr kLoAppMemBeg = 0x0000001000ull; - static const uptr kLoAppMemEnd = 0x0100000000ull; - static const uptr kShadowBeg = 0x0400000000ull; - static const uptr kShadowEnd = 0x1000000000ull; - static const uptr kMetaShadowBeg = 0x3100000000ull; - static const uptr kMetaShadowEnd = 0x3400000000ull; + static const uptr kLoAppMemEnd = 0x0500000000ull; + static const uptr kShadowBeg = 0x2000000000ull; + static const uptr kShadowEnd = 0x4000000000ull; + static const uptr kMetaShadowBeg = 0x4000000000ull; + static const uptr kMetaShadowEnd = 0x4800000000ull; static const uptr kMidAppMemBeg = 0x5500000000ull; - static const uptr kMidAppMemEnd = 0x5600000000ull; - static const uptr kHeapMemBeg = 0x7c00000000ull; + static const uptr kMidAppMemEnd = 0x5a00000000ull; + static const uptr kHeapMemBeg = 0x7a00000000ull; static const uptr kHeapMemEnd = 0x7d00000000ull; - static const uptr kHiAppMemBeg = 0x7e00000000ull; + static const uptr kHiAppMemBeg = 0x7d00000000ull; static const uptr kHiAppMemEnd = 0x7fffffffffull; - static const uptr kShadowMsk = 0x7800000000ull; - static const uptr kShadowXor = 0x0200000000ull; - static const uptr kShadowAdd = 0x0000000000ull; + static const uptr kShadowMsk = 0x7000000000ull; + static const uptr kShadowXor = 0x1000000000ull; + static const uptr kShadowAdd = 0x0000000000ull; + static const uptr kVdsoBeg = 0x7f00000000ull; +}; +#else +/* +Sometimes 12 GB heap is not enough, so for this particular case let's +create another mapping, which will require more calculations and +more shadow memory, also it will make tsan less performant, +but will give us more heap space. + +C/C++ on linux/aarch64 (39-bit VMA) +0000 0010 00 - 0500 0000 00: main binary (20 GB) +0500 0000 00 - 0600 0000 00: - +0600 0000 00 - 4000 0000 00: shadow memory (232 GB) + / ------------------------------------------------------------ \ + 0600 0000 00 - 1000 0000 00: 2x Low App (40 GB) + 1000 0000 00 - 1a00 0000 00: 2x Mid App (40 GB) + 1a00 0000 00 - 3a00 0000 00: 2x Heap (128 GB) + 3a00 0000 00 - 4000 0000 00: 2x High App (24 GB) + \ ------------------------------------------------------------ / +4000 0000 00 - 5000 0000 00: metainfo (64 GB) + / ------------------------------------------------------------ \ + 4000 0000 00 - 4280 0000 00: 0.5x Low App (10 GB) + 4280 0000 00 - 4500 0000 00: 0.5x Mid App (10 GB) + 4500 0000 00 - 4d00 0000 00: 0.5x Heap (32 GB) + 4d00 0000 00 - 5000 0000 00: 0.5x High App (6+ GB) + \ ------------------------------------------------------------ / +5000 0000 00 - 5500 0000 00: - +5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB) +5600 0000 00 - 6d00 0000 00: - +6d00 0000 00 - 7d00 0000 00: heap (64 GB) +7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB) +*/ +struct MappingAarch64_39 { + static const uptr kLoAppMemBeg = 0x0000001000ull; + static const uptr kLoAppMemEnd = 0x0500000000ull; + static const uptr kShadowBeg = 0x0600000000ull; + static const uptr kLoShadowBeg = 0x0600000000ull; + static const uptr kLoShadowEnd = 0x1000000000ull; + static const uptr kMidShadowBeg = 0x1000000000ull; + static const uptr kMidShadowEnd = 0x1a00000000ull; + static const uptr kHeapShadowBeg = 0x1a00000000ull; + static const uptr kHeapShadowEnd = 0x3a00000000ull; + static const uptr kHighShadowBeg = 0x3a00000000ull; + static const uptr kHighShadowEnd = 0x4000000000ull; + static const uptr kShadowEnd = 0x4000000000ull; + static const uptr kMetaShadowBeg = 0x4000000000ull; + static const uptr kLoMetaShadowBeg = 0x4000000000ull; + static const uptr kLoMetaShadowEnd = 0x4280000000ull; + static const uptr kMidMetaShadowBeg = 0x4280000000ull; + static const uptr kMidMetaShadowEnd = 0x4500000000ull; + static const uptr kHeapMetaShadowBeg = 0x4500000000ull; + static const uptr kHeapMetaShadowEnd = 0x4d00000000ull; + static const uptr kHighMetaShadowBeg = 0x4d00000000ull; + static const uptr kHighMetaShadowEnd = 0x5000000000ull; + static const uptr kMetaShadowEnd = 0x5000000000ull; + static const uptr kMidAppMemBeg = 0x5500000000ull; + static const uptr kMidAppMemEnd = 0x5a00000000ull; + static const uptr kHeapMemBeg = 0x6d00000000ull; + static const uptr kHeapMemEnd = 0x7d00000000ull; + static const uptr kHiAppMemBeg = 0x7d00000000ull; + static const uptr kHiAppMemEnd = 0x7fffffffffull; + static const uptr kShadowMsk = 0x7000000000ull; // These values are not + static const uptr kShadowXor = 0x1000000000ull; // used, but not removed + static const uptr kShadowAdd = 0x0000000000ull; // for compatibility static const uptr kVdsoBeg = 0x7f00000000ull; }; +#endif /* C/C++ on linux/aarch64 (42-bit VMA) -00000 0010 00 - 01000 0000 00: main binary -01000 0000 00 - 08000 0000 00: - -08000 0000 00 - 10000 0000 00: shadow memory -10000 0000 00 - 26000 0000 00: - -26000 0000 00 - 28000 0000 00: metainfo -28000 0000 00 - 2aa00 0000 00: - -2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) -2ab00 0000 00 - 3e000 0000 00: - -3e000 0000 00 - 3f000 0000 00: heap -3f000 0000 00 - 3ffff ffff ff: modules and main thread stack +00000 0010 00 - 02000 0000 00: main binary (128 GB) +02000 0000 00 - 08000 0000 00: - +10000 0000 00 - 20000 0000 00: shadow memory (1024 GB) +20000 0000 00 - 24000 0000 00: metainfo (256 GB) +24000 0000 00 - 2aa00 0000 00: - +2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB) +2c000 0000 00 - 3c000 0000 00: - +3c000 0000 00 - 3f000 0000 00: heap (192 GB) +3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB) */ struct MappingAarch64_42 { - static const uptr kBroken = kBrokenReverseMapping; static const uptr kLoAppMemBeg = 0x00000001000ull; - static const uptr kLoAppMemEnd = 0x01000000000ull; - static const uptr kShadowBeg = 0x08000000000ull; - static const uptr kShadowEnd = 0x10000000000ull; - static const uptr kMetaShadowBeg = 0x26000000000ull; - static const uptr kMetaShadowEnd = 0x28000000000ull; + static const uptr kLoAppMemEnd = 0x02000000000ull; + static const uptr kShadowBeg = 0x10000000000ull; + static const uptr kShadowEnd = 0x20000000000ull; + static const uptr kMetaShadowBeg = 0x20000000000ull; + static const uptr kMetaShadowEnd = 0x24000000000ull; static const uptr kMidAppMemBeg = 0x2aa00000000ull; - static const uptr kMidAppMemEnd = 0x2ab00000000ull; - static const uptr kHeapMemBeg = 0x3e000000000ull; + static const uptr kMidAppMemEnd = 0x2c000000000ull; + static const uptr kHeapMemBeg = 0x3c000000000ull; static const uptr kHeapMemEnd = 0x3f000000000ull; static const uptr kHiAppMemBeg = 0x3f000000000ull; static const uptr kHiAppMemEnd = 0x3ffffffffffull; - static const uptr kShadowMsk = 0x3c000000000ull; - static const uptr kShadowXor = 0x04000000000ull; - static const uptr kShadowAdd = 0x00000000000ull; + static const uptr kShadowMsk = 0x38000000000ull; + static const uptr kShadowXor = 0x08000000000ull; + static const uptr kShadowAdd = 0x00000000000ull; static const uptr kVdsoBeg = 0x37f00000000ull; }; +/* +C/C++ on linux/aarch64 (48-bit VMA) +0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB) +0a00 0000 1000 - 1554 0000 0000: - +1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB) +5400 0000 1000 - 8000 0000 0000: - +8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB) +a000 0000 1000 - aaaa 0000 0000: - +aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB) +ac00 0000 1000 - fc00 0000 0000: - +fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB) + +N.B. the shadow memory region has a strange start address, because it +contains the shadows for the mid, high and low app regions (in this +unusual order). +*/ struct MappingAarch64_48 { static const uptr kLoAppMemBeg = 0x0000000001000ull; static const uptr kLoAppMemEnd = 0x0000200000000ull; @@ -759,13 +841,54 @@ uptr MetaShadowBeg(void) { return SelectMapping(kMetaShadowBeg); } ALWAYS_INLINE uptr MetaShadowEnd(void) { return SelectMapping(kMetaShadowEnd); } +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP +ALWAYS_INLINE +uptr LinearMappingWiden(uptr x, uptr fromL, uptr fromR, uptr toL, uptr toR) { + assert(fromL != fromR); + assert(toL != toR); + uptr fromDiff = (x - fromL); + // uptr toDiff = (toR - toL); + return (fromDiff << 1) + toL; +} + +ALWAYS_INLINE +uptr LinearMappingShrink(uptr x, uptr fromL, uptr fromR, uptr toL, uptr toR) { + assert(fromL != fromR); + assert(toL != toR); + uptr fromDiff = (x - fromL); + // uptr toDiff = (toR - toL); + return (fromDiff >> 1) + toL; +} +#endif + struct IsAppMemImpl { + + template + static bool IsHeapMem(uptr mem) { + return mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd; + } + + template + static bool IsLoAppMem(uptr mem) { + return mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd; + } + + template + static bool IsMidAppMem(uptr mem) { + return mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd; + } + + template + static bool IsHiAppMem(uptr mem) { + return mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd; + } + template static bool Apply(uptr mem) { - return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || - (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || - (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || - (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); + return IsHeapMem(mem) || + IsLoAppMem(mem) || + IsMidAppMem(mem) || + IsHiAppMem(mem); } }; @@ -796,15 +919,45 @@ bool IsMetaMem(const u32 *p) { return SelectMapping(reinterpret_cast(p)); } -struct MemToShadowImpl { +namespace { + template - static uptr Apply(uptr x) { + static uptr MemToShadowImplApply(uptr x) { DCHECK(IsAppMemImpl::Apply(x)); return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^ Mapping::kShadowXor) * kShadowMultiplier + Mapping::kShadowAdd; } + +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP + using M = MappingAarch64_39; + template <> + uptr MemToShadowImplApply(uptr x) { + DCHECK(IsAppMemImpl::Apply(x)); + #define CHECK_AND_RETURN_SHADOW(x, memType, shadowType) \ + if (IsAppMemImpl::Is##memType(x)) {\ + return LinearMappingWiden(x & ~(kShadowCell - 1),\ + M::k ## memType ## Beg,\ + M::k ## memType ## End,\ + M::k ## shadowType ## Beg,\ + M::k ## shadowType ## End);\ + } else {} + CHECK_AND_RETURN_SHADOW(x, HeapMem, HeapShadow); + CHECK_AND_RETURN_SHADOW(x, LoAppMem, LoShadow); + CHECK_AND_RETURN_SHADOW(x, MidAppMem, MidShadow); + CHECK_AND_RETURN_SHADOW(x, HiAppMem, HighShadow); + #undef CHECK_AND_RETURN_SHADOW + __builtin_unreachable(); + } +#endif +} + +struct MemToShadowImpl { + template + static uptr Apply(uptr x) { + return MemToShadowImplApply(x); + } }; ALWAYS_INLINE @@ -812,22 +965,53 @@ RawShadow *MemToShadow(uptr x) { return reinterpret_cast(SelectMapping(x)); } -struct MemToMetaImpl { +namespace { + template - static u32 *Apply(uptr x) { + u32 *MemToMetaImplApply(uptr x) { DCHECK(IsAppMemImpl::Apply(x)); return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) / kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg); } + +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP + using M = MappingAarch64_39; + template <> + u32 *MemToMetaImplApply(uptr x) { + DCHECK(IsAppMemImpl::Apply(x)); + #define CHECK_AND_RETURN_META(x, memType, shadowType) \ + if(IsAppMemImpl::Is##memType(x)) {\ + return (u32 *)LinearMappingShrink(x & ~(kMetaShadowCell - 1),\ + M::k ## memType ## Beg,\ + M::k ## memType ## End,\ + M::k ## shadowType ## Beg,\ + M::k ## shadowType ## End);\ + } else {} + CHECK_AND_RETURN_META(x, HeapMem, HeapMetaShadow); + CHECK_AND_RETURN_META(x, LoAppMem, LoMetaShadow); + CHECK_AND_RETURN_META(x, MidAppMem, MidMetaShadow); + CHECK_AND_RETURN_META(x, HiAppMem, HighMetaShadow); + #undef CHECK_AND_RETURN_META + __builtin_unreachable(); + } +#endif +} + +struct MemToMetaImpl { + template + static u32 *Apply(uptr x) { + return MemToMetaImplApply(x); + } }; ALWAYS_INLINE u32 *MemToMeta(uptr x) { return SelectMapping(x); } -struct ShadowToMemImpl { + +namespace { template - static uptr Apply(uptr sp) { + uptr ShadowToMemImplApply(uptr sp) { if (!IsShadowMemImpl::Apply(sp)) return 0; // The shadow mapping is non-linear and we've lost some bits, so we don't @@ -848,6 +1032,40 @@ struct ShadowToMemImpl { } return p | Mapping::kShadowMsk; } + +#ifdef AARCH64_TSAN_REQUIRE_BIGGER_HEAP + using M = MappingAarch64_39; + template <> + uptr ShadowToMemImplApply(uptr sp) { + if (!IsShadowMemImpl::Apply(sp)) + return 0; + + #define CHECK_AND_MAP_TO_APP(x, shadowType, memType) \ + if (M::k##shadowType##Beg <= sp &&\ + sp < M::k##shadowType##End) {\ + return LinearMappingShrink(x, \ + M::k ## shadowType ## Beg,\ + M::k ## shadowType ## End,\ + M::k ## memType ## Beg,\ + M::k ## memType ## End);\ + } else {} + CHECK_AND_MAP_TO_APP(sp, HeapShadow, HeapMem); + CHECK_AND_MAP_TO_APP(sp, LoShadow, LoAppMem); + CHECK_AND_MAP_TO_APP(sp, MidShadow, MidAppMem); + CHECK_AND_MAP_TO_APP(sp, HighShadow, HiAppMem); + #undef CHECK_AND_MAP_TO_APP + __builtin_unreachable(); + } +#endif +} + +struct ShadowToMemImpl { + + template + static uptr Apply(uptr sp) { + return ShadowToMemImplApply(sp); + } + }; ALWAYS_INLINE