diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go index 597a196ba8c45b20b0164fa458ea0da6eb565b70..d243ebb7cd583415c8a6354dfd5d1217092cb9e2 100644 --- a/src/cmd/compile/internal/ssagen/ssa.go +++ b/src/cmd/compile/internal/ssagen/ssa.go @@ -4244,36 +4244,41 @@ func InitTables() { makeAtomicGuardedIntrinsicARM64 := func(op0, op1 ssa.Op, typ, rtyp types.Kind, emit atomicOpEmitter) intrinsicBuilder { return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { - // Target Atomic feature is identified by dynamic detection - addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARM64HasATOMICS, s.sb) - v := s.load(types.Types[types.TBOOL], addr) - b := s.endBlock() - b.Kind = ssa.BlockIf - b.SetControl(v) - bTrue := s.f.NewBlock(ssa.BlockPlain) - bFalse := s.f.NewBlock(ssa.BlockPlain) - bEnd := s.f.NewBlock(ssa.BlockPlain) - b.AddEdgeTo(bTrue) - b.AddEdgeTo(bFalse) - b.Likely = ssa.BranchLikely - - // We have atomic instructions - use it directly. - s.startBlock(bTrue) - emit(s, n, args, op1, typ) - s.endBlock().AddEdgeTo(bEnd) - - // Use original instruction sequence. - s.startBlock(bFalse) - emit(s, n, args, op0, typ) - s.endBlock().AddEdgeTo(bEnd) - - // Merge results. - s.startBlock(bEnd) + if buildcfg.GOARM64.KPAtomicOpt { + emit(s, n, args, op0, typ) + } else { + // Target Atomic feature is identified by dynamic detection + addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARM64HasATOMICS, s.sb) + v := s.load(types.Types[types.TBOOL], addr) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.SetControl(v) + bTrue := s.f.NewBlock(ssa.BlockPlain) + bFalse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + b.AddEdgeTo(bTrue) + b.AddEdgeTo(bFalse) + b.Likely = ssa.BranchLikely + + // We have atomic instructions - use it directly. + s.startBlock(bTrue) + emit(s, n, args, op1, typ) + s.endBlock().AddEdgeTo(bEnd) + + // Use original instruction sequence. + s.startBlock(bFalse) + emit(s, n, args, op0, typ) + s.endBlock().AddEdgeTo(bEnd) + + // Merge results. + s.startBlock(bEnd) + } if rtyp == types.TNIL { return nil } else { return s.variable(n, types.Types[rtyp]) } + } } diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index 8973a8716828160cf57b0172b161713454be9325..ce573686ec8f590bf66bb5556654ba8f3f533df4 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -33,6 +33,7 @@ var ( gohostos string goos string goarm string + goarm64 string go386 string goamd64 string gomips string @@ -147,6 +148,12 @@ func xinit() { } goarm = b + b = os.Getenv("GOARM64") + if b == "" { + b = "v8.0" + } + goarm64 = b + b = os.Getenv("GO386") if b == "" { b = "sse2" @@ -230,6 +237,7 @@ func xinit() { os.Setenv("GOAMD64", goamd64) os.Setenv("GOARCH", goarch) os.Setenv("GOARM", goarm) + os.Setenv("GOARM64", goarm64) os.Setenv("GOHOSTARCH", gohostarch) os.Setenv("GOHOSTOS", gohostos) os.Setenv("GOOS", goos) @@ -1227,6 +1235,9 @@ func cmdenv() { if goarch == "arm" { xprintf(format, "GOARM", goarm) } + if goarch == "arm64" { + xprintf(format, "GOARM64", goarm64) + } if goarch == "386" { xprintf(format, "GO386", go386) } diff --git a/src/cmd/dist/buildruntime.go b/src/cmd/dist/buildruntime.go index 1de78f0fdb2eb10f5384a49081de95d9021c565d..816b944400a83a4803dd887eae565c4080bdff10 100644 --- a/src/cmd/dist/buildruntime.go +++ b/src/cmd/dist/buildruntime.go @@ -54,6 +54,7 @@ func mkbuildcfg(file string) { fmt.Fprintf(&buf, "const defaultGO386 = `%s`\n", go386) fmt.Fprintf(&buf, "const defaultGOAMD64 = `%s`\n", goamd64) fmt.Fprintf(&buf, "const defaultGOARM = `%s`\n", goarm) + fmt.Fprintf(&buf, "const defaultGOARM64 = `%s`\n", goarm64) fmt.Fprintf(&buf, "const defaultGOMIPS = `%s`\n", gomips) fmt.Fprintf(&buf, "const defaultGOMIPS64 = `%s`\n", gomips64) fmt.Fprintf(&buf, "const defaultGOPPC64 = `%s`\n", goppc64) diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 8caa22a93df91f42306a9f782e8659e5b01890f6..3b591a17d0432f656e4805d27d4846c7bcbd5f29 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -410,6 +410,7 @@ var ( // Used in envcmd.MkEnv and build ID computations. GOARM = envOr("GOARM", fmt.Sprint(buildcfg.GOARM)) + GOARM64 = envOr("GOARM64", fmt.Sprint(buildcfg.GOARM64)) GO386 = envOr("GO386", buildcfg.GO386) GOAMD64 = envOr("GOAMD64", fmt.Sprintf("%s%d", "v", buildcfg.GOAMD64)) GOMIPS = envOr("GOMIPS", buildcfg.GOMIPS) @@ -436,6 +437,8 @@ func GetArchEnv() (key, val string) { switch Goarch { case "arm": return "GOARM", GOARM + case "arm64": + return "GOARM64", GOARM64 case "386": return "GO386", GO386 case "amd64": diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 26b4e0f490df3fa3df5c54bd05724b4080b5e9d4..f682219b3bed01a461c8f4b219b76afa7f60ff6a 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "fmt" + "internal/buildcfg" "internal/platform" "io" "log" @@ -398,6 +399,16 @@ func asmArgs(a *Action, p *load.Package) []any { } } + if cfg.Goarch == "arm64" { + g, err := buildcfg.ParseGoarm64(cfg.GOARM64) + if err == nil && g.LSE { + args = append(args, "-D", "GOARM64_LSE") + } + if err == nil && g.KPAtomicOpt { + args = append(args, "-D", "KPAtomicOpt") + } + } + return args } diff --git a/src/internal/buildcfg/cfg.go b/src/internal/buildcfg/cfg.go index b97b9c1b53a91e221bcbb42ef6007876f845bb6e..e7469a1a9fee957503cb56f3329cff686de132a4 100644 --- a/src/internal/buildcfg/cfg.go +++ b/src/internal/buildcfg/cfg.go @@ -27,6 +27,7 @@ var ( GO386 = envOr("GO386", defaultGO386) GOAMD64 = goamd64() GOARM = goarm() + GOARM64 = goarm64() GOMIPS = gomips() GOMIPS64 = gomips64() GOPPC64 = goppc64() @@ -87,6 +88,100 @@ func goarm() int { return int(def[0] - '0') } +type Goarm64Features struct { + Version string + // Large System Extension + LSE bool + // Kunpeng atomic optimize + KPAtomicOpt bool +} + +func (g Goarm64Features) String() string { + arm64Str := g.Version + if g.LSE { + arm64Str += ",lse" + } + + if g.KPAtomicOpt { + arm64Str += ",kpatomicopt" + } + + return arm64Str +} + +func ParseGoarm64(v string) (g Goarm64Features, e error) { + const ( + lseOpt = ",lse" + kpAtomicOpt = ",kpatomicopt" + ) + + g.LSE = false + g.KPAtomicOpt = false + + // We allow any combination of suffixes, in any order + for { + if strings.HasSuffix(v, lseOpt) { + g.LSE = true + v = v[:len(v)-len(lseOpt)] + continue + } + + if strings.HasSuffix(v, kpAtomicOpt) { + if os.Getenv("KP_AI_OPT") == "1" { + g.KPAtomicOpt = true + } + v = v[:len(v)-len(kpAtomicOpt)] + continue + } + + break + } + + switch v { + case "v8.0", "v8.1", "v8.2", "v8.3", "v8.4", "v8.5", "v8.6", "v8.7", "v8.8", "v8.9", + "v9.0", "v9.1", "v9.2", "v9.4", "v9.5": + g.Version = v + default: + e = fmt.Errorf("invalid GOARM64: must start with v8.{0-9} or v9.{0-5} and may optionally end in %q and/or %q", + lseOpt, kpAtomicOpt) + g.Version = defaultGOARM64 + } + + return +} + +func goarm64() (g Goarm64Features) { + g, Error = ParseGoarm64(envOr("GOARM64", defaultGOARM64)) + return +} + +func (g Goarm64Features) Supports(s string) bool { + // We only accept "v{8-9}.{0-9}. Everything else is malformed. + if len(s) != 4 { + return false + } + + major := s[1] + minor := s[3] + + if major < '8' || major > '9' || + minor < '0' || minor > '9' || + s[0] != 'v' || s[2] != '.' { + return false + } + + g_major := g.Version[1] + g_minor := g.Version[3] + + if major == g_major { + return minor <= g_minor + } else if g_major == '9' { + return minor <= g_minor+5 + } else { + return false + } +} + func gomips() string { switch v := envOr("GOMIPS", defaultGOMIPS); v { case "hardfloat", "softfloat": @@ -183,6 +278,8 @@ func GOGOARCH() (name, value string) { return "GOAMD64", fmt.Sprintf("v%d", GOAMD64) case "arm": return "GOARM", strconv.Itoa(GOARM) + case "arm64": + return "GOARM64", GOARM64.String() case "mips", "mipsle": return "GOMIPS", GOMIPS case "mips64", "mips64le": @@ -211,6 +308,19 @@ func gogoarchTags() []string { list = append(list, fmt.Sprintf("%s.%d", GOARCH, i)) } return list + case "arm64": + var list []string + major := int(GOARM64.Version[1] - '0') + minor := int(GOARM64.Version[3] - '0') + for i := 0; i <= minor; i++ { + list = append(list, fmt.Sprintf("%s.v%d.%d", GOARCH, major, i)) + } + if major == 9 { + for i := 0; i <= minor+5 && i <= 9; i++ { + list = append(list, fmt.Sprint("%s.v%d.%d", GOARCH, 8, i)) + } + } + return list case "mips", "mipsle": return []string{GOARCH + "." + GOMIPS} case "mips64", "mips64le": diff --git a/src/internal/cfg/cfg.go b/src/internal/cfg/cfg.go index 2af0ec707868947bda6c966a3156625241db12c7..7ef5bb7be67411ddc4eebe82f90ceb17afe61a77 100644 --- a/src/internal/cfg/cfg.go +++ b/src/internal/cfg/cfg.go @@ -36,6 +36,7 @@ const KnownEnv = ` GOAMD64 GOARCH GOARM + GOARM64 GOBIN GOCACHE GOCACHEPROG diff --git a/src/reflect/asm_arm64.s b/src/reflect/asm_arm64.s index 5e91e62aa145251afc8058150ef64b9f1508f265..23677a9e6d36ffdd53d2ba73b7966130a8129ff8 100644 --- a/src/reflect/asm_arm64.s +++ b/src/reflect/asm_arm64.s @@ -19,6 +19,13 @@ #define LOCAL_RETVALID 40 #define LOCAL_REGARGS 48 +GLOBL ·kpAtomicOpt(SB),(NOPTR|WRAPPER),$1 +#ifndef KPAtomicOpt +DATA ·kpAtomicOpt(SB)/1,$0 +#else +DATA ·kpAtomicOpt(SB)/1,$1 +#endif + // The frame size of the functions below is // 32 (args of callReflect) + 8 (bool + padding) + 392 (abi.RegArgs) = 432. diff --git a/src/reflect/type.go b/src/reflect/type.go index 9fd242e732787e362fb2cffcfc1b485053c31a3d..b943352d3c8aee925bed56ebfc1a150e30aa8a75 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -1714,9 +1714,18 @@ func MapOf(key, elem Type) Type { } var funcTypes []Type -var funcTypesMutex sync.Mutex +var funcTypesMutex sync.RWMutex +var kpAtomicOpt bool func initFuncTypes(n int) Type { + if kpAtomicOpt { + funcTypesMutex.RLock() + if n < len(funcTypes) && funcTypes[n] != nil { + defer funcTypesMutex.RUnlock() + return funcTypes[n] + } + funcTypesMutex.RUnlock() + } funcTypesMutex.Lock() defer funcTypesMutex.Unlock() if n >= len(funcTypes) { diff --git a/src/runtime/internal/atomic/atomic_arm64.s b/src/runtime/internal/atomic/atomic_arm64.s index 5f77d92deba9378fa06f8558d3fbd9a7ca065a73..aadb2f85e5b36dc5cb6cd85c54e8a61852384942 100644 --- a/src/runtime/internal/atomic/atomic_arm64.s +++ b/src/runtime/internal/atomic/atomic_arm64.s @@ -128,17 +128,21 @@ TEXT ·Store64(SB), NOSPLIT, $0-16 TEXT ·Xchg(SB), NOSPLIT, $0-20 MOVD ptr+0(FP), R0 MOVW new+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif SWPALW R1, (R0), R2 MOVW R2, ret+16(FP) RET +#ifndef GOARM64_LSE load_store_loop: LDAXRW (R0), R2 STLXRW R1, (R0), R3 CBNZ R3, load_store_loop MOVW R2, ret+16(FP) RET +#endif // uint64 Xchg64(ptr *uint64, new uint64) // Atomically: @@ -148,17 +152,21 @@ load_store_loop: TEXT ·Xchg64(SB), NOSPLIT, $0-24 MOVD ptr+0(FP), R0 MOVD new+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif SWPALD R1, (R0), R2 MOVD R2, ret+16(FP) RET +#ifndef GOARM64_LSE load_store_loop: LDAXR (R0), R2 STLXR R1, (R0), R3 CBNZ R3, load_store_loop MOVD R2, ret+16(FP) RET +#endif // bool Cas(uint32 *ptr, uint32 old, uint32 new) // Atomically: @@ -171,14 +179,17 @@ TEXT ·Cas(SB), NOSPLIT, $0-17 MOVD ptr+0(FP), R0 MOVW old+8(FP), R1 MOVW new+12(FP), R2 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif MOVD R1, R3 CASALW R3, (R0), R2 CMP R1, R3 CSET EQ, R0 MOVB R0, ret+16(FP) RET +#ifndef GOARM64_LSE load_store_loop: LDAXRW (R0), R3 CMPW R1, R3 @@ -189,6 +200,7 @@ ok: CSET EQ, R0 MOVB R0, ret+16(FP) RET +#endif // bool ·Cas64(uint64 *ptr, uint64 old, uint64 new) // Atomically: @@ -202,14 +214,17 @@ TEXT ·Cas64(SB), NOSPLIT, $0-25 MOVD ptr+0(FP), R0 MOVD old+8(FP), R1 MOVD new+16(FP), R2 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif MOVD R1, R3 CASALD R3, (R0), R2 CMP R1, R3 CSET EQ, R0 MOVB R0, ret+24(FP) RET +#ifndef GOARM64_LSE load_store_loop: LDAXR (R0), R3 CMP R1, R3 @@ -220,6 +235,7 @@ ok: CSET EQ, R0 MOVB R0, ret+24(FP) RET +#endif // uint32 xadd(uint32 volatile *ptr, int32 delta) // Atomically: @@ -228,12 +244,15 @@ ok: TEXT ·Xadd(SB), NOSPLIT, $0-20 MOVD ptr+0(FP), R0 MOVW delta+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif LDADDALW R1, (R0), R2 ADD R1, R2 MOVW R2, ret+16(FP) RET +#ifndef GOARM64_LSE load_store_loop: LDAXRW (R0), R2 ADDW R2, R1, R2 @@ -241,6 +260,7 @@ load_store_loop: CBNZ R3, load_store_loop MOVW R2, ret+16(FP) RET +#endif // uint64 Xadd64(uint64 volatile *ptr, int64 delta) // Atomically: @@ -249,12 +269,15 @@ load_store_loop: TEXT ·Xadd64(SB), NOSPLIT, $0-24 MOVD ptr+0(FP), R0 MOVD delta+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif LDADDALD R1, (R0), R2 ADD R1, R2 MOVD R2, ret+16(FP) RET +#ifndef GOARM64_LSE load_store_loop: LDAXR (R0), R2 ADD R2, R1, R2 @@ -262,6 +285,7 @@ load_store_loop: CBNZ R3, load_store_loop MOVD R2, ret+16(FP) RET +#endif TEXT ·Xchgint32(SB), NOSPLIT, $0-20 B ·Xchg(SB) @@ -275,59 +299,75 @@ TEXT ·Xchguintptr(SB), NOSPLIT, $0-24 TEXT ·And8(SB), NOSPLIT, $0-9 MOVD ptr+0(FP), R0 MOVB val+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif MVN R1, R2 LDCLRALB R2, (R0), R3 RET +#ifndef GOARM64_LSE load_store_loop: LDAXRB (R0), R2 AND R1, R2 STLXRB R2, (R0), R3 CBNZ R3, load_store_loop RET +#endif TEXT ·Or8(SB), NOSPLIT, $0-9 MOVD ptr+0(FP), R0 MOVB val+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif LDORALB R1, (R0), R2 RET +#ifndef GOARM64_LSE load_store_loop: LDAXRB (R0), R2 ORR R1, R2 STLXRB R2, (R0), R3 CBNZ R3, load_store_loop RET +#endif // func And(addr *uint32, v uint32) TEXT ·And(SB), NOSPLIT, $0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif MVN R1, R2 LDCLRALW R2, (R0), R3 RET +#ifndef GOARM64_LSE load_store_loop: LDAXRW (R0), R2 AND R1, R2 STLXRW R2, (R0), R3 CBNZ R3, load_store_loop RET +#endif // func Or(addr *uint32, v uint32) TEXT ·Or(SB), NOSPLIT, $0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 +#ifndef GOARM64_LSE MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4 CBZ R4, load_store_loop +#endif LDORALW R1, (R0), R2 RET +#ifndef GOARM64_LSE load_store_loop: LDAXRW (R0), R2 ORR R1, R2 STLXRW R2, (R0), R3 CBNZ R3, load_store_loop RET +#endif diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 44479cc2be262ccd81ebcf1c9e6144d3cbdac3fa..7b44e2372d2758d197b948a3c8381cb588628e3c 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -1258,6 +1258,8 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // Maybe just all noscan objects? x = add(x, size-dataSize) } + + sys.Prefetch(uintptr(unsafe.Add(x, size))) return x } diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index de5ae0ae00c4f1e6b12b104e98cb43704caa0555..8c3f98f3ca1d77394d384cd6b91e74c1bf2130f3 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -181,7 +181,7 @@ func gcinit() { // Initialize GC pacer state. // Use the environment variable GOGC for the initial gcPercent value. // Use the environment variable GOMEMLIMIT for the initial memoryLimit value. - gcController.init(readGOGC(), readGOMEMLIMIT()) + gcController.init(readGOGC(), readGOMEMLIMIT(), readGOGCRATIO()) work.startSema = 1 work.markDoneSema = 1 diff --git a/src/runtime/mgclimit.go b/src/runtime/mgclimit.go index ef3cc081cec3d2fce215ac7f104994c56573f98e..65bbaf56e0a98e27bd0e2e810dbeab337297f93c 100644 --- a/src/runtime/mgclimit.go +++ b/src/runtime/mgclimit.go @@ -229,7 +229,7 @@ func (l *gcCPULimiterState) updateLocked(now int64) { // Compute total GC time. windowGCTime := assistTime if l.gcEnabled { - windowGCTime += int64(float64(windowTotalTime) * gcBackgroundUtilization) + windowGCTime += int64(float64(windowTotalTime) * gcController.gcRatio) } // Subtract out all idle time from the total time. Do this after computing diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index 32e19f96e101b430c731439cdcdeeccb389705be..8e5799acec153d2d3f8fa1e9ec679a986ea8f5f8 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -12,30 +12,6 @@ import ( ) const ( - // gcGoalUtilization is the goal CPU utilization for - // marking as a fraction of GOMAXPROCS. - // - // Increasing the goal utilization will shorten GC cycles as the GC - // has more resources behind it, lessening costs from the write barrier, - // but comes at the cost of increasing mutator latency. - gcGoalUtilization = gcBackgroundUtilization - - // gcBackgroundUtilization is the fixed CPU utilization for background - // marking. It must be <= gcGoalUtilization. The difference between - // gcGoalUtilization and gcBackgroundUtilization will be made up by - // mark assists. The scheduler will aim to use within 50% of this - // goal. - // - // As a general rule, there's little reason to set gcBackgroundUtilization - // < gcGoalUtilization. One reason might be in mostly idle applications, - // where goroutines are unlikely to assist at all, so the actual - // utilization will be lower than the goal. But this is moot point - // because the idle mark workers already soak up idle CPU resources. - // These two values are still kept separate however because they are - // distinct conceptually, and in previous iterations of the pacer the - // distinction was more important. - gcBackgroundUtilization = 0.25 - // gcCreditSlack is the amount of scan work credit that can // accumulate locally before updating gcController.heapScanWork and, // optionally, gcController.bgScanCredit. Lower values give a more @@ -72,6 +48,14 @@ const ( // to maintain the memory limit. memoryLimitHeapGoalHeadroomPercent = 3 ) +// gcGoalUtilization is the goal CPU utilization for +// marking as a fraction of GOMAXPROCS. +// +// Increasing the goal utilization will shorten GC cycles as the GC +// has more resources behind it, lessening costs from the write barrier, +// but comes at the cost of increasing mutator latency. +var gcGoalUtilization = gcController.gcRatio + // gcController implements the GC pacing controller that determines // when to trigger concurrent garbage collection and how much marking @@ -88,6 +72,11 @@ const ( var gcController gcControllerState type gcControllerState struct { + // gcBackgroundUtilization be optional, value equals gcratio/100.0. + // Initialized from GOGCRATIO, which in the range of (1, 99). + // Default GOGCRATIO is 25. + gcRatio float64 + // Initialized from GOGC. GOGC=off means no GC. gcPercent atomic.Int32 @@ -366,11 +355,12 @@ type gcControllerState struct { _ cpu.CacheLinePad } -func (c *gcControllerState) init(gcPercent int32, memoryLimit int64) { +func (c *gcControllerState) init(gcPercent int32, memoryLimit int64, gcRatio float64) { c.heapMinimum = defaultHeapMinimum c.triggered = ^uint64(0) c.setGCPercent(gcPercent) c.setMemoryLimit(memoryLimit) + c.setGOGCRatio(gcRatio) c.commit(true) // No sweep phase in the first GC cycle. // N.B. Don't bother calling traceHeapGoal. Tracing is never enabled at // initialization time. @@ -398,7 +388,7 @@ func (c *gcControllerState) startCycle(markStartTime int64, procs int, trigger g // dedicated workers so that the utilization is closest to // 25%. For small GOMAXPROCS, this would introduce too much // error, so we add fractional workers in that case. - totalUtilizationGoal := float64(procs) * gcBackgroundUtilization + totalUtilizationGoal := float64(procs) * gcController.gcRatio dedicatedMarkWorkersNeeded := int64(totalUtilizationGoal + 0.5) utilError := float64(dedicatedMarkWorkersNeeded)/totalUtilizationGoal - 1 const maxUtilError = 0.3 @@ -604,7 +594,7 @@ func (c *gcControllerState) endCycle(now int64, procs int, userForced bool) { assistDuration := now - c.markStartTime // Assume background mark hit its utilization goal. - utilization := gcBackgroundUtilization + utilization := gcController.gcRatio // Add assist utilization; avoid divide by zero. if assistDuration > 0 { utilization += float64(c.assistTime.Load()) / float64(assistDuration*int64(procs)) @@ -1333,6 +1323,39 @@ func readGOMEMLIMIT() int64 { return n } +func (c *gcControllerState) setGOGCRatio(in float64) float64 { + if !c.test { + assertWorldStoppedOrLockHeld(&mheap_.lock) + } + + out := c.gcRatio + c.gcRatio = in + + return out +} + +func readGOGCRATIO() float64 { + p := gogetenv("GOGCRATIO") + if p == "" { + return 0.25 + } + n, ok := parseByteCount(p) + if !ok { + print("GOGCRATIO=", p, "\n") + throw("malformed GOGCRATIO; get the wrong value") + } + + if n < 1 { + n = 1 + } else if n > 99 { + n = 99 + } + + out := float64(n)/100.0 + + return out +} + // addIdleMarkWorker attempts to add a new idle mark worker. // // If this returns true, the caller must become an idle mark worker unless