From d1f50c938bd5ac287a1d53d9ef4f73ff9aec29dc Mon Sep 17 00:00:00 2001 From: guojunding Date: Fri, 20 Sep 2024 15:41:52 +0800 Subject: [PATCH] Backport two patchea --- ...ssorRenewExtendPileup-race-condition.patch | 161 ++++++++++++++++++ ...when-validation-on-conf-change-faile.patch | 38 +++++ etcd.spec | 10 +- 3 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 0014-Backport-TestLessorRenewExtendPileup-race-condition.patch create mode 100644 0015-backport-print-error-log-when-validation-on-conf-change-faile.patch diff --git a/0014-Backport-TestLessorRenewExtendPileup-race-condition.patch b/0014-Backport-TestLessorRenewExtendPileup-race-condition.patch new file mode 100644 index 0000000..61ef4d2 --- /dev/null +++ b/0014-Backport-TestLessorRenewExtendPileup-race-condition.patch @@ -0,0 +1,161 @@ +From 06216228c3af8998940bdb175ef9c80a83b64d17 Mon Sep 17 00:00:00 2001 +From: Lucas Rodriguez +Date: Mon, 9 Sep 2024 12:40:10 -0500 +Subject: [PATCH] Backport TestLessorRenewExtendPileup race condition fix for + release-3.4 + +Signed-off-by: Lucas Rodriguez + +--- + lease/lessor.go | 21 ++++++++++++++++----- + lease/lessor_test.go | 19 +++++++++---------- + 2 files changed, 25 insertions(+), 15 deletions(-) + +diff --git a/lease/lessor.go b/lease/lessor.go +index b16099f..a9fdd6c 100644 +--- a/lease/lessor.go ++++ b/lease/lessor.go +@@ -41,8 +41,8 @@ var ( + + leaseBucketName = []byte("lease") + +- // maximum number of leases to revoke per second; configurable for tests +- leaseRevokeRate = 1000 ++ // default number of leases to revoke per second; configurable for tests ++ defaultLeaseRevokeRate = 1000 + + // maximum number of lease checkpoints recorded to the consensus log per second; configurable for tests + leaseCheckpointRate = 1000 +@@ -169,6 +169,9 @@ type lessor struct { + // requests for shorter TTLs are extended to the minimum TTL. + minLeaseTTL int64 + ++ // maximum number of leases to revoke per second ++ leaseRevokeRate int ++ + expiredC chan []*Lease + // stopC is a channel whose closure indicates that the lessor should be stopped. + stopC chan struct{} +@@ -187,6 +190,8 @@ type LessorConfig struct { + MinLeaseTTL int64 + CheckpointInterval time.Duration + ExpiredLeasesRetryInterval time.Duration ++ ++ leaseRevokeRate int + } + + func NewLessor(lg *zap.Logger, b backend.Backend, cfg LessorConfig) Lessor { +@@ -196,12 +201,16 @@ func NewLessor(lg *zap.Logger, b backend.Backend, cfg LessorConfig) Lessor { + func newLessor(lg *zap.Logger, b backend.Backend, cfg LessorConfig) *lessor { + checkpointInterval := cfg.CheckpointInterval + expiredLeaseRetryInterval := cfg.ExpiredLeasesRetryInterval ++ leaseRevokeRate := cfg.leaseRevokeRate + if checkpointInterval == 0 { + checkpointInterval = defaultLeaseCheckpointInterval + } + if expiredLeaseRetryInterval == 0 { + expiredLeaseRetryInterval = defaultExpiredleaseRetryInterval + } ++ if leaseRevokeRate == 0 { ++ leaseRevokeRate = defaultLeaseRevokeRate ++ } + l := &lessor{ + leaseMap: make(map[LeaseID]*Lease), + itemMap: make(map[LeaseItem]LeaseID), +@@ -209,6 +218,7 @@ func newLessor(lg *zap.Logger, b backend.Backend, cfg LessorConfig) *lessor { + leaseCheckpointHeap: make(LeaseQueue, 0), + b: b, + minLeaseTTL: cfg.MinLeaseTTL, ++ leaseRevokeRate: leaseRevokeRate, + checkpointInterval: checkpointInterval, + expiredLeaseRetryInterval: expiredLeaseRetryInterval, + // expiredC is a small buffered chan to avoid unnecessary blocking. +@@ -449,7 +459,7 @@ func (le *lessor) Promote(extend time.Duration) { + le.leaseExpiredNotifier.RegisterOrUpdate(item) + } + +- if len(le.leaseMap) < leaseRevokeRate { ++ if len(le.leaseMap) < le.leaseRevokeRate { + // no possibility of lease pile-up + return + } +@@ -463,7 +473,7 @@ func (le *lessor) Promote(extend time.Duration) { + expires := 0 + // have fewer expires than the total revoke rate so piled up leases + // don't consume the entire revoke limit +- targetExpiresPerSecond := (3 * leaseRevokeRate) / 4 ++ targetExpiresPerSecond := (3 * le.leaseRevokeRate) / 4 + for _, l := range leases { + remaining := l.Remaining() + if remaining > nextWindow { +@@ -487,6 +497,7 @@ func (le *lessor) Promote(extend time.Duration) { + le.leaseExpiredNotifier.RegisterOrUpdate(item) + le.scheduleCheckpointIfNeeded(l) + } ++ + } + + type leasesByExpiry []*Lease +@@ -602,7 +613,7 @@ func (le *lessor) revokeExpiredLeases() { + var ls []*Lease + + // rate limit +- revokeLimit := leaseRevokeRate / 2 ++ revokeLimit := le.leaseRevokeRate / 2 + + le.mu.RLock() + if le.isPrimary() { +diff --git a/lease/lessor_test.go b/lease/lessor_test.go +index c79c32e..66497c7 100644 +--- a/lease/lessor_test.go ++++ b/lease/lessor_test.go +@@ -283,17 +283,16 @@ func TestLessorRenewWithCheckpointer(t *testing.T) { + // TestLessorRenewExtendPileup ensures Lessor extends leases on promotion if too many + // expire at the same time. + func TestLessorRenewExtendPileup(t *testing.T) { +- oldRevokeRate := leaseRevokeRate +- defer func() { leaseRevokeRate = oldRevokeRate }() ++ leaseRevokeRate := 10 + lg := zap.NewNop() +- leaseRevokeRate = 10 ++ + + dir, be := NewTestBackend(t) + defer os.RemoveAll(dir) + +- le := newLessor(lg, be, LessorConfig{MinLeaseTTL: minLeaseTTL}) ++ le := newLessor(lg, be, LessorConfig{MinLeaseTTL: minLeaseTTL,leaseRevokeRate: leaseRevokeRate}) + ttl := int64(10) +- for i := 1; i <= leaseRevokeRate*10; i++ { ++ for i := 1; i <= le.leaseRevokeRate*10; i++ { + if _, err := le.Grant(LeaseID(2*i), ttl); err != nil { + t.Fatal(err) + } +@@ -310,7 +309,7 @@ func TestLessorRenewExtendPileup(t *testing.T) { + bcfg.Path = filepath.Join(dir, "be") + be = backend.New(bcfg) + defer be.Close() +- le = newLessor(lg, be, LessorConfig{MinLeaseTTL: minLeaseTTL}) ++ le = newLessor(lg, be, LessorConfig{MinLeaseTTL: minLeaseTTL,leaseRevokeRate: leaseRevokeRate}) + defer le.Stop() + + // extend after recovery should extend expiration on lease pile-up +@@ -325,11 +324,11 @@ func TestLessorRenewExtendPileup(t *testing.T) { + + for i := ttl; i < ttl+20; i++ { + c := windowCounts[i] +- if c > leaseRevokeRate { +- t.Errorf("expected at most %d expiring at %ds, got %d", leaseRevokeRate, i, c) ++ if c > le.leaseRevokeRate { ++ t.Errorf("expected at most %d expiring at %ds, got %d", le.leaseRevokeRate, i, c) + } +- if c < leaseRevokeRate/2 { +- t.Errorf("expected at least %d expiring at %ds, got %d", leaseRevokeRate/2, i, c) ++ if c < le.leaseRevokeRate/2 { ++ t.Errorf("expected at least %d expiring at %ds, got %d", le.leaseRevokeRate/2, i, c) + } + } + } +-- +2.9.3.windows.1 + diff --git a/0015-backport-print-error-log-when-validation-on-conf-change-faile.patch b/0015-backport-print-error-log-when-validation-on-conf-change-faile.patch new file mode 100644 index 0000000..4211a38 --- /dev/null +++ b/0015-backport-print-error-log-when-validation-on-conf-change-faile.patch @@ -0,0 +1,38 @@ +From f0276cb4fa4e3f509783df28f9ed1038e3c2af15 Mon Sep 17 00:00:00 2001 +From: Benjamin Wang +Date: Thu, 6 Jun 2024 19:34:44 +0100 +Subject: [PATCH] print error log when validation on conf change failed + +Signed-off-by: Benjamin Wang + +--- + etcdserver/server.go | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/etcdserver/server.go b/etcdserver/server.go +index a341625..c58c533 100644 +--- a/etcdserver/server.go ++++ b/etcdserver/server.go +@@ -2258,13 +2258,18 @@ func (s *EtcdServer) applyEntryNormal(e *raftpb.Entry) { + // applyConfChange applies a ConfChange to the server. It is only + // invoked with a ConfChange that has already passed through Raft + func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, confState *raftpb.ConfState) (bool, error) { ++ lg := s.getLogger() + if err := s.cluster.ValidateConfigurationChange(cc); err != nil { ++ if lg != nil { ++ lg.Error("Validation on configuration change failed", zap.Error(err)) ++ } else { ++ plog.Errorf("Validation on configuration change failed: %v", err) ++ } + cc.NodeID = raft.None + s.r.ApplyConfChange(cc) + return false, err + } + +- lg := s.getLogger() + *confState = *s.r.ApplyConfChange(cc) + switch cc.Type { + case raftpb.ConfChangeAddNode, raftpb.ConfChangeAddLearnerNode: +-- +2.9.3.windows.1 + diff --git a/etcd.spec b/etcd.spec index b25400c..92be437 100644 --- a/etcd.spec +++ b/etcd.spec @@ -31,7 +31,7 @@ system.} %global gosupfiles integration/fixtures/* etcdserver/api/v2http/testdata/* Name: etcd -Release: 15 +Release: 16 Summary: Distributed reliable key-value store for the most critical data of a distributed system # Upstream license specification: Apache-2.0 @@ -57,6 +57,8 @@ Patch10: 0010-backport-Suppress-noisy-basic-auth-token-deletion-log.patch Patch11: 0011-backport-Fix-wait-time-docs-tests-to-indicate-trigger-deadline.patch Patch12: 0012-backport-ignore-raft-messages-if-member-id-mismatch.patch Patch13: 0013-backport-Update-the-compaction-log-when-bootstrap-and-update-compacts-signature.patch +Patch14: 0014-backport-TestLessorRenewExtendPileup-race-condition.patch +Patch15: 0015-backport-print-error-log-when-validation-on-conf-change-faile.patch BuildRequires: golang BuildRequires: python3-devel %{?systemd_requires} @@ -83,6 +85,8 @@ Requires(pre): shadow-utils %patch11 -p1 %patch12 -p1 %patch13 -p1 +%patch14 -p1 +%patch15 -p1 %ifarch sw_64 %patch3 -p1 %endif @@ -171,6 +175,10 @@ getent passwd %{name} >/dev/null || useradd -r -g %{name} -d %{_sharedstatedir}/ %endif %changelog +* Fri Sep 20 2024 guojunding - 3.4.14-16 +- Backport TestLessorRenewExtendPileup race condition +- Print error log when validation on conf change failed + * Tue Sep 3 2024 guojunding - 3.4.14-15 - Fix server: ignore raft messages if member id - Update the compaction log when bootstrap and update compact's signature -- Gitee