From d1c5d907b3d3b3fc0482323ba2ef482b00eb68c2 Mon Sep 17 00:00:00 2001 From: zvier Date: Sat, 28 Nov 2020 11:20:04 +0800 Subject: [PATCH 1/2] sync patches from internal Signed-off-by: liuzekun --- VERSION-openeuler | 2 +- docker-engine-openeuler.spec | 22 +++++- ...e-containerd-object-on-start-failure.patch | 54 +++++++++++++ ...62-docker-remove-redundant-word-item.patch | 25 ++++++ ...-delete-event-is-not-need-to-process.patch | 27 +++++++ ...ess-exit-file-when-kill-process-dire.patch | 36 +++++++++ patch/0165-docker-sync-cli-vendor.patch | 38 +++++++++ patch/0167-docker-fix-CVE-2020-13401.patch | 69 ++++++++++++++++ .../0168-docker-do-not-add-w-to-LDFLAGS.patch | 79 +++++++++++++++++++ ...69-docker-add-files-in-proc-for-mask.patch | 42 ++++++++++ ...70-docker-fix-docker-load-files-leak.patch | 27 +++++++ ...-do-not-sync-if-BYPAAS_SYNC-is-false.patch | 28 +++++++ ...ix-panic-on-single-character-volumes.patch | 27 +++++++ ...fix-stats-memory-usage-display-error.patch | 38 +++++++++ ...-messages-for-ops-when-device-not-fo.patch | 27 +++++++ patch/0175-docker-mask-proc-pin_memory.patch | 25 ++++++ series.conf | 15 ++++ 17 files changed, 579 insertions(+), 2 deletions(-) create mode 100644 patch/0161-docker-Delete-stale-containerd-object-on-start-failure.patch create mode 100644 patch/0162-docker-remove-redundant-word-item.patch create mode 100644 patch/0163-docker-delete-event-is-not-need-to-process.patch create mode 100644 patch/0164-docker-stat-process-exit-file-when-kill-process-dire.patch create mode 100644 patch/0165-docker-sync-cli-vendor.patch create mode 100644 patch/0167-docker-fix-CVE-2020-13401.patch create mode 100644 patch/0168-docker-do-not-add-w-to-LDFLAGS.patch create mode 100644 patch/0169-docker-add-files-in-proc-for-mask.patch create mode 100644 patch/0170-docker-fix-docker-load-files-leak.patch create mode 100644 patch/0171-docker-do-not-sync-if-BYPAAS_SYNC-is-false.patch create mode 100644 patch/0172-docker-fix-panic-on-single-character-volumes.patch create mode 100644 patch/0173-docker-fix-stats-memory-usage-display-error.patch create mode 100644 patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch create mode 100644 patch/0175-docker-mask-proc-pin_memory.patch diff --git a/VERSION-openeuler b/VERSION-openeuler index 32c6fec..9ce0e89 100644 --- a/VERSION-openeuler +++ b/VERSION-openeuler @@ -1 +1 @@ -18.09.0.100 +18.09.0.102 diff --git a/docker-engine-openeuler.spec b/docker-engine-openeuler.spec index affba47..5e5cf89 100644 --- a/docker-engine-openeuler.spec +++ b/docker-engine-openeuler.spec @@ -1,6 +1,6 @@ Name: docker-engine Version: 18.09.0 -Release: 101 +Release: 102 Summary: The open-source application container engine Group: Tools/Docker @@ -200,3 +200,23 @@ fi %endif %changelog +* Sat Dec 28 2020 liuzekun - 18.09.0-102 +- Type:bugfix +- ID:NA +- CVE:NA +- SUG:restart +- DESC: +1.delete stale containerd object on start failure +2.remove redundant word item +3.delete event is not need to process +4.stat process exit file when kill process dire +5.sync cli vendor +6.fix CVE-2020-13401 +7.do not add w to LDFLAGS +8.add files in proc for mask +9.fix docker load files leak +10.do not sync if BYPAAS_SYNC is false +11.fix panic on single character volumes +12.fix stats memory usage display error +13.add more messages for ops when device not found +14.mask proc pin_memory diff --git a/patch/0161-docker-Delete-stale-containerd-object-on-start-failure.patch b/patch/0161-docker-Delete-stale-containerd-object-on-start-failure.patch new file mode 100644 index 0000000..8d8c1cc --- /dev/null +++ b/patch/0161-docker-Delete-stale-containerd-object-on-start-failure.patch @@ -0,0 +1,54 @@ +From 5ba30cd1dc6000ee53b34f628cbff91d7f6d7231 Mon Sep 17 00:00:00 2001 +From: Brian Goff +Date: Wed, 12 Dec 2018 12:04:09 -0800 +Subject: [PATCH] Delete stale containerd object on start failure + +containerd has two objects with regard to containers. +There is a "container" object which is metadata and a "task" which is +manging the actual runtime state. + +When docker starts a container, it creartes both the container metadata +and the task at the same time. So when a container exits, docker deletes +both of these objects as well. + +This ensures that if, on start, when we go to create the container metadata object +in containerd, if there is an error due to a name conflict that we go +ahead and clean that up and try again. + +Signed-off-by: Brian Goff +--- + components/engine/daemon/start.go | 17 +++++++++++++++-- + 1 file changed, 15 insertions(+), 2 deletions(-) + +diff --git a/components/engine/daemon/start.go b/components/engine/daemon/start.go +index 393e00b..57a7267 100644 +--- a/components/engine/daemon/start.go ++++ b/components/engine/daemon/start.go +@@ -177,9 +177,22 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint + return err + } + +- err = daemon.containerd.Create(context.Background(), container.ID, spec, createOptions) ++ ctx := context.TODO() ++ ++ err = daemon.containerd.Create(ctx, container.ID, spec, createOptions) + if err != nil { +- return translateContainerdStartErr(container.Path, container.SetExitCode, err) ++ if errdefs.IsConflict(err) { ++ logrus.WithError(err).WithField("container", container.ID).Error("Container not cleaned up from containerd from previous run") ++ // best effort to clean up old container object ++ daemon.containerd.DeleteTask(ctx, container.ID) ++ if err := daemon.containerd.Delete(ctx, container.ID); err != nil && !errdefs.IsNotFound(err) { ++ logrus.WithError(err).WithField("container", container.ID).Error("Error cleaning up stale containerd container object") ++ } ++ err = daemon.containerd.Create(ctx, container.ID, spec, createOptions) ++ } ++ if err != nil { ++ return translateContainerdStartErr(container.Path, container.SetExitCode, err) ++ } + } + + // TODO(mlaventure): we need to specify checkpoint options here +-- +1.8.3.1 + diff --git a/patch/0162-docker-remove-redundant-word-item.patch b/patch/0162-docker-remove-redundant-word-item.patch new file mode 100644 index 0000000..69de191 --- /dev/null +++ b/patch/0162-docker-remove-redundant-word-item.patch @@ -0,0 +1,25 @@ +From 92266f008637a02ebffa2aa2704a09701b07a405 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Mon, 27 Apr 2020 09:43:21 +0800 +Subject: [PATCH] docker: remove redundant word item + +Signed-off-by: liuzekun +--- + components/cli/vendor/github.com/asaskevich/govalidator/types.go | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/components/cli/vendor/github.com/asaskevich/govalidator/types.go b/components/cli/vendor/github.com/asaskevich/govalidator/types.go +index 4f7e9274..78be68c6 100644 +--- a/components/cli/vendor/github.com/asaskevich/govalidator/types.go ++++ b/components/cli/vendor/github.com/asaskevich/govalidator/types.go +@@ -370,7 +370,6 @@ var ISO3166List = []ISO3166Entry{ + {"Spain", "Espagne (l')", "ES", "ESP", "724"}, + {"South Sudan", "Soudan du Sud (le)", "SS", "SSD", "728"}, + {"Sudan (the)", "Soudan (le)", "SD", "SDN", "729"}, +- {"Western Sahara*", "Sahara occidental (le)*", "EH", "ESH", "732"}, + {"Suriname", "Suriname (le)", "SR", "SUR", "740"}, + {"Svalbard and Jan Mayen", "Svalbard et l'Île Jan Mayen (le)", "SJ", "SJM", "744"}, + {"Swaziland", "Swaziland (le)", "SZ", "SWZ", "748"}, +-- +2.19.1 + diff --git a/patch/0163-docker-delete-event-is-not-need-to-process.patch b/patch/0163-docker-delete-event-is-not-need-to-process.patch new file mode 100644 index 0000000..fef6e36 --- /dev/null +++ b/patch/0163-docker-delete-event-is-not-need-to-process.patch @@ -0,0 +1,27 @@ +From 0fe29ca9d45ddcb36f009a8da5f858f49a8e2844 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Wed, 8 Apr 2020 17:32:03 +0800 +Subject: [PATCH] docker: delete event is not need to access processEvent + +reason: delete event is not need to access processEvent, continue it + +Signed-off-by: liuzekun +--- + components/engine/libcontainerd/client_daemon.go | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/components/engine/libcontainerd/client_daemon.go b/components/engine/libcontainerd/client_daemon.go +index 858d6429..05c439c5 100755 +--- a/components/engine/libcontainerd/client_daemon.go ++++ b/components/engine/libcontainerd/client_daemon.go +@@ -895,6 +895,7 @@ func (c *client) processEventStream(ctx context.Context, ns string) { + "ExitStatus": t.ExitStatus, + "ExitedAt": t.ExitedAt, + }).Infof("event") ++ continue + default: + c.logger.WithFields(logrus.Fields{ + "topic": ev.Topic, +-- +2.19.1 + diff --git a/patch/0164-docker-stat-process-exit-file-when-kill-process-dire.patch b/patch/0164-docker-stat-process-exit-file-when-kill-process-dire.patch new file mode 100644 index 0000000..4fbb7de --- /dev/null +++ b/patch/0164-docker-stat-process-exit-file-when-kill-process-dire.patch @@ -0,0 +1,36 @@ +From 3f285224ade14c9d64dfc81cf9b5d969343a641e Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Wed, 8 Apr 2020 19:49:38 +0800 +Subject: [PATCH] docker: stat process exit file when kill process directly + +reason: stat process exit file when kill process directly + +Signed-off-by: liuzekun +--- + components/engine/daemon/container_operations_unix.go | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/components/engine/daemon/container_operations_unix.go b/components/engine/daemon/container_operations_unix.go +index 2cc2b2e3..df2f3261 100644 +--- a/components/engine/daemon/container_operations_unix.go ++++ b/components/engine/daemon/container_operations_unix.go +@@ -346,6 +346,16 @@ func killProcessDirectly(cntr *container.Container) error { + // Ensure that we don't kill ourselves + if pid := cntr.GetPID(); pid != 0 { + logrus.Infof("Container %s failed to exit within 10 seconds of kill - trying direct SIGKILL", stringid.TruncateID(cntr.ID)) ++ pattern := fmt.Sprintf("/var/run/docker/containerd/exit/moby/%s.%d.*", cntr.ID, pid) ++ efiles, err := filepath.Glob(pattern) ++ if err != nil { ++ logrus.Warnf("Match exit file with pattern %q failed: %s", pattern, err.Error()) ++ } ++ if len(efiles) != 0 { ++ logrus.Infof("Find process exit files with pattern %q: %+v, skip force kill because the process is exit already", pattern, efiles) ++ return errNoSuchProcess{pid, 9} ++ } ++ + if err := unix.Kill(pid, 9); err != nil { + if err != unix.ESRCH { + return err +-- +2.19.1 + diff --git a/patch/0165-docker-sync-cli-vendor.patch b/patch/0165-docker-sync-cli-vendor.patch new file mode 100644 index 0000000..cfdcee2 --- /dev/null +++ b/patch/0165-docker-sync-cli-vendor.patch @@ -0,0 +1,38 @@ +From c4ead7f7e914244e43eab849cf68c34c3460b41c Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Thu, 14 May 2020 22:57:37 +0800 +Subject: [PATCH] docker: sync cli vendor + +Change-Id: I9dbfd3e2c918d47806abdcdc27bf709c0e297780 +Signed-off-by: jingrui +--- + .../docker/docker/builder/remotecontext/git/gitutils.go | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/components/cli/vendor/github.com/docker/docker/builder/remotecontext/git/gitutils.go b/components/cli/vendor/github.com/docker/docker/builder/remotecontext/git/gitutils.go +index 77a45beff3..a9079153e0 100644 +--- a/components/cli/vendor/github.com/docker/docker/builder/remotecontext/git/gitutils.go ++++ b/components/cli/vendor/github.com/docker/docker/builder/remotecontext/git/gitutils.go +@@ -102,6 +102,10 @@ func parseRemoteURL(remoteURL string) (gitRepo, error) { + u.Fragment = "" + repo.remote = u.String() + } ++ ++ if strings.HasPrefix(repo.ref, "-") { ++ return gitRepo{}, errors.Errorf("invalid refspec: %s", repo.ref) ++ } + return repo, nil + } + +@@ -124,7 +128,7 @@ func fetchArgs(remoteURL string, ref string) []string { + args = append(args, "--depth", "1") + } + +- return append(args, "origin", ref) ++ return append(args, "origin", "--", ref) + } + + // Check if a given git URL supports a shallow git clone, +-- +2.17.1 + diff --git a/patch/0167-docker-fix-CVE-2020-13401.patch b/patch/0167-docker-fix-CVE-2020-13401.patch new file mode 100644 index 0000000..b3bd2b3 --- /dev/null +++ b/patch/0167-docker-fix-CVE-2020-13401.patch @@ -0,0 +1,69 @@ +From 727ce265564d1dc3031221a84f95abad20a20f11 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Thu, 11 Jun 2020 21:55:49 +0800 +Subject: [PATCH] docker: fix CVE-2020-13401 + +Change-Id: I267bde21d88927a0beb7599651b856a2dd1371d3 +Signed-off-by: jingrui +--- + .../libnetwork/drivers/bridge/bridge.go | 6 ++++++ + .../libnetwork/drivers/bridge/setup_device.go | 19 +++++++++++++++++++ + 2 files changed, 25 insertions(+) + +diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go +index 535da3c1ad..3288ff8652 100644 +--- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go ++++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go +@@ -679,6 +679,12 @@ func (d *driver) createNetwork(config *networkConfiguration) (err error) { + bridgeAlreadyExists := bridgeIface.exists() + if !bridgeAlreadyExists { + bridgeSetup.queueStep(setupDevice) ++ bridgeSetup.queueStep(setupDefaultSysctl) ++ } ++ ++ // For the default bridge, set expected sysctls ++ if config.DefaultBridge { ++ bridgeSetup.queueStep(setupDefaultSysctl) + } + + // Even if a bridge exists try to setup IPv4. +diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go +index a9dfd06771..9822236dfd 100644 +--- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go ++++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go +@@ -2,6 +2,9 @@ package bridge + + import ( + "fmt" ++ "io/ioutil" ++ "os" ++ "path/filepath" + + "github.com/docker/docker/pkg/parsers/kernel" + "github.com/docker/libnetwork/netutils" +@@ -50,6 +53,22 @@ func setupDevice(config *networkConfiguration, i *bridgeInterface) error { + return err + } + ++func setupDefaultSysctl(config *networkConfiguration, i *bridgeInterface) error { ++ // Disable IPv6 router advertisements originating on the bridge ++ sysPath := filepath.Join("/proc/sys/net/ipv6/conf/", config.BridgeName, "accept_ra") ++ if _, err := os.Stat(sysPath); err != nil { ++ logrus. ++ WithField("bridge", config.BridgeName). ++ WithField("syspath", sysPath). ++ Info("failed to read ipv6 net.ipv6.conf..accept_ra") ++ return nil ++ } ++ if err := ioutil.WriteFile(sysPath, []byte{'0', '\n'}, 0644); err != nil { ++ return fmt.Errorf("libnetwork: Unable to disable IPv6 router advertisement: %v", err) ++ } ++ return nil ++} ++ + // SetupDeviceUp ups the given bridge interface. + func setupDeviceUp(config *networkConfiguration, i *bridgeInterface) error { + err := i.nlh.LinkSetUp(i.Link) +-- +2.17.1 + diff --git a/patch/0168-docker-do-not-add-w-to-LDFLAGS.patch b/patch/0168-docker-do-not-add-w-to-LDFLAGS.patch new file mode 100644 index 0000000..5486839 --- /dev/null +++ b/patch/0168-docker-do-not-add-w-to-LDFLAGS.patch @@ -0,0 +1,79 @@ +From b23e7a179e68f194516b542bea375c44122e1037 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Tue, 11 Aug 2020 08:00:12 +0800 +Subject: [PATCH] docker: do not add "-w" to LDFLAG + +reason: for gdb debug, do not add "-w" to LDFLAGS + +Signed-off-by: xiadanni1 +--- + components/cli/scripts/build/.variables | 1 - + components/cli/scripts/build/dynbinary | 5 ++++- + components/engine/hack/make.sh | 5 ----- + components/engine/hack/make/.binary | 5 ++++- + 4 files changed, 8 insertions(+), 8 deletions(-) + +diff --git a/components/cli/scripts/build/.variables b/components/cli/scripts/build/.variables +index a23e379..7b78e62 100755 +--- a/components/cli/scripts/build/.variables ++++ b/components/cli/scripts/build/.variables +@@ -13,7 +13,6 @@ if test -n "${PLATFORM}"; then + fi + + export LDFLAGS="\ +- -w \ + ${PLATFORM_LDFLAGS} \ + -X \"github.com/docker/cli/cli.GitCommit=${GITCOMMIT}\" \ + -X \"github.com/docker/cli/cli.BuildTime=${BUILDTIME}\" \ +diff --git a/components/cli/scripts/build/dynbinary b/components/cli/scripts/build/dynbinary +index 2442166..40941bb 100755 +--- a/components/cli/scripts/build/dynbinary ++++ b/components/cli/scripts/build/dynbinary +@@ -13,7 +13,10 @@ export CGO_ENABLED=1 + BEP_DIR=/tmp/docker-build-bep + BEP_FLAGS="-tmpdir=$BEP_DIR" + mkdir -p $BEP_DIR ++GC_FLAGS="-gcflags=-trimpath=$GOPATH" ++ASM_FLAGS="-asmflags=-trimpath=$GOPATH" + +-go build -o "${TARGET}" -tags pkcs11 --ldflags " -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $BEP_FLAGS ${LDFLAGS}" -buildmode=pie "${SOURCE}" ++set -x ++go build $GC_FLAGS $ASM_FLAGS -o "${TARGET}" -tags pkcs11 --ldflags " -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $BEP_FLAGS ${LDFLAGS}" -buildmode=pie "${SOURCE}" + + ln -sf "$(basename "${TARGET}")" build/docker +diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh +index f4a51e7..d24a7b7 100755 +--- a/components/engine/hack/make.sh ++++ b/components/engine/hack/make.sh +@@ -137,12 +137,7 @@ if \ + fi + + # Use these flags when compiling the tests and final binary +- + IAMSTATIC='true' +-if [ -z "$DOCKER_DEBUG" ]; then +- LDFLAGS='-w' +-fi +- + LDFLAGS_STATIC='' + EXTLDFLAGS_STATIC='-static' + # ORIG_BUILDFLAGS is necessary for the cross target which cannot always build +diff --git a/components/engine/hack/make/.binary b/components/engine/hack/make/.binary +index f76b6f7..35bb836 100755 +--- a/components/engine/hack/make/.binary ++++ b/components/engine/hack/make/.binary +@@ -63,7 +63,10 @@ echo "Building: $DEST/$BINARY_FULLNAME" + BEP_DIR=/tmp/dockerd-build-bep + BEP_FLAGS="-tmpdir=$BEP_DIR" + mkdir -p $BEP_DIR +-go build \ ++GC_FLAGS="-gcflags=-trimpath=$GOPATH" ++ASM_FLAGS="-asmflags=-trimpath=$GOPATH" ++set -x ++go build $GC_FLAGS $ASM_FLAGS \ + -o "$DEST/$BINARY_FULLNAME" \ + "${BUILDFLAGS[@]}" \ + -ldflags " +-- +1.8.3.1 + diff --git a/patch/0169-docker-add-files-in-proc-for-mask.patch b/patch/0169-docker-add-files-in-proc-for-mask.patch new file mode 100644 index 0000000..18d6522 --- /dev/null +++ b/patch/0169-docker-add-files-in-proc-for-mask.patch @@ -0,0 +1,42 @@ +From c9db33aaad779afff04db8beb5b6d7e7e512a66d Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Mon, 24 Aug 2020 09:42:21 -0400 +Subject: [PATCH] docker: add files in proc for mask + +Signed-off-by: liuzekun +--- + components/engine/oci/defaults.go | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/components/engine/oci/defaults.go b/components/engine/oci/defaults.go +index cd4985f5..ec748a6d 100644 +--- a/components/engine/oci/defaults.go ++++ b/components/engine/oci/defaults.go +@@ -65,7 +65,7 @@ func DefaultLinuxSpec() specs.Spec { + Effective: defaultCapabilities(), + }, + }, +- Root: &specs.Root{}, ++ Root: &specs.Root{}, + Hooks: &specs.Hooks{}, + } + s.Mounts = []specs.Mount{ +@@ -128,9 +128,15 @@ func DefaultLinuxSpec() specs.Spec { + "/proc/keys", + "/proc/latency_stats", + "/proc/livepatch", ++ "/proc/lru_info", ++ "/proc/lru_info_file", + "/proc/memstat", + "/proc/net_namespace", + "/proc/oom_extend", ++ "/proc/pagealloc_statistics", ++ "/proc/pagealloc_bt", ++ "/proc/slaballoc_bt", ++ "/proc/slaballoc_module", + "/proc/sched_debug", + "/proc/scsi", + "/proc/sig_catch", +-- +2.19.1 + diff --git a/patch/0170-docker-fix-docker-load-files-leak.patch b/patch/0170-docker-fix-docker-load-files-leak.patch new file mode 100644 index 0000000..75bfa1b --- /dev/null +++ b/patch/0170-docker-fix-docker-load-files-leak.patch @@ -0,0 +1,27 @@ +From e7dd426dc3d962eae0e934dcb8fe4d805f8ea4ca Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Wed, 16 Sep 2020 22:40:49 -0400 +Subject: [PATCH] fix docker load files leak + +Signed-off-by: liuzekun +--- + components/engine/daemon/graphdriver/devmapper/deviceset.go | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go +index f5c0b04..ff90c44 100644 +--- a/components/engine/daemon/graphdriver/devmapper/deviceset.go ++++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go +@@ -2285,6 +2285,9 @@ func (devices *DeviceSet) unmountAndDeactivateAll(dir string) { + // and the device will be released when that container dies. + if err := unix.Unmount(fullname, unix.MNT_DETACH); err != nil && err != unix.EINVAL { + logger.Warnf("Shutdown unmounting %s, error: %s", fullname, err) ++ } else if err == nil { ++ logger.Debugf("Remove %s", fullname) ++ os.RemoveAll(fullname) + } + + if devInfo, err := devices.lookupDevice(name); err != nil { +-- +2.19.1 + diff --git a/patch/0171-docker-do-not-sync-if-BYPAAS_SYNC-is-false.patch b/patch/0171-docker-do-not-sync-if-BYPAAS_SYNC-is-false.patch new file mode 100644 index 0000000..63334a2 --- /dev/null +++ b/patch/0171-docker-do-not-sync-if-BYPAAS_SYNC-is-false.patch @@ -0,0 +1,28 @@ +From bbc6fce3870ff7f43c87efe13247bb185817aa67 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Wed, 14 Oct 2020 04:36:56 -0400 +Subject: [PATCH] do not sync if BYPAAS_SYNC is false + +Signed-off-by: liuzekun +--- + components/engine/pkg/devicemapper/devmapper.go | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go +index 06ddc3e9..a5c30cb3 100644 +--- a/components/engine/pkg/devicemapper/devmapper.go ++++ b/components/engine/pkg/devicemapper/devmapper.go +@@ -477,7 +477,9 @@ func BlockDeviceDiscard(path string) error { + + // Without this sometimes the remove of the device that happens after + // discard fails with EBUSY. +- unix.Sync() ++ if os.Getenv("DOCKER_BYPASS_SYNC_SYSCALL") != "false" { ++ unix.Sync() ++ } + + return nil + } +-- +2.19.1 + diff --git a/patch/0172-docker-fix-panic-on-single-character-volumes.patch b/patch/0172-docker-fix-panic-on-single-character-volumes.patch new file mode 100644 index 0000000..f0bdb97 --- /dev/null +++ b/patch/0172-docker-fix-panic-on-single-character-volumes.patch @@ -0,0 +1,27 @@ +From 0a2c746c0d560d18502f84078d233166934e9eb9 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Wed, 11 Nov 2020 23:47:30 -0500 +Subject: [PATCH] fix panic on single-character volumes + +Signed-off-by: liuzekun +--- + components/cli/cli/compose/loader/volume.go | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/components/cli/cli/compose/loader/volume.go b/components/cli/cli/compose/loader/volume.go +index 9c2792e0..f043f4aa 100644 +--- a/components/cli/cli/compose/loader/volume.go ++++ b/components/cli/cli/compose/loader/volume.go +@@ -111,6 +111,9 @@ func isFilePath(source string) bool { + case '.', '/', '~': + return true + } ++ if len([]rune(source)) == 1 { ++ return false ++ } + + // windows named pipes + if strings.HasPrefix(source, `\\`) { +-- +2.19.1 + diff --git a/patch/0173-docker-fix-stats-memory-usage-display-error.patch b/patch/0173-docker-fix-stats-memory-usage-display-error.patch new file mode 100644 index 0000000..08d6827 --- /dev/null +++ b/patch/0173-docker-fix-stats-memory-usage-display-error.patch @@ -0,0 +1,38 @@ +From d26341e4c447ddbb6bd289845b7b47f0e4348c62 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Wed, 11 Nov 2020 17:35:06 +0800 +Subject: [PATCH] docker:fix stats memory usage display error + +fix stats memory usage display error +use total_inactive_file not cache to calculate memory usage +The new stat definition corresponds to containerd/CRI and cadvisor. + +https://github.com/containerd/cri/blob/c1115d4e57f55a5f45fb3efd29d3181ce26d5c6a/pkg/server/container_stats_list_unix.go#L106-L129 +https://github.com/google/cadvisor/commit/307d1b1cb320fef66fab02db749f07a459245451 + +Signed-off-by: xiadanni1 +Signed-off-by: Akihiro Suda +--- + components/cli/cli/command/container/stats_helpers.go | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/components/cli/cli/command/container/stats_helpers.go b/components/cli/cli/command/container/stats_helpers.go +index 2300ce5..c254212 100644 +--- a/components/cli/cli/command/container/stats_helpers.go ++++ b/components/cli/cli/command/container/stats_helpers.go +@@ -226,7 +226,11 @@ func calculateNetwork(network map[string]types.NetworkStats) (float64, float64) + // calculateMemUsageUnixNoCache calculate memory usage of the container. + // Page cache is intentionally excluded to avoid misinterpretation of the output. + func calculateMemUsageUnixNoCache(mem types.MemoryStats) float64 { +- return float64(mem.Usage - mem.Stats["cache"]) ++ if v, isCgroup1 := mem.Stats["total_inactive_file"]; isCgroup1 && v < mem.Usage { ++ return float64(mem.Usage - v) ++ } ++ ++ return float64(mem.Usage) + } + + func calculateMemPercentUnixNoCache(limit float64, usedNoCache float64) float64 { +-- +1.8.3.1 + diff --git a/patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch b/patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch new file mode 100644 index 0000000..4863e76 --- /dev/null +++ b/patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch @@ -0,0 +1,27 @@ +From 19ce3a9a435ddb67a4e7a081cd23bb5cc19abc92 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Thu, 30 Jul 2020 05:09:42 -0400 +Subject: [PATCH] dockerd: add more messages for ops when device not found in + the host + +Signed-off-by: liuzekun +--- + components/engine/daemon/daemon_unix.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go +index 5a59b324..af50fa37 100644 +--- a/components/engine/daemon/daemon_unix.go ++++ b/components/engine/daemon/daemon_unix.go +@@ -262,7 +262,7 @@ func getBlkioThrottleDevices(devs []*blkiodev.ThrottleDevice) ([]specs.LinuxThro + + for _, d := range devs { + if err := unix.Stat(d.Path, &stat); err != nil { +- return nil, err ++ return nil, errors.Wrapf(err, "Failed to stat device %q", d.Path) + } + d := specs.LinuxThrottleDevice{Rate: d.Rate} + d.Major = int64(stat.Rdev / 256) +-- +2.19.1 + diff --git a/patch/0175-docker-mask-proc-pin_memory.patch b/patch/0175-docker-mask-proc-pin_memory.patch new file mode 100644 index 0000000..4b0502d --- /dev/null +++ b/patch/0175-docker-mask-proc-pin_memory.patch @@ -0,0 +1,25 @@ +From 017c3377f0bd5230c0fa1699bd193baa527f0b8f Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Fri, 20 Nov 2020 18:34:45 +0800 +Subject: [PATCH] docker:mask /proc/pin_memory + +Signed-off-by: xiadanni1 +--- + components/engine/oci/defaults.go | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/components/engine/oci/defaults.go b/components/engine/oci/defaults.go +index ec748a6..e763cb7 100644 +--- a/components/engine/oci/defaults.go ++++ b/components/engine/oci/defaults.go +@@ -135,6 +135,7 @@ func DefaultLinuxSpec() specs.Spec { + "/proc/oom_extend", + "/proc/pagealloc_statistics", + "/proc/pagealloc_bt", ++ "/proc/pin_memory", + "/proc/slaballoc_bt", + "/proc/slaballoc_module", + "/proc/sched_debug", +-- +1.8.3.1 + diff --git a/series.conf b/series.conf index 141c9ce..1b6b8cf 100644 --- a/series.conf +++ b/series.conf @@ -156,3 +156,18 @@ patch/0157-docker-Support-check-manifest-and-layer-s-DiffID-inf.patch patch/0158-docker-support-private-registry.patch patch/0159-docker-extend-timeout-in-cli-testcases.patch patch/0160-docker-create-a-soft-link-from-runtime-default-to-ru.patch +patch/0161-docker-Delete-stale-containerd-object-on-start-failure.patch +patch/0162-docker-remove-redundant-word-item.patch +patch/0163-docker-delete-event-is-not-need-to-process.patch +patch/0164-docker-stat-process-exit-file-when-kill-process-dire.patch +patch/0165-docker-sync-cli-vendor.patch +patch/0167-docker-fix-CVE-2020-13401.patch +patch/0168-docker-do-not-add-w-to-LDFLAGS.patch +patch/0169-docker-add-files-in-proc-for-mask.patch +patch/0170-docker-fix-docker-load-files-leak.patch +patch/0171-docker-do-not-sync-if-BYPAAS_SYNC-is-false.patch +patch/0172-docker-fix-panic-on-single-character-volumes.patch +patch/0173-docker-fix-stats-memory-usage-display-error.patch +patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch +patch/0175-docker-mask-proc-pin_memory.patch +#end -- Gitee From 7cf5d047db5f766e14849708d808fe4745c4f208 Mon Sep 17 00:00:00 2001 From: jingrui Date: Mon, 18 Jan 2021 21:27:04 +0800 Subject: [PATCH 2/2] docker: sync bugfix Change-Id: Ida64f926d5d3a2a1f99c8718918737836e256897 Signed-off-by: jingrui --- VERSION-openeuler | 2 +- docker-engine-openeuler.spec | 31 +++- ...messages-for-ops-when-device-not-fo.patch} | 0 ...-docker-clean-docker-load-leak-files.patch | 85 +++++++++++ ...ainer-process-if-its-status-is-not-r.patch | 62 ++++++++ patch/0177-resume-suspend-dm-on-start.patch | 82 +++++++++++ ...-and-restart-containerd-during-docke.patch | 85 +++++++++++ patch/0179-handle-exit-force.patch | 107 ++++++++++++++ ...-wait-io-with-timeout-in-task-delete.patch | 47 ++++++ ...-return-when-matched-registry-mirror.patch | 137 ++++++++++++++++++ ...-pagealloc_module-and-slaballoc_stat.patch | 29 ++++ ...ith-timeout-when-process-Start-faile.patch | 48 ++++++ ...age-reference-when-failed-to-get-ima.patch | 63 ++++++++ series.conf | 12 +- 14 files changed, 786 insertions(+), 4 deletions(-) rename patch/{0174-docker-add-more-messages-for-ops-when-device-not-fo.patch => 0167-dockerd-add-more-messages-for-ops-when-device-not-fo.patch} (100%) create mode 100644 patch/0175-docker-clean-docker-load-leak-files.patch create mode 100644 patch/0176-docker-kill-container-process-if-its-status-is-not-r.patch create mode 100644 patch/0177-resume-suspend-dm-on-start.patch create mode 100644 patch/0178-docker-skip-kill-and-restart-containerd-during-docke.patch create mode 100644 patch/0179-handle-exit-force.patch create mode 100644 patch/0180-wait-io-with-timeout-in-task-delete.patch create mode 100644 patch/0181-docker-do-not-return-when-matched-registry-mirror.patch create mode 100644 patch/0183-add-masked-paths-pagealloc_module-and-slaballoc_stat.patch create mode 100644 patch/0184-docker-wait-io-with-timeout-when-process-Start-faile.patch create mode 100644 patch/0185-docker-delete-image-reference-when-failed-to-get-ima.patch diff --git a/VERSION-openeuler b/VERSION-openeuler index 9ce0e89..4c88868 100644 --- a/VERSION-openeuler +++ b/VERSION-openeuler @@ -1 +1 @@ -18.09.0.102 +18.09.0.105 diff --git a/docker-engine-openeuler.spec b/docker-engine-openeuler.spec index 5e5cf89..82f5969 100644 --- a/docker-engine-openeuler.spec +++ b/docker-engine-openeuler.spec @@ -1,6 +1,6 @@ Name: docker-engine Version: 18.09.0 -Release: 102 +Release: 105 Summary: The open-source application container engine Group: Tools/Docker @@ -200,7 +200,34 @@ fi %endif %changelog -* Sat Dec 28 2020 liuzekun - 18.09.0-102 +* Mon Jan 18 2021 jingrui - 18.09.0-105 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:sync bugfix include + 1. fix image cleanup failed. + 2. cleanup load tmp files. + 3. kill residual container process. + 4. resume suspend dm device. + 5. dont kill containerd during dockerd starting. + 6. handle exit event for restore failed container. + 7. wait io with timeout when start failed. + 8. support hostname mirror registry. + 9. mask unused proc files. + +* Tue Dec 8 2020 xiadanni - 18.09.0-104 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:runc don't deny all devices when update cgroup resource + +* Thu Dec 3 2020 xiadanni - 18.09.0-103 +- Type:bugfix +- ID:NA +- SUG:restart +- DESC:containerd fix CVE-2020-15257 + +* Fri Nov 27 2020 liuzekun - 18.09.0-102 - Type:bugfix - ID:NA - CVE:NA diff --git a/patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch b/patch/0167-dockerd-add-more-messages-for-ops-when-device-not-fo.patch similarity index 100% rename from patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch rename to patch/0167-dockerd-add-more-messages-for-ops-when-device-not-fo.patch diff --git a/patch/0175-docker-clean-docker-load-leak-files.patch b/patch/0175-docker-clean-docker-load-leak-files.patch new file mode 100644 index 0000000..f0a32c3 --- /dev/null +++ b/patch/0175-docker-clean-docker-load-leak-files.patch @@ -0,0 +1,85 @@ +From a74f1c3e4ab7c6f4a043904a8e68edf04864d98a Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Wed, 2 Dec 2020 17:20:50 +0800 +Subject: [PATCH] docker: clean docker load leak files + +Change-Id: I09b66e204f655a9fef660bb85619f5711fb5700b +Signed-off-by: jingrui +--- + components/engine/daemon/daemon.go | 39 +++++++++++++++++++ + .../daemon/graphdriver/devmapper/deviceset.go | 3 +- + 2 files changed, 41 insertions(+), 1 deletion(-) + +diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go +index 3ff5691257..1acd355a15 100644 +--- a/components/engine/daemon/daemon.go ++++ b/components/engine/daemon/daemon.go +@@ -613,11 +613,50 @@ func (daemon *Daemon) restore() error { + + group.Wait() + ++ daemon.cleanExit() + logrus.Info("Loading containers: done.") + + return nil + } + ++func (daemon *Daemon) cleanExit() { ++ mnt := filepath.Join(daemon.root, "devicemapper/mnt") ++ if dir, err := ioutil.ReadDir(mnt); err == nil { ++ for _, f := range dir { ++ fname := filepath.Join(mnt, f.Name()) ++ data, err := ioutil.ReadFile(fname) ++ if err != nil { ++ continue ++ } ++ if string(data) == "exit" { ++ logrus.Infof("cleanExit remove mnt %s", fname) ++ os.Remove(fname) ++ } ++ } ++ } ++ ++ tmp := filepath.Join(daemon.root, "image/devicemapper/layerdb/tmp") ++ if dir, err := ioutil.ReadDir(tmp); err == nil { ++ for _, f := range dir { ++ if strings.Contains(f.Name(), "write-set-") { ++ fname := filepath.Join(tmp, f.Name()) ++ logrus.Infof("cleanExit remove layerdb %s", fname) ++ os.RemoveAll(fname) ++ } ++ } ++ } ++ ++ if dir, err := ioutil.ReadDir(os.Getenv("TMPDIR")); err == nil { ++ for _, f := range dir { ++ if strings.Contains(f.Name(), "docker-import-") { ++ fname := filepath.Join(os.Getenv("TMPDIR"), f.Name()) ++ logrus.Infof("cleanExit remove tmpdir %s", fname) ++ os.RemoveAll(fname) ++ } ++ } ++ } ++} ++ + // RestartSwarmContainers restarts any autostart container which has a + // swarm endpoint. + func (daemon *Daemon) RestartSwarmContainers() { +diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go +index ff90c44ce3..750f2b13f8 100644 +--- a/components/engine/daemon/graphdriver/devmapper/deviceset.go ++++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go +@@ -2286,8 +2286,9 @@ func (devices *DeviceSet) unmountAndDeactivateAll(dir string) { + if err := unix.Unmount(fullname, unix.MNT_DETACH); err != nil && err != unix.EINVAL { + logger.Warnf("Shutdown unmounting %s, error: %s", fullname, err) + } else if err == nil { +- logger.Debugf("Remove %s", fullname) ++ logger.Infof("cleanExit prepare %s", fullname) + os.RemoveAll(fullname) ++ ioutil.WriteFile(fullname, []byte("exit"), 0600) + } + + if devInfo, err := devices.lookupDevice(name); err != nil { +-- +2.17.1 + diff --git a/patch/0176-docker-kill-container-process-if-its-status-is-not-r.patch b/patch/0176-docker-kill-container-process-if-its-status-is-not-r.patch new file mode 100644 index 0000000..49693f2 --- /dev/null +++ b/patch/0176-docker-kill-container-process-if-its-status-is-not-r.patch @@ -0,0 +1,62 @@ +From 544d24895836ec576febaf94be8affde56449fba Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Fri, 27 Nov 2020 16:31:56 +0800 +Subject: [PATCH] docker: kill container process if its status is not running + when start daemon + +Signed-off-by: xiadanni1 +--- + components/engine/daemon/daemon.go | 11 ++++++++++- + 1 file changed, 10 insertions(+), 1 deletion(-) + +diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go +index 3ff5691..3cc2a20 100644 +--- a/components/engine/daemon/daemon.go ++++ b/components/engine/daemon/daemon.go +@@ -17,8 +17,10 @@ import ( + "runtime" + "strings" + "sync" ++ "syscall" + "time" + ++ "golang.org/x/sys/unix" + "google.golang.org/grpc" + + "github.com/containerd/containerd" +@@ -43,6 +45,7 @@ import ( + "github.com/moby/buildkit/util/resolver" + "github.com/moby/buildkit/util/tracing" + "github.com/sirupsen/logrus" ++ + // register graph drivers + _ "github.com/docker/docker/daemon/graphdriver/register" + "github.com/docker/docker/daemon/stats" +@@ -51,7 +54,7 @@ import ( + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + "github.com/docker/docker/libcontainerd" +- "github.com/docker/docker/migrate/v1" ++ v1 "github.com/docker/docker/migrate/v1" + "github.com/docker/docker/pkg/idtools" + "github.com/docker/docker/pkg/locker" + "github.com/docker/docker/pkg/plugingetter" +@@ -389,6 +392,15 @@ func (daemon *Daemon) restore() error { + } + } + ++ if alive && !c.IsRunning() && pid > 1 { ++ if c.Pid == 0 { ++ c.Pid = pid ++ } ++ err := unix.Kill(pid, syscall.SIGKILL) ++ logrus.Warnf("process %v is killed as container=%s is alive but not running, err: %v", pid, c.ID, err) ++ return ++ } ++ + if c.IsRunning() || c.IsPaused() { + c.RestartManager().Cancel() // manually start containers because some need to wait for swarm networking + +-- +1.8.3.1 + diff --git a/patch/0177-resume-suspend-dm-on-start.patch b/patch/0177-resume-suspend-dm-on-start.patch new file mode 100644 index 0000000..2ee80f3 --- /dev/null +++ b/patch/0177-resume-suspend-dm-on-start.patch @@ -0,0 +1,82 @@ +From 37e3e3dfb31f30b2599d05f021671f6e682f37d6 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Wed, 9 Dec 2020 17:37:02 +0800 +Subject: [PATCH] resume suspend dm on start + +Change-Id: Ibe215c80aa62b4d4b464749cc6e995d2e0e845af +Signed-off-by: jingrui +--- + components/engine/cmd/dockerd/daemon.go | 43 +++++++++++++++++++++++++ + 1 file changed, 43 insertions(+) + +diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go +index 0b3fa0e037..dbf37f3338 100644 +--- a/components/engine/cmd/dockerd/daemon.go ++++ b/components/engine/cmd/dockerd/daemon.go +@@ -6,6 +6,7 @@ import ( + "fmt" + "io/ioutil" + "os" ++ "os/exec" + "path/filepath" + "runtime" + "strings" +@@ -72,6 +73,45 @@ func NewDaemonCli() *DaemonCli { + return &DaemonCli{} + } + ++func resumeDM() { ++ c := make(chan struct{}) ++ go func() { ++ defer close(c) ++ out, err := exec.Command("dmsetup", "info", "-c", "--sort", "minor", "--noheadings", "--separator", ",", "-o", "attr,name").CombinedOutput() ++ if err != nil { ++ logrus.Errorf("resume-dm dmsetup info failed: %v", err) ++ return ++ } ++ ++ args := []string{"resume"} ++ for _, line := range strings.Split(string(out), "\n") { ++ aa := strings.Split(line, ",") ++ if len(aa) != 2 || !strings.Contains(aa[0], "s") || strings.Index(aa[1], "docker-") != 0 { ++ continue ++ } ++ args = append(args, aa[1]) ++ } ++ if len(args) == 1 { ++ return ++ } ++ ++ logrus.Infof("resume-dm start resume suspended dm %v", args) ++ _, err = exec.Command("dmsetup", args...).CombinedOutput() ++ if err != nil { ++ logrus.Errorf("resume-dm %s failed: %v", err) ++ return ++ } ++ logrus.Infof("resume-dm finished resume suspended dm") ++ }() ++ select { ++ case <-c: ++ return ++ case <-time.After(10*time.Second): ++ logrus.Warnf("resume-dm timeout, continue anyway.") ++ return ++ } ++} ++ + func cleanupLocalDB(db string) { + _, err := os.Stat(db) + if err == nil { +@@ -150,6 +190,9 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { + }) + + system.InitLCOW(cli.Config.Experimental) ++ if cli.Config.GraphDriver == "devicemapper" { ++ resumeDM() ++ } + + if err := setDefaultUmask(); err != nil { + return fmt.Errorf("Failed to set umask: %v", err) +-- +2.17.1 + diff --git a/patch/0178-docker-skip-kill-and-restart-containerd-during-docke.patch b/patch/0178-docker-skip-kill-and-restart-containerd-during-docke.patch new file mode 100644 index 0000000..69d9794 --- /dev/null +++ b/patch/0178-docker-skip-kill-and-restart-containerd-during-docke.patch @@ -0,0 +1,85 @@ +From a56def385f835885df056d0d54372111abdc1507 Mon Sep 17 00:00:00 2001 +From: xiadanni +Date: Sat, 19 Dec 2020 18:56:38 +0800 +Subject: [PATCH] docker:skip kill and restart containerd during docker daemon + is starting + +Signed-off-by: xiadanni +--- + components/engine/cmd/dockerd/daemon.go | 4 +++- + components/engine/libcontainerd/supervisor/remote_daemon.go | 9 +++++++++ + 2 files changed, 12 insertions(+), 1 deletion(-) + +diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go +index dbf37f3..c25ee0e 100644 +--- a/components/engine/cmd/dockerd/daemon.go ++++ b/components/engine/cmd/dockerd/daemon.go +@@ -10,6 +10,7 @@ import ( + "path/filepath" + "runtime" + "strings" ++ "sync/atomic" + "time" + + containerddefaults "github.com/containerd/containerd/defaults" +@@ -106,7 +107,7 @@ func resumeDM() { + select { + case <-c: + return +- case <-time.After(10*time.Second): ++ case <-time.After(10 * time.Second): + logrus.Warnf("resume-dm timeout, continue anyway.") + return + } +@@ -304,6 +305,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { + + logrus.Info("Daemon has completed initialization") + ++ atomic.StoreInt32(&supervisor.IsDockerUp, 1) + cli.d = d + + routerOptions, err := newRouterOptions(cli.Config, d) +diff --git a/components/engine/libcontainerd/supervisor/remote_daemon.go b/components/engine/libcontainerd/supervisor/remote_daemon.go +index 62ea58c..19582cd 100644 +--- a/components/engine/libcontainerd/supervisor/remote_daemon.go ++++ b/components/engine/libcontainerd/supervisor/remote_daemon.go +@@ -11,6 +11,7 @@ import ( + "strconv" + "strings" + "sync" ++ "sync/atomic" + "time" + + "github.com/BurntSushi/toml" +@@ -19,6 +20,7 @@ import ( + "github.com/docker/docker/pkg/system" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ++ "golang.org/x/sys/unix" + ) + + const ( +@@ -31,6 +33,8 @@ const ( + pidFile = "containerd.pid" + ) + ++var IsDockerUp int32 ++ + type pluginConfigs struct { + Plugins map[string]interface{} `toml:"plugins"` + } +@@ -314,6 +318,11 @@ func (r *remote) monitorDaemon(ctx context.Context) { + } + + if system.IsProcessAlive(r.daemonPid) { ++ if atomic.LoadInt32(&IsDockerUp) == 0 { ++ r.logger.WithField("pid", r.daemonPid).Info("dockerd is starting, skip killing containerd") ++ unix.Kill(r.daemonPid, unix.SIGCONT) ++ continue ++ } + r.logger.WithField("pid", r.daemonPid).Info("killing and restarting containerd") + r.killDaemon() + } +-- +1.8.3.1 + diff --git a/patch/0179-handle-exit-force.patch b/patch/0179-handle-exit-force.patch new file mode 100644 index 0000000..76c1a5e --- /dev/null +++ b/patch/0179-handle-exit-force.patch @@ -0,0 +1,107 @@ +From 66b6e3065b160bd7d480f183156acbe1cb9bf2e0 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Tue, 15 Dec 2020 16:05:56 +0800 +Subject: [PATCH] handle exit force + +Change-Id: If08483f57b4f04d6c4961c9f588e4d599009eddc +Signed-off-by: jingrui +--- + components/engine/daemon/monitor.go | 9 +++++++++ + components/engine/libcontainerd/client_daemon.go | 14 ++++++++++++++ + components/engine/libcontainerd/types.go | 1 + + .../plugin/executor/containerd/containerd.go | 5 +++++ + 4 files changed, 29 insertions(+) + +diff --git a/components/engine/daemon/monitor.go b/components/engine/daemon/monitor.go +index e041bd5c69..1b577c0dae 100644 +--- a/components/engine/daemon/monitor.go ++++ b/components/engine/daemon/monitor.go +@@ -26,6 +26,14 @@ func (daemon *Daemon) setStateCounter(c *container.Container) { + } + } + ++func (daemon *Daemon) IsContainerRunning(id string) bool { ++ c, err := daemon.GetContainer(id) ++ if err != nil { ++ return false ++ } ++ return c.IsRunning() ++} ++ + // ProcessEvent is called by libcontainerd whenever an event occurs + func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libcontainerd.EventInfo) error { + c, err := daemon.GetContainer(id) +@@ -51,6 +59,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc + case libcontainerd.EventExit: + if int(ei.Pid) == c.Pid { + c.Lock() ++ logrus.Infof("handle exit event cid=%s pid=%d", c.ID, c.Pid) + _, _, err := daemon.containerd.DeleteTask(context.Background(), c.ID) + if err != nil { + logrus.WithError(err).Warnf("failed to delete container %s from containerd", c.ID) +diff --git a/components/engine/libcontainerd/client_daemon.go b/components/engine/libcontainerd/client_daemon.go +index 05c439c540..502796bd25 100755 +--- a/components/engine/libcontainerd/client_daemon.go ++++ b/components/engine/libcontainerd/client_daemon.go +@@ -517,9 +517,16 @@ func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, ti + return status.ExitCode(), status.ExitTime(), nil + } + ++func (c *client) deleteForce(ctx context.Context, id string) { ++ if ctr, err := c.client.LoadContainer(ctx, id); err == nil { ++ logrus.Warnf("delete containerd meta id=%s force: error=%v", id, ctr.Delete(ctx)) ++ } ++} ++ + func (c *client) Delete(ctx context.Context, containerID string) error { + ctr := c.getContainer(containerID) + if ctr == nil { ++ c.deleteForce(ctx, containerID) + return errors.WithStack(newNotFoundError("no such container")) + } + +@@ -907,6 +914,13 @@ func (c *client) processEventStream(ctx context.Context, ns string) { + ctr = c.getContainer(ei.ContainerID) + if ctr == nil { + c.logger.WithField("container", ei.ContainerID).Warn("unknown container") ++ if et == EventExit && ei.ProcessID == ei.ContainerID && c.backend.IsContainerRunning(ei.ContainerID) { ++ c.logger.WithField("container", ei.ContainerID).Warn("handle exit event force ...") ++ c.eventQ.append(ei.ContainerID, func() { ++ c.logger.WithField("container", ei.ContainerID).Warnf("handle exit event force: error=%v", ++ c.backend.ProcessEvent(ei.ContainerID, et, ei)) ++ }) ++ } + continue + } + +diff --git a/components/engine/libcontainerd/types.go b/components/engine/libcontainerd/types.go +index c4de5e674d..0b9df9193b 100644 +--- a/components/engine/libcontainerd/types.go ++++ b/components/engine/libcontainerd/types.go +@@ -60,6 +60,7 @@ type EventInfo struct { + // Backend defines callbacks that the client of the library needs to implement. + type Backend interface { + ProcessEvent(containerID string, event EventType, ei EventInfo) error ++ IsContainerRunning(id string) bool + } + + // Client provides access to containerd features. +diff --git a/components/engine/plugin/executor/containerd/containerd.go b/components/engine/plugin/executor/containerd/containerd.go +index a3401dce79..f75771fe41 100644 +--- a/components/engine/plugin/executor/containerd/containerd.go ++++ b/components/engine/plugin/executor/containerd/containerd.go +@@ -141,6 +141,11 @@ func (e *Executor) ProcessEvent(id string, et libcontainerd.EventType, ei libcon + return nil + } + ++func (e *Executor) IsContainerRunning(id string) bool { ++ ok, _ := e.IsRunning(id) ++ return ok ++} ++ + type rio struct { + cio.IO + +-- +2.17.1 + diff --git a/patch/0180-wait-io-with-timeout-in-task-delete.patch b/patch/0180-wait-io-with-timeout-in-task-delete.patch new file mode 100644 index 0000000..5ac8675 --- /dev/null +++ b/patch/0180-wait-io-with-timeout-in-task-delete.patch @@ -0,0 +1,47 @@ +From 0f3aa35a1c38fe7fc49cd6fb66fc47a993ad6bb8 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Wed, 16 Dec 2020 18:39:00 +0800 +Subject: [PATCH] wait io with timeout in task delete + +Change-Id: I23ed40d69279b14a216b6ffb9988439475be5cad +Signed-off-by: jingrui +--- + .../github.com/containerd/containerd/task.go | 15 +++++++++++++-- + 1 file changed, 13 insertions(+), 2 deletions(-) + +diff --git a/components/engine/vendor/github.com/containerd/containerd/task.go b/components/engine/vendor/github.com/containerd/containerd/task.go +index 6806e11620..7421432bed 100644 +--- a/components/engine/vendor/github.com/containerd/containerd/task.go ++++ b/components/engine/vendor/github.com/containerd/containerd/task.go +@@ -44,6 +44,7 @@ import ( + "github.com/opencontainers/image-spec/specs-go/v1" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + // UnknownExitStatus is returned when containerd is unable to +@@ -287,8 +288,18 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat + return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "task must be stopped before deletion: %s", status.Status) + } + if t.io != nil { +- t.io.Cancel() +- t.io.Wait() ++ done := make(chan struct{}) ++ go func() { ++ t.io.Cancel() ++ t.io.Wait() ++ close(done) ++ }() ++ select { ++ case <-time.After(3 * time.Second): ++ logrus.Warnf("task delete wait io close timeout, some fifo io may be dropped.") ++ case <-done: ++ // ok ++ } + } + r, err := t.client.TaskService().Delete(ctx, &tasks.DeleteTaskRequest{ + ContainerID: t.id, +-- +2.17.1 + diff --git a/patch/0181-docker-do-not-return-when-matched-registry-mirror.patch b/patch/0181-docker-do-not-return-when-matched-registry-mirror.patch new file mode 100644 index 0000000..54b67f3 --- /dev/null +++ b/patch/0181-docker-do-not-return-when-matched-registry-mirror.patch @@ -0,0 +1,137 @@ +From 8cc3f33020152d51d38927593ba49ad3dfacf62e Mon Sep 17 00:00:00 2001 +From: shaobao.feng +Date: Mon, 7 Dec 2020 15:30:11 +0800 +Subject: [PATCH] docker: do not return when matched registry mirror + +Change-Id: I5317b91b60293e1f4c50f5a327790c5509537f9b +reason: append hostname itself to make sure the hostname itself will be tried. +--- + components/engine/registry/service_v2.go | 86 +++++++++++------------- + 1 file changed, 41 insertions(+), 45 deletions(-) + +diff --git a/components/engine/registry/service_v2.go b/components/engine/registry/service_v2.go +index adeb10c550..df66cd7451 100644 +--- a/components/engine/registry/service_v2.go ++++ b/components/engine/registry/service_v2.go +@@ -19,8 +19,7 @@ func (s *DefaultService) lookupV2Endpoints(hostname string) (endpoints []APIEndp + if reg != nil { + var regEndpoints []registrytypes.Endpoint = reg.Mirrors + +- lastIndex := len(regEndpoints) - 1 +- for i, regEP := range regEndpoints { ++ for _, regEP := range regEndpoints { + official := regEP.Address == registrytypes.DefaultEndpoint.Address + regURL := regEP.GetURL() + +@@ -41,49 +40,48 @@ func (s *DefaultService) lookupV2Endpoints(hostname string) (endpoints []APIEndp + TLSConfig: tlsConfig, + Prefix: hostname, + // the last endpoint is not considered a mirror +- Mirror: i != lastIndex, ++ Mirror: true, + }) + } +- return endpoints, nil ++ // don't return here, otherwise the hostname itself will not be appended to the endpoints, ++ // and the hostname itself will not be tried, which is not a desired action. + } +- } else { ++ } ++ if hostname == DefaultNamespace || hostname == IndexHostname { + tlsConfig = tlsconfig.ServerDefault() +- if hostname == DefaultNamespace || hostname == IndexHostname { +- // v2 mirrors +- for _, mirror := range s.config.Mirrors { +- if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") { +- mirror = "https://" + mirror +- } +- mirrorURL, err := url.Parse(mirror) +- if err != nil { +- return nil, err +- } +- mirrorTLSConfig, err := s.tlsConfigForMirror(mirrorURL) +- if err != nil { +- return nil, err +- } +- endpoints = append(endpoints, APIEndpoint{ +- URL: mirrorURL, +- // guess mirrors are v2 +- Version: APIVersion2, +- Mirror: true, +- TrimHostname: true, +- TLSConfig: mirrorTLSConfig, +- }) ++ // v2 mirrors ++ for _, mirror := range s.config.Mirrors { ++ if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") { ++ mirror = "https://" + mirror ++ } ++ mirrorURL, err := url.Parse(mirror) ++ if err != nil { ++ return nil, err ++ } ++ mirrorTLSConfig, err := s.tlsConfigForMirror(mirrorURL) ++ if err != nil { ++ return nil, err + } +- // v2 registry + endpoints = append(endpoints, APIEndpoint{ +- URL: DefaultV2Registry, ++ URL: mirrorURL, ++ // guess mirrors are v2 + Version: APIVersion2, +- Official: true, ++ Mirror: true, + TrimHostname: true, +- TLSConfig: tlsConfig, ++ TLSConfig: mirrorTLSConfig, + }) +- +- return endpoints, nil + } +- } ++ // v2 registry ++ endpoints = append(endpoints, APIEndpoint{ ++ URL: DefaultV2Registry, ++ Version: APIVersion2, ++ Official: true, ++ TrimHostname: true, ++ TLSConfig: tlsConfig, ++ }) + ++ return endpoints, nil ++ } + ana := allowNondistributableArtifacts(s.config, hostname) + + tlsConfig, err = s.tlsConfig(hostname) +@@ -91,18 +89,16 @@ func (s *DefaultService) lookupV2Endpoints(hostname string) (endpoints []APIEndp + return nil, err + } + +- endpoints = []APIEndpoint{ +- { +- URL: &url.URL{ +- Scheme: "https", +- Host: hostname, +- }, +- Version: APIVersion2, +- AllowNondistributableArtifacts: ana, +- TrimHostname: true, +- TLSConfig: tlsConfig, ++ endpoints = append(endpoints, APIEndpoint{ ++ URL: &url.URL{ ++ Scheme: "https", ++ Host: hostname, + }, +- } ++ Version: APIVersion2, ++ AllowNondistributableArtifacts: ana, ++ TrimHostname: true, ++ TLSConfig: tlsConfig, ++ }) + + if tlsConfig.InsecureSkipVerify { + endpoints = append(endpoints, APIEndpoint{ +-- +2.17.1 + diff --git a/patch/0183-add-masked-paths-pagealloc_module-and-slaballoc_stat.patch b/patch/0183-add-masked-paths-pagealloc_module-and-slaballoc_stat.patch new file mode 100644 index 0000000..5ed2ef1 --- /dev/null +++ b/patch/0183-add-masked-paths-pagealloc_module-and-slaballoc_stat.patch @@ -0,0 +1,29 @@ +From fada5f66fcc555d706603dd3c7832e78e9955501 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Thu, 31 Dec 2020 03:07:42 -0500 +Subject: add masked paths pagealloc_module and slaballoc_statistics + +Signed-off-by: liuzekun +--- + components/engine/oci/defaults.go | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/components/engine/oci/defaults.go b/components/engine/oci/defaults.go +index e763cb75..ff027d89 100644 +--- a/components/engine/oci/defaults.go ++++ b/components/engine/oci/defaults.go +@@ -135,9 +135,11 @@ func DefaultLinuxSpec() specs.Spec { + "/proc/oom_extend", + "/proc/pagealloc_statistics", + "/proc/pagealloc_bt", ++ "/proc/pagealloc_module", + "/proc/pin_memory", + "/proc/slaballoc_bt", + "/proc/slaballoc_module", ++ "/proc/slaballoc_statistics", + "/proc/sched_debug", + "/proc/scsi", + "/proc/sig_catch", +-- +2.19.1 + diff --git a/patch/0184-docker-wait-io-with-timeout-when-process-Start-faile.patch b/patch/0184-docker-wait-io-with-timeout-when-process-Start-faile.patch new file mode 100644 index 0000000..c9784dc --- /dev/null +++ b/patch/0184-docker-wait-io-with-timeout-when-process-Start-faile.patch @@ -0,0 +1,48 @@ +From ef64f4dd5d532b550bb68f60e6373e139fdf5382 Mon Sep 17 00:00:00 2001 +From: xiadanni +Date: Fri, 15 Jan 2021 11:23:04 +0800 +Subject: [PATCH] docker: wait io with timeout when process Start failed + +Signed-off-by: xiadanni +--- + .../vendor/github.com/containerd/containerd/process.go | 17 ++++++++++++++--- + 1 file changed, 14 insertions(+), 3 deletions(-) + +diff --git a/components/engine/vendor/github.com/containerd/containerd/process.go b/components/engine/vendor/github.com/containerd/containerd/process.go +index ff7d838..4d0dca9 100644 +--- a/components/engine/vendor/github.com/containerd/containerd/process.go ++++ b/components/engine/vendor/github.com/containerd/containerd/process.go +@@ -26,6 +26,7 @@ import ( + "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/errdefs" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + // Process represents a system process +@@ -111,9 +112,19 @@ func (p *process) Start(ctx context.Context) error { + ExecID: p.id, + }) + if err != nil { +- p.io.Cancel() +- p.io.Wait() +- p.io.Close() ++ done := make(chan struct{}) ++ go func() { ++ p.io.Cancel() ++ p.io.Wait() ++ p.io.Close() ++ close(done) ++ }() ++ select { ++ case <-time.After(30 * time.Second): ++ logrus.Warnf("process start failed with error %v, wait io close timeout, some fifo io may be dropped.", err) ++ case <-done: ++ // ok ++ } + return errdefs.FromGRPC(err) + } + p.pid = r.Pid +-- +1.8.3.1 + diff --git a/patch/0185-docker-delete-image-reference-when-failed-to-get-ima.patch b/patch/0185-docker-delete-image-reference-when-failed-to-get-ima.patch new file mode 100644 index 0000000..bf84f7b --- /dev/null +++ b/patch/0185-docker-delete-image-reference-when-failed-to-get-ima.patch @@ -0,0 +1,63 @@ +From cfc92becb2605d67a7391c43261e698d0fdd57bd Mon Sep 17 00:00:00 2001 +From: xiadanni +Date: Fri, 15 Jan 2021 15:37:42 +0800 +Subject: [PATCH] docker: delete image reference when failed to get image + configuration to avoid docker pull error + +according to patch 0110-docker-Fix-can-t-pull-image-while-the-image-i.patch, +if the layers of image has been damaged, image reference should be +deleted from repositories.json to avoid docker pull failed. + +however, when imageStore.Get failed, isExist flag has not been set to +false, which cause the image reference has still not been deleted, only +warning is printed. + +flood warnings printed every time user restarts docker daemon, like: +Jan 15 14:09:52 localhost dockerd[3952467]: +time="2021-01-15T14:09:52.705664179+08:00" level=warning msg="Failed to +get image configration for image id +sha256:d0a015ffac5ba3b9d2a641de56b3b2ed24409b7082c7811ebac4c2f4977b0965, +error: failed to get digest +sha256:d0a015ffac5ba3b9d2a641de56b3b2ed24409b7082c7811ebac4c2f4977b0965: +open +/var/lib/docker/image/devicemapper/imagedb/content/sha256/d0a015ffac5ba3b9d2a641de56b3b2ed24409b7082c7811ebac4c2f4977b0965: +no such file or directory" + +so we fix the logic, delete image reference when failed to get image +configuration. + +Signed-off-by: xiadanni +--- + components/engine/daemon/daemon.go | 7 +++---- + 1 file changed, 3 insertions(+), 4 deletions(-) + +diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go +index e826f6a..ed268d2 100644 +--- a/components/engine/daemon/daemon.go ++++ b/components/engine/daemon/daemon.go +@@ -1097,11 +1097,10 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S + return nil, err + } + +- // delete reference of image not nornamlly loaded to imageStore +- var isExist bool ++ // delete reference of image not normally loaded to imageStore + for _, imageID := range rs.List() { ++ isExist := false + if img, err := imageStore.Get(image.ID(imageID)); err == nil { +- isExist = false + if chainID := img.RootFS.ChainID(); chainID != "" { + l, err := layerStores[runtime.GOOS].Get(chainID) + if err == nil { +@@ -1112,7 +1111,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S + isExist = true + } + } else { +- logrus.Warnf("Failed to get image configration for image id %s, error: %s", imageID, err) ++ logrus.Warnf("Failed to get image configuration for image id %s, error: %s", imageID, err) + } + + // If the image not exist locally, delete its reference +-- +1.8.3.1 + diff --git a/series.conf b/series.conf index 1b6b8cf..13fd73b 100644 --- a/series.conf +++ b/series.conf @@ -162,12 +162,22 @@ patch/0163-docker-delete-event-is-not-need-to-process.patch patch/0164-docker-stat-process-exit-file-when-kill-process-dire.patch patch/0165-docker-sync-cli-vendor.patch patch/0167-docker-fix-CVE-2020-13401.patch +patch/0167-dockerd-add-more-messages-for-ops-when-device-not-fo.patch patch/0168-docker-do-not-add-w-to-LDFLAGS.patch patch/0169-docker-add-files-in-proc-for-mask.patch patch/0170-docker-fix-docker-load-files-leak.patch patch/0171-docker-do-not-sync-if-BYPAAS_SYNC-is-false.patch patch/0172-docker-fix-panic-on-single-character-volumes.patch patch/0173-docker-fix-stats-memory-usage-display-error.patch -patch/0174-docker-add-more-messages-for-ops-when-device-not-fo.patch patch/0175-docker-mask-proc-pin_memory.patch +patch/0175-docker-clean-docker-load-leak-files.patch +patch/0176-docker-kill-container-process-if-its-status-is-not-r.patch +patch/0177-resume-suspend-dm-on-start.patch +patch/0178-docker-skip-kill-and-restart-containerd-during-docke.patch +patch/0179-handle-exit-force.patch +patch/0180-wait-io-with-timeout-in-task-delete.patch +patch/0181-docker-do-not-return-when-matched-registry-mirror.patch +patch/0183-add-masked-paths-pagealloc_module-and-slaballoc_stat.patch +patch/0184-docker-wait-io-with-timeout-when-process-Start-faile.patch +patch/0185-docker-delete-image-reference-when-failed-to-get-ima.patch #end -- Gitee