diff --git a/README.en.md b/README.en.md deleted file mode 100644 index 1e52e232a07270ce6daad5db5b4f949d77fb64a0..0000000000000000000000000000000000000000 --- a/README.en.md +++ /dev/null @@ -1,36 +0,0 @@ -# containerd - -#### Description -{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**} - -#### Software Architecture -Software architecture description - -#### Installation - -1. xxxx -2. xxxx -3. xxxx - -#### Instructions - -1. xxxx -2. xxxx -3. xxxx - -#### Contribution - -1. Fork the repository -2. Create Feat_xxx branch -3. Commit your code -4. Create Pull Request - - -#### Gitee Feature - -1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md -2. Gitee blog [blog.gitee.com](https://blog.gitee.com) -3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) -4. The most valuable open source project [GVP](https://gitee.com/gvp) -5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) -6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/apply-patch b/apply-patch new file mode 100755 index 0000000000000000000000000000000000000000..2a96e0d240149d66984ae1feadb076c92541d4b0 --- /dev/null +++ b/apply-patch @@ -0,0 +1,36 @@ +#! /bin/bash + +# Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved. +# Description: This shell script is used to apply patches for the project +# Author: zhangyu235@huawei.com +# Create: 2019-05-17 + +set -ex + +pkg=containerd-1.2.0 +cwd=$PWD +src=$cwd/$pkg + +unzip $pkg.zip + +cd $src +git init +git add . +git config user.name 'build' +git config user.email 'build@obs.com' +git commit -m 'init build' +cd $cwd + +series=$cwd/series.conf +while IPF= read -r line +do + if [[ "$line" =~ ^patch* ]]; then + echo git apply $cwd/$line + cd $src && git apply $cwd/$line + fi +done <"$series" + +cd $cwd +cp -rf $src/* . +mv $src/.git $src/git +rm -rf containerd-1.2.0 diff --git a/containerd-1.2.0.zip b/containerd-1.2.0.zip new file mode 100644 index 0000000000000000000000000000000000000000..724bf394bd7162f785ba610c34d47eaaacd32813 Binary files /dev/null and b/containerd-1.2.0.zip differ diff --git a/containerd.spec b/containerd.spec new file mode 100644 index 0000000000000000000000000000000000000000..5803f05f28e03a64edbc25d503a0af676cccb7d8 --- /dev/null +++ b/containerd.spec @@ -0,0 +1,43 @@ +%global goipath github.com/containerd/containerd +%global debug_package %{nil} +Version: 1.2.0 +Name: containerd +Release: 100 +Summary: An industry-standard container runtime +License: ASL 2.0 +URL: https://containerd.io +Source0: %{name}-%{version}.tar.gz + +BuildRequires: golang glibc-static make btrfs-progs-devel + +%description +containerd is an industry-standard container runtime with an emphasis on +simplicity, robustness and portability. It is available as a daemon for Linux +and Windows, which can manage the complete container lifecycle of its host +system: image transfer and storage, container execution and supervision, +low-level storage and network attachments, etc. + +%prep +%setup -c -n containerd + +%build +./apply-patch + +GO_BUILD_PATH=$PWD/_build +install -m 0755 -vd $(dirname $GO_BUILD_PATH/src/%{goipath}) +ln -fs $PWD $GO_BUILD_PATH/src/%{goipath} +cd $GO_BUILD_PATH/src/%{goipath} +export GOPATH=$GO_BUILD_PATH:%{gopath} +export BUILDTAGS="no_btrfs no_cri" +make + +%install +install -d $RPM_BUILD_ROOT/%{_bindir} +install -p -m 755 bin/containerd $RPM_BUILD_ROOT/%{_bindir}/containerd +install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim + +%files +%{_bindir}/containerd +%{_bindir}/containerd-shim + +%changelog diff --git a/patch/0001-grpc-vendor-grpc-fix-grpc-map-panic.patch b/patch/0001-grpc-vendor-grpc-fix-grpc-map-panic.patch new file mode 100644 index 0000000000000000000000000000000000000000..d350d2414e8e72733e9d39edf1931a257ae1f09f --- /dev/null +++ b/patch/0001-grpc-vendor-grpc-fix-grpc-map-panic.patch @@ -0,0 +1,63 @@ +From fe090d706a522392e30dd4c44447f915ec99c1a0 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sat, 22 Dec 2018 15:16:53 +0800 +Subject: [PATCH 01/27] vendor: grpc: fix grpc map panic + +reason: Fix grpc map panic + +cherry-pick from containerd-0.2.8 + +a8cdda827867cec97568318368a7aa40097d0487 + +Fix grpc map panic + +Description: + In golang, if we read/write map in different goroutine, it may panic. + We need to add lock to protect the map data when read/write the map. + +Now the grpc map is only protected by a mutex while register, not +protected in reading process(handleStream function). + +This MR will use a RWMutex to protect this map. + +Change-Id: I786bd99234461c40fcb57621fd7c1fb4faa0c208 +Signed-off-by: jingrui +--- + vendor/google.golang.org/grpc/server.go | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go +index 4969331..77f7840 100644 +--- a/vendor/google.golang.org/grpc/server.go ++++ b/vendor/google.golang.org/grpc/server.go +@@ -90,7 +90,7 @@ type service struct { + type Server struct { + opts options + +- mu sync.Mutex // guards following ++ mu sync.RWMutex // guards following + lis map[net.Listener]bool + conns map[io.Closer]bool + serve bool +@@ -438,6 +438,8 @@ type ServiceInfo struct { + // Service names include the package names, in the form of .. + func (s *Server) GetServiceInfo() map[string]ServiceInfo { + ret := make(map[string]ServiceInfo) ++ s.mu.RLock() ++ defer s.mu.RUnlock() + for n, srv := range s.m { + methods := make([]MethodInfo, 0, len(srv.md)+len(srv.sd)) + for m := range srv.md { +@@ -1221,7 +1223,9 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str + } + service := sm[:pos] + method := sm[pos+1:] ++ s.mu.RLock() + srv, ok := s.m[service] ++ s.mu.RUnlock() + if !ok { + if unknownDesc := s.opts.unknownStreamDesc; unknownDesc != nil { + s.processStreamingRPC(t, stream, nil, unknownDesc, trInfo) +-- +2.7.4.3 + diff --git a/patch/0002-sys-sys-count-steal-time-when-calculating-Sys.patch b/patch/0002-sys-sys-count-steal-time-when-calculating-Sys.patch new file mode 100644 index 0000000000000000000000000000000000000000..57dc16bdd680a05d10ff397a623bea952c6cf35c --- /dev/null +++ b/patch/0002-sys-sys-count-steal-time-when-calculating-Sys.patch @@ -0,0 +1,44 @@ +From 003dc7956765712fdf4a893c2d541af2e2d0f300 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sat, 22 Dec 2018 15:44:50 +0800 +Subject: [PATCH 02/27] sys: sys: count steal time when calculating + SystemCPUUsage + +reason: count steal time when calculating SystemCPUUsage + +cherry-pick from containerd-0.2.8 + +13f22eecd33d30520ace277822ac5f0acb387e75 + +containerd: count steal time when calculating SystemCPUUsage + +[Changelog]: when counting docker stat in virtual machines, now containerd do not count steal time when calculating SystemCPUUsage, which causes that cpuusage value larger than its actua$ +[Author]git + +Change-Id: I2b62c9508cbdc444d514116f4bea5aad3d292af5 +Signed-off-by: jingrui +--- + sys/proc.go | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/sys/proc.go b/sys/proc.go +index 496eb1f..82a6351 100644 +--- a/sys/proc.go ++++ b/sys/proc.go +@@ -61,11 +61,11 @@ func GetSystemCPUUsage() (uint64, error) { + parts := strings.Fields(line) + switch parts[0] { + case "cpu": +- if len(parts) < 8 { ++ if len(parts) < 9 { + return 0, fmt.Errorf("bad format of cpu stats") + } + var totalClockTicks uint64 +- for _, i := range parts[1:8] { ++ for _, i := range parts[1:9] { + v, err := strconv.ParseUint(i, 10, 64) + if err != nil { + return 0, fmt.Errorf("error parsing cpu stats") +-- +2.7.4.3 + diff --git a/patch/0003-oci-oci-add-files-cgroups-support.patch b/patch/0003-oci-oci-add-files-cgroups-support.patch new file mode 100644 index 0000000000000000000000000000000000000000..41c89e942cf688f6e455d08606a68c6540112cf2 --- /dev/null +++ b/patch/0003-oci-oci-add-files-cgroups-support.patch @@ -0,0 +1,51 @@ +From c9cc468949d80c663524f5b764e2c661af13bca2 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sat, 22 Dec 2018 16:25:07 +0800 +Subject: [PATCH 03/27] oci: oci: add files cgroups support + +reason: Add file fds limit + +cherry-pick from containerd-0.2.8 + +29b822599b86f823d5a9f94df1cdceea485e0b19 + +Add file fds limit + +With the patch(https://lwn.net/Articles/604129/),we can limit the +num of open files in container. + +Change-Id: I72b45430dd7535727c4af9e190bbb345ba8ee316 +Signed-off-by: jingrui +--- + vendor/github.com/opencontainers/runtime-spec/specs-go/config.go | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go +index f32698c..ac24cde 100644 +--- a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go ++++ b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go +@@ -314,6 +314,12 @@ type LinuxPids struct { + Limit int64 `json:"limit"` + } + ++// Files for Linux cgroup 'files' resource management (https://lwn.net/Articles/604129/) ++type Files struct { ++ // Maximum number of open files". ++ Limit *int64 `json:"limit,omitempty"` ++} ++ + // LinuxNetwork identification and priority configuration + type LinuxNetwork struct { + // Set class identifier for container's network packets +@@ -340,6 +346,8 @@ type LinuxResources struct { + CPU *LinuxCPU `json:"cpu,omitempty"` + // Task resource restriction configuration. + Pids *LinuxPids `json:"pids,omitempty"` ++ // Files resource restriction configuration. ++ Files *Files `json:"files,omitempty"` + // BlockIO restriction configuration + BlockIO *LinuxBlockIO `json:"blockIO,omitempty"` + // Hugetlb limit (in bytes) +-- +2.7.4.3 + diff --git a/patch/0004-runv-vendor-runv-compatibility.patch b/patch/0004-runv-vendor-runv-compatibility.patch new file mode 100644 index 0000000000000000000000000000000000000000..d86faddacaf2f934ad913239b2b95eb6842e97c6 --- /dev/null +++ b/patch/0004-runv-vendor-runv-compatibility.patch @@ -0,0 +1,63 @@ +From 5fa863a6ea74ed24cfcc0c16eaa5e5a4e77387ec Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Wed, 26 Dec 2018 12:08:20 +0800 +Subject: [PATCH 04/27] runv: vendor: runv compatibility + +reason: fix manslaughter of runtime delete process + +cherry-pick from containerd-0.2.8 + +reference: + +7906753998667b5a9fa9a996f4a0e41d4736d5c1 + +contaierd-17: fix manslaughter of runtime delete process + +fix manslaughter of runtime delete process + +f82956a89ca7d7cea3bdd5fcd4d4fd70c313f378 + +containerd-17: fix qemu remaining when dockerd restart + +fix qemu remaining when dockerd restart and container start concurrency + +Change-Id: Id23456e90961041194c946a289ae790327b874c8 +Signed-off-by: jingrui +--- + vendor/github.com/containerd/go-runc/command_linux.go | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/vendor/github.com/containerd/go-runc/command_linux.go b/vendor/github.com/containerd/go-runc/command_linux.go +index 71b52f9..6ad27be 100644 +--- a/vendor/github.com/containerd/go-runc/command_linux.go ++++ b/vendor/github.com/containerd/go-runc/command_linux.go +@@ -20,9 +20,17 @@ import ( + "context" + "os" + "os/exec" ++ "strings" + "syscall" + ) + ++func (r *Runc) isrunv() bool { ++ if strings.Contains(r.Command, "runv") { ++ return true ++ } ++ return false ++} ++ + func (r *Runc) command(context context.Context, args ...string) *exec.Cmd { + command := r.Command + if command == "" { +@@ -33,7 +41,7 @@ func (r *Runc) command(context context.Context, args ...string) *exec.Cmd { + Setpgid: r.Setpgid, + } + cmd.Env = os.Environ() +- if r.PdeathSignal != 0 { ++ if r.PdeathSignal != 0 && !r.isrunv() { + cmd.SysProcAttr.Pdeathsig = r.PdeathSignal + } + +-- +2.7.4.3 + diff --git a/patch/0005-containerd-add-spec-for-build.patch b/patch/0005-containerd-add-spec-for-build.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a67a028fe3f22ec0226ee07438a051eb2c14c14 --- /dev/null +++ b/patch/0005-containerd-add-spec-for-build.patch @@ -0,0 +1,69 @@ +From 8e46f370733951e6decec6dd36b0c13308ced2c2 Mon Sep 17 00:00:00 2001 +From: caihaomin +Date: Mon, 21 Jan 2019 22:31:05 +0800 +Subject: [PATCH 05/27] containerd: add spec for build + +reason:add spec for build + +Change-Id: I42d9d32e4898c006194df1ead4735155b4785584 +Signed-off-by: caihaomin +--- + hack/containerd.spec | 46 ++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 46 insertions(+) + create mode 100644 hack/containerd.spec + +diff --git a/hack/containerd.spec b/hack/containerd.spec +new file mode 100644 +index 0000000..f53c37b +--- /dev/null ++++ b/hack/containerd.spec +@@ -0,0 +1,46 @@ ++%global goipath github.com/containerd/containerd ++%global debug_package %{nil} ++Version: 1.2.0 ++ ++Name: containerd ++Release: 1%{?dist} ++Summary: An industry-standard container runtime ++License: ASL 2.0 ++URL: https://containerd.io ++Source0: containerd-1.2.0.tar.gz ++ ++BuildRequires: golang glibc-static make ++BuildRequires: btrfs-progs-devel ++ ++ ++%description ++containerd is an industry-standard container runtime with an emphasis on ++simplicity, robustness and portability. It is available as a daemon for Linux ++and Windows, which can manage the complete container lifecycle of its host ++system: image transfer and storage, container execution and supervision, ++low-level storage and network attachments, etc. ++ ++ ++%prep ++%setup -c -n containerd ++ ++%build ++GO_BUILD_PATH=$PWD/_build ++install -m 0755 -vd $(dirname $GO_BUILD_PATH/src/%{goipath}) ++ln -fs $PWD $GO_BUILD_PATH/src/%{goipath} ++cd $GO_BUILD_PATH/src/%{goipath} ++export GOPATH=$GO_BUILD_PATH:%{gopath} ++export BUILDTAGS="no_btrfs no_cri" ++make ++ ++%install ++install -d $RPM_BUILD_ROOT/%{_bindir} ++install -p -m 755 bin/containerd $RPM_BUILD_ROOT/%{_bindir}/containerd ++install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim ++ ++%files ++%{_bindir}/containerd ++%{_bindir}/containerd-shim ++ ++ ++%changelog +-- +2.7.4.3 + diff --git a/patch/0006-shim-optimize-shim-lock-in-runtime-v1.patch b/patch/0006-shim-optimize-shim-lock-in-runtime-v1.patch new file mode 100644 index 0000000000000000000000000000000000000000..5b46980e80a939fa13badd0ee93191ada8e9ce38 --- /dev/null +++ b/patch/0006-shim-optimize-shim-lock-in-runtime-v1.patch @@ -0,0 +1,320 @@ +From 31621148229d56835575189c71e80339fba9f1fc Mon Sep 17 00:00:00 2001 +From: lujingxiao +Date: Wed, 23 Jan 2019 14:55:27 +0800 +Subject: [PATCH 06/27] shim: optimize shim lock in runtime v1 + +reason: apply lock only around process map of shim service, +avoid lock affect other procs operations. + +Cherry-pick from upstream c206da795 + +Change-Id: I33f0f6b3537673533fdb60afb7a0295ac9665f11 +Signed-off-by: Ace-Tang +Signed-off-by: lujingxiao +--- + runtime/v1/shim/service.go | 144 +++++++++++++++++++++++---------------------- + 1 file changed, 75 insertions(+), 69 deletions(-) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index d76d580..679982a 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -114,9 +114,6 @@ type Service struct { + + // Create a new initial process and container with the underlying OCI runtime + func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *shimapi.CreateTaskResponse, err error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- + var mounts []proc.Mount + for _, m := range r.Rootfs { + mounts = append(mounts, proc.Mount{ +@@ -158,6 +155,10 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ * + return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m) + } + } ++ ++ s.mu.Lock() ++ defer s.mu.Unlock() ++ + process, err := newInit( + ctx, + s.config.Path, +@@ -187,11 +188,9 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ * + + // Start a process + func (s *Service) Start(ctx context.Context, r *shimapi.StartRequest) (*shimapi.StartResponse, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[r.ID] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process %s", r.ID) ++ p, err := s.getExecProcess(r.ID) ++ if err != nil { ++ return nil, err + } + if err := p.Start(ctx); err != nil { + return nil, err +@@ -204,16 +203,16 @@ func (s *Service) Start(ctx context.Context, r *shimapi.StartRequest) (*shimapi. + + // Delete the initial process and container + func (s *Service) Delete(ctx context.Context, r *ptypes.Empty) (*shimapi.DeleteResponse, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[s.id] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getInitProcess() ++ if err != nil { ++ return nil, err + } + if err := p.Delete(ctx); err != nil { + return nil, err + } ++ s.mu.Lock() + delete(s.processes, s.id) ++ s.mu.Unlock() + s.platform.Close() + return &shimapi.DeleteResponse{ + ExitStatus: uint32(p.ExitStatus()), +@@ -227,11 +226,9 @@ func (s *Service) DeleteProcess(ctx context.Context, r *shimapi.DeleteProcessReq + if r.ID == s.id { + return nil, status.Errorf(codes.InvalidArgument, "cannot delete init process with DeleteProcess") + } +- s.mu.Lock() +- p := s.processes[r.ID] +- s.mu.Unlock() +- if p == nil { +- return nil, errors.Wrapf(errdefs.ErrNotFound, "process %s", r.ID) ++ p, err := s.getExecProcess(r.ID) ++ if err != nil { ++ return nil, err + } + if err := p.Delete(ctx); err != nil { + return nil, err +@@ -249,13 +246,14 @@ func (s *Service) DeleteProcess(ctx context.Context, r *shimapi.DeleteProcessReq + // Exec an additional process inside the container + func (s *Service) Exec(ctx context.Context, r *shimapi.ExecProcessRequest) (*ptypes.Empty, error) { + s.mu.Lock() +- defer s.mu.Unlock() + + if p := s.processes[r.ID]; p != nil { ++ s.mu.Unlock() + return nil, errdefs.ToGRPCf(errdefs.ErrAlreadyExists, "id %s", r.ID) + } + + p := s.processes[s.id] ++ s.mu.Unlock() + if p == nil { + return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + } +@@ -271,14 +269,14 @@ func (s *Service) Exec(ctx context.Context, r *shimapi.ExecProcessRequest) (*pty + if err != nil { + return nil, errdefs.ToGRPC(err) + } ++ s.mu.Lock() + s.processes[r.ID] = process ++ s.mu.Unlock() + return empty, nil + } + + // ResizePty of a process + func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (*ptypes.Empty, error) { +- s.mu.Lock() +- defer s.mu.Unlock() + if r.ID == "" { + return nil, errdefs.ToGRPCf(errdefs.ErrInvalidArgument, "id not provided") + } +@@ -286,7 +284,9 @@ func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (* + Width: uint16(r.Width), + Height: uint16(r.Height), + } ++ s.mu.Lock() + p := s.processes[r.ID] ++ s.mu.Unlock() + if p == nil { + return nil, errors.Errorf("process does not exist %s", r.ID) + } +@@ -298,11 +298,9 @@ func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (* + + // State returns runtime state information for a process + func (s *Service) State(ctx context.Context, r *shimapi.StateRequest) (*shimapi.StateResponse, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[r.ID] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process id %s", r.ID) ++ p, err := s.getExecProcess(r.ID) ++ if err != nil { ++ return nil, err + } + st, err := p.Status(ctx) + if err != nil { +@@ -338,11 +336,9 @@ func (s *Service) State(ctx context.Context, r *shimapi.StateRequest) (*shimapi. + + // Pause the container + func (s *Service) Pause(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[s.id] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getInitProcess() ++ if err != nil { ++ return nil, err + } + if err := p.(*proc.Init).Pause(ctx); err != nil { + return nil, err +@@ -352,11 +348,9 @@ func (s *Service) Pause(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, er + + // Resume the container + func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[s.id] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getInitProcess() ++ if err != nil { ++ return nil, err + } + if err := p.(*proc.Init).Resume(ctx); err != nil { + return nil, err +@@ -366,12 +360,10 @@ func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, e + + // Kill a process with the provided signal + func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Empty, error) { +- s.mu.Lock() +- defer s.mu.Unlock() + if r.ID == "" { +- p := s.processes[s.id] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getInitProcess() ++ if err != nil { ++ return nil, err + } + if err := p.Kill(ctx, r.Signal, r.All); err != nil { + return nil, errdefs.ToGRPC(err) +@@ -379,9 +371,9 @@ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Emp + return empty, nil + } + +- p := s.processes[r.ID] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process id %s not found", r.ID) ++ p, err := s.getExecProcess(r.ID) ++ if err != nil { ++ return nil, err + } + if err := p.Kill(ctx, r.Signal, r.All); err != nil { + return nil, errdefs.ToGRPC(err) +@@ -422,11 +414,9 @@ func (s *Service) ListPids(ctx context.Context, r *shimapi.ListPidsRequest) (*sh + + // CloseIO of a process + func (s *Service) CloseIO(ctx context.Context, r *shimapi.CloseIORequest) (*ptypes.Empty, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[r.ID] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process does not exist %s", r.ID) ++ p, err := s.getExecProcess(r.ID) ++ if err != nil { ++ return nil, err + } + if stdin := p.Stdin(); stdin != nil { + if err := stdin.Close(); err != nil { +@@ -438,11 +428,9 @@ func (s *Service) CloseIO(ctx context.Context, r *shimapi.CloseIORequest) (*ptyp + + // Checkpoint the container + func (s *Service) Checkpoint(ctx context.Context, r *shimapi.CheckpointTaskRequest) (*ptypes.Empty, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[s.id] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getInitProcess() ++ if err != nil { ++ return nil, err + } + var options runctypes.CheckpointOptions + if r.Options != nil { +@@ -475,11 +463,9 @@ func (s *Service) ShimInfo(ctx context.Context, r *ptypes.Empty) (*shimapi.ShimI + + // Update a running container + func (s *Service) Update(ctx context.Context, r *shimapi.UpdateTaskRequest) (*ptypes.Empty, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[s.id] +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getInitProcess() ++ if err != nil { ++ return nil, err + } + if err := p.(*proc.Init).Update(ctx, r.Resources); err != nil { + return nil, errdefs.ToGRPC(err) +@@ -489,11 +475,9 @@ func (s *Service) Update(ctx context.Context, r *shimapi.UpdateTaskRequest) (*pt + + // Wait for a process to exit + func (s *Service) Wait(ctx context.Context, r *shimapi.WaitRequest) (*shimapi.WaitResponse, error) { +- s.mu.Lock() +- p := s.processes[r.ID] +- s.mu.Unlock() +- if p == nil { +- return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getExecProcess(r.ID) ++ if err != nil { ++ return nil, err + } + p.Wait() + +@@ -563,11 +547,9 @@ func shouldKillAllOnExit(bundlePath string) (bool, error) { + } + + func (s *Service) getContainerPids(ctx context.Context, id string) ([]uint32, error) { +- s.mu.Lock() +- defer s.mu.Unlock() +- p := s.processes[s.id] +- if p == nil { +- return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "container must be created") ++ p, err := s.getInitProcess() ++ if err != nil { ++ return nil, err + } + + ps, err := p.(*proc.Init).Runtime().Ps(ctx, id) +@@ -589,6 +571,30 @@ func (s *Service) forward(publisher events.Publisher) { + } + } + ++// getInitProcess returns initial process ++func (s *Service) getInitProcess() (rproc.Process, error) { ++ s.mu.Lock() ++ defer s.mu.Unlock() ++ ++ p := s.processes[s.id] ++ if p == nil { ++ return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") ++ } ++ return p, nil ++} ++ ++// getExecProcess returns exec process ++func (s *Service) getExecProcess(id string) (rproc.Process, error) { ++ s.mu.Lock() ++ defer s.mu.Unlock() ++ ++ p := s.processes[id] ++ if p == nil { ++ return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process %s does not exist", id) ++ } ++ return p, nil ++} ++ + func getTopic(ctx context.Context, e interface{}) string { + switch e.(type) { + case *eventstypes.TaskCreate: +-- +2.7.4.3 + diff --git a/patch/0007-shim-Increase-reaper-buffer-size-and-non-bl.patch b/patch/0007-shim-Increase-reaper-buffer-size-and-non-bl.patch new file mode 100644 index 0000000000000000000000000000000000000000..5f17e233e8c095e3c91a36b39e7d7b1747489775 --- /dev/null +++ b/patch/0007-shim-Increase-reaper-buffer-size-and-non-bl.patch @@ -0,0 +1,109 @@ +From 2e143a25ff02800afb569352c407cf71a9c0312b Mon Sep 17 00:00:00 2001 +From: lujingxiao +Date: Wed, 23 Jan 2019 14:56:19 +0800 +Subject: [PATCH 07/27] shim: Increase reaper buffer size and + non-blocking send + +reason: Fixes #2709 + +This increases the buffer size for process exit subscribers. It also +implements a non-blocking send on the subscriber channel. It is better +to drop an exit even than it is to block a shim for one slow subscriber. + +Cherry-pick from upstream 232a063496 + +Change-Id: Ibf9f06cc82945a8592fb02a87816d69d5dac2b6b +Signed-off-by: Michael Crosby +Signed-off-by: lujingxiao +--- + runtime/v1/shim/reaper.go | 14 +++++++++++--- + runtime/v2/shim/reaper_unix.go | 14 +++++++++++--- + 2 files changed, 22 insertions(+), 6 deletions(-) + +diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go +index 2937f1a..10d5c30 100644 +--- a/runtime/v1/shim/reaper.go ++++ b/runtime/v1/shim/reaper.go +@@ -26,12 +26,13 @@ import ( + "github.com/containerd/containerd/sys" + runc "github.com/containerd/go-runc" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + // ErrNoSuchProcess is returned when the process no longer exists + var ErrNoSuchProcess = errors.New("no such process") + +-const bufferSize = 32 ++const bufferSize = 2048 + + // Reap should be called when the process receives an SIGCHLD. Reap will reap + // all exited processes and close their wait channels +@@ -41,13 +42,20 @@ func Reap() error { + Default.Lock() + for c := range Default.subscribers { + for _, e := range exits { +- c <- runc.Exit{ ++ select { ++ case c <- runc.Exit{ + Timestamp: now, + Pid: e.Pid, + Status: e.Status, ++ }: ++ default: ++ logrus.WithFields(logrus.Fields{ ++ "subscriber": c, ++ "pid": e.Pid, ++ "status": e.Status, ++ }).Warn("failed to send exit to subscriber") + } + } +- + } + Default.Unlock() + return err +diff --git a/runtime/v2/shim/reaper_unix.go b/runtime/v2/shim/reaper_unix.go +index 2937f1a..10d5c30 100644 +--- a/runtime/v2/shim/reaper_unix.go ++++ b/runtime/v2/shim/reaper_unix.go +@@ -26,12 +26,13 @@ import ( + "github.com/containerd/containerd/sys" + runc "github.com/containerd/go-runc" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + // ErrNoSuchProcess is returned when the process no longer exists + var ErrNoSuchProcess = errors.New("no such process") + +-const bufferSize = 32 ++const bufferSize = 2048 + + // Reap should be called when the process receives an SIGCHLD. Reap will reap + // all exited processes and close their wait channels +@@ -41,13 +42,20 @@ func Reap() error { + Default.Lock() + for c := range Default.subscribers { + for _, e := range exits { +- c <- runc.Exit{ ++ select { ++ case c <- runc.Exit{ + Timestamp: now, + Pid: e.Pid, + Status: e.Status, ++ }: ++ default: ++ logrus.WithFields(logrus.Fields{ ++ "subscriber": c, ++ "pid": e.Pid, ++ "status": e.Status, ++ }).Warn("failed to send exit to subscriber") + } + } +- + } + Default.Unlock() + return err +-- +2.7.4.3 + diff --git a/patch/0008-runtime-Use-named-pipes-for-shim-logs.patch b/patch/0008-runtime-Use-named-pipes-for-shim-logs.patch new file mode 100644 index 0000000000000000000000000000000000000000..ba947856ca5a7c7c6c6ea63d09ff642e6aa34257 --- /dev/null +++ b/patch/0008-runtime-Use-named-pipes-for-shim-logs.patch @@ -0,0 +1,578 @@ +From 9bdd5d485c6796c44356ae9482df8de467463feb Mon Sep 17 00:00:00 2001 +From: lujingxiao +Date: Wed, 23 Jan 2019 14:57:41 +0800 +Subject: [PATCH 08/27] runtime: Use named pipes for shim logs + +reason: TestDaemonRestart hangs if shim_debug is enabled +Relating to issue [#2606](https://github.com/containerd/containerd/issues/2606) + +Co-authored-by: Oliver Stenbom +Co-authored-by: Georgi Sabev +Co-authored-by: Giuseppe Capizzi +Co-authored-by: Danail Branekov + +Cherry-pick from upstream 1d4105cacf + +Change-Id: I0038401dda88c234750e8d1378a4dd97230400b0 +Signed-off-by: Oliver Stenbom +Signed-off-by: Georgi Sabev +Signed-off-by: Giuseppe Capizzi +Signed-off-by: Danail Branekov +Signed-off-by: lujingxiao +--- + client_test.go | 49 +++++++-- + cmd/containerd-shim/main_unix.go | 28 ++++++ + container_linux_test.go | 209 +++++++++++++++++++++++++++++++++++++++ + runtime/v1/linux/runtime.go | 26 +++++ + runtime/v1/shim.go | 38 +++++++ + runtime/v1/shim/client/client.go | 34 +++++-- + 6 files changed, 370 insertions(+), 14 deletions(-) + create mode 100644 runtime/v1/shim.go + +diff --git a/client_test.go b/client_test.go +index a6b1d59..1a4cf39 100644 +--- a/client_test.go ++++ b/client_test.go +@@ -21,6 +21,8 @@ import ( + "context" + "flag" + "fmt" ++ "io" ++ "io/ioutil" + "os" + "os/exec" + "testing" +@@ -36,11 +38,12 @@ import ( + ) + + var ( +- address string +- noDaemon bool +- noCriu bool +- supportsCriu bool +- testNamespace = "testing" ++ address string ++ noDaemon bool ++ noCriu bool ++ supportsCriu bool ++ testNamespace = "testing" ++ ctrdStdioFilePath string + + ctrd = &daemon{} + ) +@@ -76,13 +79,26 @@ func TestMain(m *testing.M) { + if !noDaemon { + sys.ForceRemoveAll(defaultRoot) + +- err := ctrd.start("containerd", address, []string{ ++ stdioFile, err := ioutil.TempFile("", "") ++ if err != nil { ++ fmt.Fprintf(os.Stderr, "could not create a new stdio temp file: %s\n", err) ++ os.Exit(1) ++ } ++ defer func() { ++ stdioFile.Close() ++ os.Remove(stdioFile.Name()) ++ }() ++ ctrdStdioFilePath = stdioFile.Name() ++ stdioWriter := io.MultiWriter(stdioFile, buf) ++ ++ err = ctrd.start("containerd", address, []string{ + "--root", defaultRoot, + "--state", defaultState, + "--log-level", "debug", +- }, buf, buf) ++ "--config", createShimDebugConfig(), ++ }, stdioWriter, stdioWriter) + if err != nil { +- fmt.Fprintf(os.Stderr, "%s: %s", err, buf.String()) ++ fmt.Fprintf(os.Stderr, "%s: %s\n", err, buf.String()) + os.Exit(1) + } + } +@@ -137,6 +153,7 @@ func TestMain(m *testing.M) { + fmt.Fprintln(os.Stderr, "failed to wait for containerd", err) + } + } ++ + if err := sys.ForceRemoveAll(defaultRoot); err != nil { + fmt.Fprintln(os.Stderr, "failed to remove test root dir", err) + os.Exit(1) +@@ -343,3 +360,19 @@ func TestClientReconnect(t *testing.T) { + t.Errorf("client closed returned error %v", err) + } + } ++ ++func createShimDebugConfig() string { ++ f, err := ioutil.TempFile("", "containerd-config-") ++ if err != nil { ++ fmt.Fprintf(os.Stderr, "Failed to create config file: %s\n", err) ++ os.Exit(1) ++ } ++ defer f.Close() ++ ++ if _, err := f.WriteString("[plugins.linux]\n\tshim_debug = true\n"); err != nil { ++ fmt.Fprintf(os.Stderr, "Failed to write to config file %s: %s\n", f.Name(), err) ++ os.Exit(1) ++ } ++ ++ return f.Name() ++} +diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go +index ca0a90a..6c59cd1 100644 +--- a/cmd/containerd-shim/main_unix.go ++++ b/cmd/containerd-shim/main_unix.go +@@ -23,6 +23,7 @@ import ( + "context" + "flag" + "fmt" ++ "io" + "net" + "os" + "os/exec" +@@ -36,6 +37,7 @@ import ( + + "github.com/containerd/containerd/events" + "github.com/containerd/containerd/namespaces" ++ shimlog "github.com/containerd/containerd/runtime/v1" + "github.com/containerd/containerd/runtime/v1/linux/proc" + "github.com/containerd/containerd/runtime/v1/shim" + shimapi "github.com/containerd/containerd/runtime/v1/shim/v1" +@@ -92,12 +94,38 @@ func main() { + runtime.GOMAXPROCS(2) + } + ++ stdout, stderr, err := openStdioKeepAlivePipes(workdirFlag) ++ if err != nil { ++ fmt.Fprintf(os.Stderr, "containerd-shim: %s\n", err) ++ os.Exit(1) ++ } ++ defer func() { ++ stdout.Close() ++ stderr.Close() ++ }() ++ + if err := executeShim(); err != nil { + fmt.Fprintf(os.Stderr, "containerd-shim: %s\n", err) + os.Exit(1) + } + } + ++// If containerd server process dies, we need the shim to keep stdout/err reader ++// FDs so that Linux does not SIGPIPE the shim process if it tries to use its end of ++// these pipes. ++func openStdioKeepAlivePipes(dir string) (io.ReadCloser, io.ReadCloser, error) { ++ background := context.Background() ++ keepStdoutAlive, err := shimlog.OpenShimStdoutLog(background, dir) ++ if err != nil { ++ return nil, nil, err ++ } ++ keepStderrAlive, err := shimlog.OpenShimStderrLog(background, dir) ++ if err != nil { ++ return nil, nil, err ++ } ++ return keepStdoutAlive, keepStderrAlive, nil ++} ++ + func executeShim() error { + // start handling signals as soon as possible so that things are properly reaped + // or if runtime exits before we hit the handler +diff --git a/container_linux_test.go b/container_linux_test.go +index 60b0336..fa764d7 100644 +--- a/container_linux_test.go ++++ b/container_linux_test.go +@@ -24,7 +24,9 @@ import ( + "fmt" + "io" + "io/ioutil" ++ "os" + "os/exec" ++ "path/filepath" + "runtime" + "strings" + "sync" +@@ -258,6 +260,213 @@ func TestDaemonRestart(t *testing.T) { + <-statusC + } + ++func TestShimDoesNotLeakPipes(t *testing.T) { ++ containerdPid := ctrd.cmd.Process.Pid ++ initialPipes, err := numPipes(containerdPid) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ client, err := newClient(t, address) ++ if err != nil { ++ t.Fatal(err) ++ } ++ defer client.Close() ++ ++ var ( ++ image Image ++ ctx, cancel = testContext() ++ id = t.Name() ++ ) ++ defer cancel() ++ ++ image, err = client.GetImage(ctx, testImage) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30"))) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ task, err := container.NewTask(ctx, empty()) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ exitChannel, err := task.Wait(ctx) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ if err := task.Start(ctx); err != nil { ++ t.Fatal(err) ++ } ++ ++ if err := task.Kill(ctx, syscall.SIGKILL); err != nil { ++ t.Fatal(err) ++ } ++ ++ <-exitChannel ++ ++ if _, err := task.Delete(ctx); err != nil { ++ t.Fatal(err) ++ } ++ ++ if err := container.Delete(ctx, WithSnapshotCleanup); err != nil { ++ t.Fatal(err) ++ } ++ ++ currentPipes, err := numPipes(containerdPid) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ if initialPipes != currentPipes { ++ t.Errorf("Pipes have leaked after container has been deleted. Initially there were %d pipes, after container deletion there were %d pipes", initialPipes, currentPipes) ++ } ++} ++ ++func numPipes(pid int) (int, error) { ++ cmd := exec.Command("sh", "-c", fmt.Sprintf("lsof -p %d | grep pipe", pid)) ++ ++ var stdout bytes.Buffer ++ cmd.Stdout = &stdout ++ if err := cmd.Run(); err != nil { ++ return 0, err ++ } ++ return strings.Count(stdout.String(), "\n"), nil ++} ++ ++func TestDaemonReconnectsToShimIOPipesOnRestart(t *testing.T) { ++ client, err := newClient(t, address) ++ if err != nil { ++ t.Fatal(err) ++ } ++ defer client.Close() ++ ++ var ( ++ image Image ++ ctx, cancel = testContext() ++ id = t.Name() ++ ) ++ defer cancel() ++ ++ image, err = client.GetImage(ctx, testImage) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30"))) ++ if err != nil { ++ t.Fatal(err) ++ } ++ defer container.Delete(ctx, WithSnapshotCleanup) ++ ++ task, err := container.NewTask(ctx, empty()) ++ if err != nil { ++ t.Fatal(err) ++ } ++ defer task.Delete(ctx) ++ ++ _, err = task.Wait(ctx) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ if err := task.Start(ctx); err != nil { ++ t.Fatal(err) ++ } ++ ++ if err := ctrd.Restart(nil); err != nil { ++ t.Fatal(err) ++ } ++ ++ waitCtx, waitCancel := context.WithTimeout(ctx, 2*time.Second) ++ serving, err := client.IsServing(waitCtx) ++ waitCancel() ++ if !serving { ++ t.Fatalf("containerd did not start within 2s: %v", err) ++ } ++ ++ // After we restared containerd we write some messages to the log pipes, simulating shim writing stuff there. ++ // Then we make sure that these messages are available on the containerd log thus proving that the server reconnected to the log pipes ++ runtimeVersion := getRuntimeVersion() ++ logDirPath := getLogDirPath(runtimeVersion, id) ++ ++ switch runtimeVersion { ++ case "v1": ++ writeToFile(t, filepath.Join(logDirPath, "shim.stdout.log"), fmt.Sprintf("%s writing to stdout\n", id)) ++ writeToFile(t, filepath.Join(logDirPath, "shim.stderr.log"), fmt.Sprintf("%s writing to stderr\n", id)) ++ case "v2": ++ writeToFile(t, filepath.Join(logDirPath, "log"), fmt.Sprintf("%s writing to log\n", id)) ++ } ++ ++ statusC, err := task.Wait(ctx) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ if err := task.Kill(ctx, syscall.SIGKILL); err != nil { ++ t.Fatal(err) ++ } ++ ++ <-statusC ++ ++ stdioContents, err := ioutil.ReadFile(ctrdStdioFilePath) ++ if err != nil { ++ t.Fatal(err) ++ } ++ ++ switch runtimeVersion { ++ case "v1": ++ if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to stdout", id)) { ++ t.Fatal("containerd did not connect to the shim stdout pipe") ++ } ++ if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to stderr", id)) { ++ t.Fatal("containerd did not connect to the shim stderr pipe") ++ } ++ case "v2": ++ if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to log", id)) { ++ t.Fatal("containerd did not connect to the shim log pipe") ++ } ++ } ++} ++ ++func writeToFile(t *testing.T, filePath, message string) { ++ writer, err := os.OpenFile(filePath, os.O_WRONLY, 0600) ++ if err != nil { ++ t.Fatal(err) ++ } ++ if _, err := writer.WriteString(message); err != nil { ++ t.Fatal(err) ++ } ++ if err := writer.Close(); err != nil { ++ t.Fatal(err) ++ } ++} ++ ++func getLogDirPath(runtimeVersion, id string) string { ++ switch runtimeVersion { ++ case "v1": ++ return filepath.Join(defaultRoot, "io.containerd.runtime.v1.linux", testNamespace, id) ++ case "v2": ++ return filepath.Join(defaultState, "io.containerd.runtime.v2.task", testNamespace, id) ++ default: ++ panic(fmt.Errorf("Unsupported runtime version %s", runtimeVersion)) ++ } ++} ++ ++func getRuntimeVersion() string { ++ switch rt := os.Getenv("TEST_RUNTIME"); rt { ++ case "io.containerd.runc.v1": ++ return "v2" ++ default: ++ return "v1" ++ } ++} ++ + func TestContainerPTY(t *testing.T) { + t.Parallel() + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index d19b8e5..e1b3cac 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -21,6 +21,7 @@ package linux + import ( + "context" + "fmt" ++ "io" + "io/ioutil" + "os" + "path/filepath" +@@ -40,6 +41,7 @@ import ( + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/runtime" + "github.com/containerd/containerd/runtime/linux/runctypes" ++ "github.com/containerd/containerd/runtime/v1" + "github.com/containerd/containerd/runtime/v1/linux/proc" + shim "github.com/containerd/containerd/runtime/v1/shim/v1" + runc "github.com/containerd/go-runc" +@@ -341,6 +343,30 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + continue + } + ++ logDirPath := filepath.Join(r.root, ns, id) ++ ++ shimStdoutLog, err := v1.OpenShimStdoutLog(ctx, logDirPath) ++ if err != nil { ++ log.G(ctx).WithError(err).WithFields(logrus.Fields{ ++ "id": id, ++ "namespace": ns, ++ "logDirPath": logDirPath, ++ }).Error("opening shim stdout log pipe") ++ continue ++ } ++ go io.Copy(os.Stdout, shimStdoutLog) ++ ++ shimStderrLog, err := v1.OpenShimStderrLog(ctx, logDirPath) ++ if err != nil { ++ log.G(ctx).WithError(err).WithFields(logrus.Fields{ ++ "id": id, ++ "namespace": ns, ++ "logDirPath": logDirPath, ++ }).Error("opening shim stderr log pipe") ++ continue ++ } ++ go io.Copy(os.Stderr, shimStderrLog) ++ + t, err := newTask(id, ns, pid, s, r.events, r.tasks, bundle) + if err != nil { + log.G(ctx).WithError(err).Error("loading task type") +diff --git a/runtime/v1/shim.go b/runtime/v1/shim.go +new file mode 100644 +index 0000000..3942968 +--- /dev/null ++++ b/runtime/v1/shim.go +@@ -0,0 +1,38 @@ ++// +build !windows ++ ++/* ++ Copyright The containerd Authors. ++ ++ Licensed under the Apache License, Version 2.0 (the "License"); ++ you may not use this file except in compliance with the License. ++ You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++ Unless required by applicable law or agreed to in writing, software ++ distributed under the License is distributed on an "AS IS" BASIS, ++ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ See the License for the specific language governing permissions and ++ limitations under the License. ++*/ ++ ++package v1 ++ ++import ( ++ "context" ++ "io" ++ "path/filepath" ++ ++ "github.com/containerd/fifo" ++ "golang.org/x/sys/unix" ++) ++ ++// OpenShimStdoutLog opens the shim log for reading ++func OpenShimStdoutLog(ctx context.Context, logDirPath string) (io.ReadWriteCloser, error) { ++ return fifo.OpenFifo(ctx, filepath.Join(logDirPath, "shim.stdout.log"), unix.O_RDWR|unix.O_CREAT|unix.O_NONBLOCK, 0700) ++} ++ ++// OpenShimStderrLog opens the shim log ++func OpenShimStderrLog(ctx context.Context, logDirPath string) (io.ReadWriteCloser, error) { ++ return fifo.OpenFifo(ctx, filepath.Join(logDirPath, "shim.stderr.log"), unix.O_RDWR|unix.O_CREAT|unix.O_NONBLOCK, 0700) ++} +diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go +index 015d88c..ef74030 100644 +--- a/runtime/v1/shim/client/client.go ++++ b/runtime/v1/shim/client/client.go +@@ -37,6 +37,7 @@ import ( + + "github.com/containerd/containerd/events" + "github.com/containerd/containerd/log" ++ v1 "github.com/containerd/containerd/runtime/v1" + "github.com/containerd/containerd/runtime/v1/shim" + shimapi "github.com/containerd/containerd/runtime/v1/shim/v1" + "github.com/containerd/containerd/sys" +@@ -62,7 +63,24 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa + } + defer f.Close() + +- cmd, err := newCommand(binary, daemonAddress, debug, config, f) ++ var stdoutLog io.ReadWriteCloser ++ var stderrLog io.ReadWriteCloser ++ if debug { ++ stdoutLog, err = v1.OpenShimStdoutLog(ctx, config.WorkDir) ++ if err != nil { ++ return nil, nil, errors.Wrapf(err, "failed to create stdout log") ++ } ++ ++ stderrLog, err = v1.OpenShimStderrLog(ctx, config.WorkDir) ++ if err != nil { ++ return nil, nil, errors.Wrapf(err, "failed to create stderr log") ++ } ++ ++ go io.Copy(os.Stdout, stdoutLog) ++ go io.Copy(os.Stderr, stderrLog) ++ } ++ ++ cmd, err := newCommand(binary, daemonAddress, debug, config, f, stdoutLog, stderrLog) + if err != nil { + return nil, nil, err + } +@@ -77,6 +95,12 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa + go func() { + cmd.Wait() + exitHandler() ++ if stdoutLog != nil { ++ stderrLog.Close() ++ } ++ if stdoutLog != nil { ++ stderrLog.Close() ++ } + }() + log.G(ctx).WithFields(logrus.Fields{ + "pid": cmd.Process.Pid, +@@ -104,7 +128,7 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa + } + } + +-func newCommand(binary, daemonAddress string, debug bool, config shim.Config, socket *os.File) (*exec.Cmd, error) { ++func newCommand(binary, daemonAddress string, debug bool, config shim.Config, socket *os.File, stdout, stderr io.Writer) (*exec.Cmd, error) { + selfExe, err := os.Executable() + if err != nil { + return nil, err +@@ -137,10 +161,8 @@ func newCommand(binary, daemonAddress string, debug bool, config shim.Config, so + cmd.SysProcAttr = getSysProcAttr() + cmd.ExtraFiles = append(cmd.ExtraFiles, socket) + cmd.Env = append(os.Environ(), "GOMAXPROCS=2") +- if debug { +- cmd.Stdout = os.Stdout +- cmd.Stderr = os.Stderr +- } ++ cmd.Stdout = stdout ++ cmd.Stderr = stderr + return cmd, nil + } + +-- +2.7.4.3 + diff --git a/patch/0009-runtime-fix-pipe-in-broken-may-cause-shim-l.patch b/patch/0009-runtime-fix-pipe-in-broken-may-cause-shim-l.patch new file mode 100644 index 0000000000000000000000000000000000000000..e8f2b8640747467c384b69dc8f43dc080d0341e8 --- /dev/null +++ b/patch/0009-runtime-fix-pipe-in-broken-may-cause-shim-l.patch @@ -0,0 +1,38 @@ +From 77b025a48d9dc89666ef7c03709ef1fc2a4d0b34 Mon Sep 17 00:00:00 2001 +From: lujingxiao +Date: Wed, 23 Jan 2019 15:00:12 +0800 +Subject: [PATCH 09/27] runtime: fix pipe in broken may cause shim + lock forever for runtime v2 + +reason: fix pipe in broken may cause shim lock forever for runtime v2 + +Cherry-pick from upstream b3438f7a6f + +Change-Id: I3c324050531a1e68a5c3a688a51408a121a3f9f1 +Signed-off-by: Lifubang +Signed-off-by: lujingxiao +--- + runtime/v2/runc/service_linux.go | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/runtime/v2/runc/service_linux.go b/runtime/v2/runc/service_linux.go +index 5e30cfc..19d1fec 100644 +--- a/runtime/v2/runc/service_linux.go ++++ b/runtime/v2/runc/service_linux.go +@@ -49,9 +49,10 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console + cwg.Add(1) + go func() { + cwg.Done() +- p := bufPool.Get().(*[]byte) +- defer bufPool.Put(p) +- io.CopyBuffer(epollConsole, in, *p) ++ bp := bufPool.Get().(*[]byte) ++ defer bufPool.Put(bp) ++ io.CopyBuffer(epollConsole, in, *bp) ++ epollConsole.Shutdown(p.epoller.CloseConsole) + }() + } + +-- +2.7.4.3 + diff --git a/patch/0010-runtime-fix-pipe-in-broken-may-cause-shim-l.patch b/patch/0010-runtime-fix-pipe-in-broken-may-cause-shim-l.patch new file mode 100644 index 0000000000000000000000000000000000000000..e1a283c7026cc4c27a77ec038f33504dbeb2efe5 --- /dev/null +++ b/patch/0010-runtime-fix-pipe-in-broken-may-cause-shim-l.patch @@ -0,0 +1,52 @@ +From d0e57aafce7c98b3c9b3004c862d5a15180df86c Mon Sep 17 00:00:00 2001 +From: lujingxiao +Date: Wed, 23 Jan 2019 15:03:08 +0800 +Subject: [PATCH 10/27] runtime: fix pipe in broken may cause shim + lock forever for runtime v1 + +reason: fix pipe in broken may cause shim lock forever for runtime v1 + +Cherry-pick from upstream e76a8879eb + +Change-Id: Ie603b36f92c4a6cc41777a9cd1e6a19b8584eaf1 +Signed-off-by: Lifubang +Signed-off-by: lujingxiao +--- + runtime/v1/shim/service_linux.go | 8 +++++--- + runtime/v2/runc/service_linux.go | 1 + + 2 files changed, 6 insertions(+), 3 deletions(-) + +diff --git a/runtime/v1/shim/service_linux.go b/runtime/v1/shim/service_linux.go +index 18ae650..307e20d 100644 +--- a/runtime/v1/shim/service_linux.go ++++ b/runtime/v1/shim/service_linux.go +@@ -49,9 +49,11 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console + cwg.Add(1) + go func() { + cwg.Done() +- p := bufPool.Get().(*[]byte) +- defer bufPool.Put(p) +- io.CopyBuffer(epollConsole, in, *p) ++ bp := bufPool.Get().(*[]byte) ++ defer bufPool.Put(bp) ++ io.CopyBuffer(epollConsole, in, *bp) ++ // we need to shutdown epollConsole when pipe broken ++ epollConsole.Shutdown(p.epoller.CloseConsole) + }() + } + +diff --git a/runtime/v2/runc/service_linux.go b/runtime/v2/runc/service_linux.go +index 19d1fec..1161673 100644 +--- a/runtime/v2/runc/service_linux.go ++++ b/runtime/v2/runc/service_linux.go +@@ -52,6 +52,7 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console + bp := bufPool.Get().(*[]byte) + defer bufPool.Put(bp) + io.CopyBuffer(epollConsole, in, *bp) ++ // we need to shutdown epollConsole when pipe broken + epollConsole.Shutdown(p.epoller.CloseConsole) + }() + } +-- +2.7.4.3 + diff --git a/patch/0011-runtime-Add-timeout-and-cancel-to-shim-fifo.patch b/patch/0011-runtime-Add-timeout-and-cancel-to-shim-fifo.patch new file mode 100644 index 0000000000000000000000000000000000000000..79b4204e401d1fe61b44b01b806ff4b452c34189 --- /dev/null +++ b/patch/0011-runtime-Add-timeout-and-cancel-to-shim-fifo.patch @@ -0,0 +1,95 @@ +From 8eb1ab31006f3079d1bf95b4ab089e049a4f45f2 Mon Sep 17 00:00:00 2001 +From: lujingxiao +Date: Wed, 23 Jan 2019 15:04:03 +0800 +Subject: [PATCH 11/27] runtime: Add timeout and cancel to shim fifo + open + +reason: Add timeout and cancel to shim fifo open +There is still a special case where the client side fails to open or +load causes things to be slow and the shim can lock up when this +happens. This adds a timeout to the context for this case to abort fifo +creation. + +Cherry-pick from upstream 18f57e20b0 + +Signed-off-by: Michael Crosby +(cherry picked from commit a2a4241979f615eb0a1084c7638c21f830f48ac5) +Signed-off-by: Andrew Hsu +Signed-off-by: lujingxiao + +Change-Id: Ic7f285b149f97f4d6526b3f2c28b6ac6790332b0 +--- + runtime/v1/linux/proc/exec.go | 5 +++++ + runtime/v1/linux/proc/init.go | 5 +++++ + 2 files changed, 10 insertions(+) + +diff --git a/runtime/v1/linux/proc/exec.go b/runtime/v1/linux/proc/exec.go +index 96c425d..715a977 100644 +--- a/runtime/v1/linux/proc/exec.go ++++ b/runtime/v1/linux/proc/exec.go +@@ -172,22 +172,27 @@ func (e *execProcess) start(ctx context.Context) (err error) { + e.stdin = sc + } + var copyWaitGroup sync.WaitGroup ++ ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + if socket != nil { + console, err := socket.ReceiveMaster() + if err != nil { ++ cancel() + return errors.Wrap(err, "failed to retrieve console master") + } + if e.console, err = e.parent.Platform.CopyConsole(ctx, console, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg, ©WaitGroup); err != nil { ++ cancel() + return errors.Wrap(err, "failed to start console copy") + } + } else if !e.stdio.IsNull() { + if err := copyPipes(ctx, e.io, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg, ©WaitGroup); err != nil { ++ cancel() + return errors.Wrap(err, "failed to start io pipe copy") + } + } + copyWaitGroup.Wait() + pid, err := runc.ReadPidFile(opts.PidFile) + if err != nil { ++ cancel() + return errors.Wrap(err, "failed to retrieve OCI runtime exec pid") + } + e.pid = pid +diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go +index 5bf5f83..5b23671 100644 +--- a/runtime/v1/linux/proc/init.go ++++ b/runtime/v1/linux/proc/init.go +@@ -168,18 +168,22 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error { + p.closers = append(p.closers, sc) + } + var copyWaitGroup sync.WaitGroup ++ ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + if socket != nil { + console, err := socket.ReceiveMaster() + if err != nil { ++ cancel() + return errors.Wrap(err, "failed to retrieve console master") + } + console, err = p.Platform.CopyConsole(ctx, console, r.Stdin, r.Stdout, r.Stderr, &p.wg, ©WaitGroup) + if err != nil { ++ cancel() + return errors.Wrap(err, "failed to start console copy") + } + p.console = console + } else if !hasNoIO(r) { + if err := copyPipes(ctx, p.io, r.Stdin, r.Stdout, r.Stderr, &p.wg, ©WaitGroup); err != nil { ++ cancel() + return errors.Wrap(err, "failed to start io pipe copy") + } + } +@@ -187,6 +191,7 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error { + copyWaitGroup.Wait() + pid, err := runc.ReadPidFile(pidFile) + if err != nil { ++ cancel() + return errors.Wrap(err, "failed to retrieve OCI runtime container pid") + } + p.pid = pid +-- +2.7.4.3 + diff --git a/patch/0012-bump-bump-containerd-to-1.2.0.2.patch b/patch/0012-bump-bump-containerd-to-1.2.0.2.patch new file mode 100644 index 0000000000000000000000000000000000000000..2f8ac244bc1eea30d2fb3d114d0a9ee8986b9d3f --- /dev/null +++ b/patch/0012-bump-bump-containerd-to-1.2.0.2.patch @@ -0,0 +1,36 @@ +From ea92cca7c1d4dfbd6a563588a6ea9b56a764fc39 Mon Sep 17 00:00:00 2001 +From: lujingxiao +Date: Wed, 23 Jan 2019 15:31:56 +0800 +Subject: [PATCH 12/27] bump: bump containerd to 1.2.0.2 + +reason: bump containerd to 1.2.0.2 after cherry-picked patches from +upstream: +- runtime: Add timeout and cancel to shim fifo open +- runtime: fix pipe in broken may cause shim lock forever for runtime v1 +- runtime: fix pipe in broken may cause shim lock forever for runtime v2 +- runtime: Use named pipes for shim logs +- shim: Increase reaper buffer size and non-blocking send +- shim: optimize shim lock in runtime v1 + +Change-Id: Ibd7574e2ab18a2f783c694931101e1459bc779ad +Signed-off-by: lujingxiao +--- + hack/containerd.spec | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/hack/containerd.spec b/hack/containerd.spec +index f53c37b..c7d358d 100644 +--- a/hack/containerd.spec ++++ b/hack/containerd.spec +@@ -3,7 +3,7 @@ + Version: 1.2.0 + + Name: containerd +-Release: 1%{?dist} ++Release: 2%{?dist} + Summary: An industry-standard container runtime + License: ASL 2.0 + URL: https://containerd.io +-- +2.7.4.3 + diff --git a/patch/0013-log-support-log-init-pid-to-start-event-log.patch b/patch/0013-log-support-log-init-pid-to-start-event-log.patch new file mode 100644 index 0000000000000000000000000000000000000000..da79fdd12cda053aedf9cc007ae51a4a9ada62f1 --- /dev/null +++ b/patch/0013-log-support-log-init-pid-to-start-event-log.patch @@ -0,0 +1,50 @@ +From d4d3f8a239f4b4afd009d954453e585704ddb112 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Thu, 24 Jan 2019 11:55:10 +0800 +Subject: [PATCH 13/27] log: support log init pid to start event log + +reason: DFX support start event with init pid + +Change-Id: I8ae9c7a9652f694680979965829682416aed4055 +Signed-off-by: jingrui +--- + hack/containerd.spec | 2 +- + runtime/v1/linux/task.go | 2 ++ + 2 files changed, 3 insertions(+), 1 deletion(-) + +diff --git a/hack/containerd.spec b/hack/containerd.spec +index c7d358d..462d35e 100644 +--- a/hack/containerd.spec ++++ b/hack/containerd.spec +@@ -3,7 +3,7 @@ + Version: 1.2.0 + + Name: containerd +-Release: 2%{?dist} ++Release: 3%{?dist} + Summary: An industry-standard container runtime + License: ASL 2.0 + URL: https://containerd.io +diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go +index 38da35c..1c650c4 100644 +--- a/runtime/v1/linux/task.go ++++ b/runtime/v1/linux/task.go +@@ -36,6 +36,7 @@ import ( + "github.com/containerd/typeurl" + "github.com/gogo/protobuf/types" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + // Task on a linux based system +@@ -131,6 +132,7 @@ func (t *Task) Start(ctx context.Context) error { + t.cg = cg + t.mu.Unlock() + } ++ logrus.Infof("publish event %s for container %s with pid %d", runtime.TaskStartEventTopic, t.id, t.pid) + t.events.Publish(ctx, runtime.TaskStartEventTopic, &eventstypes.TaskStart{ + ContainerID: t.id, + Pid: uint32(t.pid), +-- +2.7.4.3 + diff --git a/patch/0014-event-resend-exit-event-when-detect-container.patch b/patch/0014-event-resend-exit-event-when-detect-container.patch new file mode 100644 index 0000000000000000000000000000000000000000..de4a80265327cc81e7ee6ff91d48dd8a5985a8d6 --- /dev/null +++ b/patch/0014-event-resend-exit-event-when-detect-container.patch @@ -0,0 +1,84 @@ +From 200ae6f4b733f8a869aac36a730da90e79213387 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sun, 10 Feb 2019 18:40:59 +0800 +Subject: [PATCH 14/27] event: resend exit event when detect + containerd restarted + +reason: testCE_docker_containerd_ABN.026.sh +fix docker stop no effect. + +Change-Id: I024b2f6a03d74fcbb5623c696212dcbfb624b285 +Signed-off-by: jingrui +--- + cmd/containerd-shim/main_unix.go | 38 +++++++++++++++++++++++++++++++++++++- + 1 file changed, 37 insertions(+), 1 deletion(-) + +diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go +index 6c59cd1..d1f41b0 100644 +--- a/cmd/containerd-shim/main_unix.go ++++ b/cmd/containerd-shim/main_unix.go +@@ -24,12 +24,14 @@ import ( + "flag" + "fmt" + "io" ++ "io/ioutil" + "net" + "os" + "os/exec" + "os/signal" + "runtime" + "runtime/debug" ++ "strconv" + "strings" + "sync" + "syscall" +@@ -263,7 +265,7 @@ type remoteEventsPublisher struct { + address string + } + +-func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error { ++func (l *remoteEventsPublisher) doPublish(ctx context.Context, topic string, event events.Event) error { + ns, _ := namespaces.Namespace(ctx) + encoded, err := typeurl.MarshalAny(event) + if err != nil { +@@ -288,3 +290,37 @@ func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event + } + return nil + } ++ ++func getContainerdPid() int { ++ pidFile := "/var/run/docker/containerd/containerd.pid" ++ data, err := ioutil.ReadFile(pidFile) ++ if err != nil { ++ return -1 ++ } ++ pid, err := strconv.Atoi(string(data)) ++ if err != nil { ++ return -1 ++ } ++ return pid ++} ++ ++func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error { ++ old := getContainerdPid() ++ for i := 1; i <= 10; i++ { ++ err := l.doPublish(ctx, topic, event) ++ logrus.Infof("try publish event(%d) %s %v %v", i, topic, event, err) ++ if err == nil { ++ new := getContainerdPid() ++ if old == new { ++ return nil ++ } ++ logrus.Warnf("containerd pid %d changed to %d", old, new) ++ old = new ++ } ++ if i == 10 { ++ return err ++ } ++ time.Sleep(time.Duration(i) * time.Second) ++ } ++ return nil ++} +-- +2.7.4.3 + diff --git a/patch/0015-restore-cleanup-container-pid-1.patch b/patch/0015-restore-cleanup-container-pid-1.patch new file mode 100644 index 0000000000000000000000000000000000000000..1cdfd2282e9260b77a3ceb7bbb5cc484fa574827 --- /dev/null +++ b/patch/0015-restore-cleanup-container-pid-1.patch @@ -0,0 +1,122 @@ +From fd1c8dda8cc02b9aef28f1e3e4e51ab216338e2b Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sun, 10 Feb 2019 15:40:52 +0800 +Subject: [PATCH 15/27] restore: cleanup container pid=-1 + +reason: fix testCE_docker_hook_spec_ABN.050.sh +when containerd killed during task create, see Runtime.Create(). the +defer function will not execute, so shim residual. cleanup shim for +container pid=-1 + +Change-Id: Ie9a7f6dff5f8a922cc97c5fcf44664ab60ac1a7a +Signed-off-by: jingrui +--- + runtime/v1/linux/runtime.go | 10 +++++++--- + runtime/v1/linux/task.go | 26 ++++++++++++++++++++++++-- + 2 files changed, 31 insertions(+), 5 deletions(-) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index e1b3cac..3b66304 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -316,6 +316,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + continue + } + id := path.Name() ++ log.G(ctx).Infof("load-task %s", id) + bundle := loadBundle( + id, + filepath.Join(r.state, ns, id), +@@ -372,6 +373,12 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + log.G(ctx).WithError(err).Error("loading task type") + continue + } ++ if pid == -1 { ++ _, err := t.DeleteForce(ctx) ++ log.G(ctx).Warnf("delete force %s Pid=-1 error=%v", id, err) ++ continue ++ } ++ log.G(ctx).Infof("load-task %s Pid=%d done", id, pid) + o = append(o, t) + } + return o, nil +@@ -380,9 +387,6 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + func (r *Runtime) cleanupAfterDeadShim(ctx context.Context, bundle *bundle, ns, id string, pid int) error { + ctx = namespaces.WithNamespace(ctx, ns) + if err := r.terminate(ctx, bundle, ns, id); err != nil { +- if r.config.ShimDebug { +- return errors.Wrap(err, "failed to terminate task, leaving bundle for debugging") +- } + log.G(ctx).WithError(err).Warn("failed to terminate task") + } + +diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go +index 1c650c4..6995156 100644 +--- a/runtime/v1/linux/task.go ++++ b/runtime/v1/linux/task.go +@@ -21,6 +21,7 @@ package linux + import ( + "context" + "sync" ++ "time" + + "github.com/containerd/cgroups" + eventstypes "github.com/containerd/containerd/api/events" +@@ -37,6 +38,7 @@ import ( + "github.com/gogo/protobuf/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ++ "golang.org/x/sys/unix" + ) + + // Task on a linux based system +@@ -86,10 +88,13 @@ func (t *Task) Namespace() string { + } + + // Delete the task and return the exit status +-func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { ++func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) { + rsp, err := t.shim.Delete(ctx, empty) + if err != nil { +- return nil, errdefs.FromGRPC(err) ++ log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force) ++ if !force { ++ return nil, errdefs.FromGRPC(err) ++ } + } + t.tasks.Delete(ctx, t.id) + if err := t.shim.KillShim(ctx); err != nil { +@@ -98,6 +103,14 @@ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { + if err := t.bundle.Delete(); err != nil { + log.G(ctx).WithError(err).Error("failed to delete bundle") + } ++ ++ if rsp == nil { ++ rsp = &shim.DeleteResponse{} ++ rsp.ExitStatus = 128 + uint32(unix.SIGKILL) ++ rsp.ExitedAt = time.Now().UTC() ++ rsp.Pid = 0 ++ } ++ + t.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{ + ContainerID: t.id, + ExitStatus: rsp.ExitStatus, +@@ -111,6 +124,15 @@ func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { + }, nil + } + ++// Delete the task and return the exit status ++func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { ++ return t.delete(ctx, false) ++} ++ ++func (t *Task) DeleteForce(ctx context.Context) (*runtime.Exit, error) { ++ return t.delete(ctx, true) ++} ++ + // Start the task + func (t *Task) Start(ctx context.Context) error { + t.mu.Lock() +-- +2.7.4.3 + diff --git a/patch/0016-create-runc-delete-force-before-create.patch b/patch/0016-create-runc-delete-force-before-create.patch new file mode 100644 index 0000000000000000000000000000000000000000..3aa5ece7aa59307d8e5f6134d6c4185739d328ca --- /dev/null +++ b/patch/0016-create-runc-delete-force-before-create.patch @@ -0,0 +1,31 @@ +From e7827a737c42861afd6b41e2e7dc953c249278fc Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Mon, 11 Feb 2019 17:40:31 +0800 +Subject: [PATCH 16/27] create: runc delete force before create + +reason: testCE_docker_hook_spec_ABN.051.sh +kill -9 shim will generate residual runc files, cleanup runc files using +runc delete before create. + +Change-Id: I3efa3c4d0989ba8d688bcb6f35ba543b6ab91b2d +Signed-off-by: jingrui +--- + vendor/github.com/containerd/go-runc/runc.go | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index 96262af..e688881 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -138,6 +138,8 @@ func (o *CreateOpts) args() (out []string, err error) { + + // Create creates a new container and returns its pid if it was created successfully + func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error { ++ r.Delete(context, id, &DeleteOpts{Force: true}) ++ + args := []string{"create", "--bundle", bundle} + if opts != nil { + oargs, err := opts.args() +-- +2.7.4.3 + diff --git a/patch/0017-exit-using-init.exit-indicate-container-is-ex.patch b/patch/0017-exit-using-init.exit-indicate-container-is-ex.patch new file mode 100644 index 0000000000000000000000000000000000000000..92c6ef49f470d0629faa4d7c99578de91cb93b3a --- /dev/null +++ b/patch/0017-exit-using-init.exit-indicate-container-is-ex.patch @@ -0,0 +1,65 @@ +From f83e391aef03283b30431a960b66f720cf0d9dd3 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Mon, 11 Feb 2019 20:12:15 +0800 +Subject: [PATCH 17/27] exit: using init.exit indicate container is + exiting + +reason: testCE_docker_hook_spec_ABN.053.sh +kill dockerd during docker stop in post-stophook, containerd will load +task and treat as ok when shim response client. add init.exit to forbid +load exiting task. + +Change-Id: I8f03cd51088d43d4fb457b32981f3eebd8558f84 +Signed-off-by: jingrui +--- + runtime/v1/linux/proc/init.go | 1 + + runtime/v1/linux/runtime.go | 5 +++++ + runtime/v1/shim/service.go | 4 +++- + 3 files changed, 9 insertions(+), 1 deletion(-) + +diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go +index 5b23671..caa31c3 100644 +--- a/runtime/v1/linux/proc/init.go ++++ b/runtime/v1/linux/proc/init.go +@@ -43,6 +43,7 @@ import ( + + // InitPidFile name of the file that contains the init pid + const InitPidFile = "init.pid" ++const InitExit = "init.exit" + + // Init represents an initial process for a container + type Init struct { +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 3b66304..123d675 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -378,6 +378,11 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + log.G(ctx).Warnf("delete force %s Pid=-1 error=%v", id, err) + continue + } ++ if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil { ++ _, err := t.DeleteForce(ctx) ++ log.G(ctx).Warnf("delete force %s Pid=%d(exiting) error=%v", id, pid, err) ++ continue ++ } + log.G(ctx).Infof("load-task %s Pid=%d done", id, pid) + o = append(o, t) + } +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index 679982a..8c7984f 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -504,7 +504,9 @@ func (s *Service) checkProcesses(e runc.Exit) { + + for _, p := range s.processes { + if p.Pid() == e.Pid { +- ++ if ip, ok := p.(*proc.Init); ok { ++ ioutil.WriteFile(filepath.Join(ip.Bundle, proc.InitExit), []byte(fmt.Sprintf("%d", e.Pid)), 0600) ++ } + if shouldKillAll { + if ip, ok := p.(*proc.Init); ok { + // Ensure all children are killed +-- +2.7.4.3 + diff --git a/patch/0018-containerd-shim-Dump-log-to-file-when-docker-.patch b/patch/0018-containerd-shim-Dump-log-to-file-when-docker-.patch new file mode 100644 index 0000000000000000000000000000000000000000..e9115139e85ec556819f8b355e50a27732cc750e --- /dev/null +++ b/patch/0018-containerd-shim-Dump-log-to-file-when-docker-.patch @@ -0,0 +1,42 @@ +From 7f483b7d5a6bd88ea35f5dcf1a5fea5d165044fe Mon Sep 17 00:00:00 2001 +From: lixiang172 +Date: Tue, 12 Feb 2019 15:22:06 +0800 +Subject: [PATCH 18/27] containerd-shim: Dump log to file when docker + received signal + +reason: Dump stack log to file when docker received "kill -SIGUSR1 +PID" signal +The name of log files is "shim-stack-[time].log". +The log file can be found at: +/run/docker/containerd/daemon/io.containerd.runtime.v1.linux/moby/container-id/shim-stack-[time].log + +Change-Id: I6d7e03c9a0fd36e9a76f1dd45cfd5312985d03f8 +Signed-off-by: lixiang172 +--- + cmd/containerd-shim/main_unix.go | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go +index d1f41b0..38b3eb4 100644 +--- a/cmd/containerd-shim/main_unix.go ++++ b/cmd/containerd-shim/main_unix.go +@@ -246,6 +246,8 @@ func handleSignals(logger *logrus.Entry, signals chan os.Signal, server *ttrpc.S + } + } + ++const stacksLogNameTemplate = "shim-stacks-%s.log" ++ + func dumpStacks(logger *logrus.Entry) { + var ( + buf []byte +@@ -258,6 +260,7 @@ func dumpStacks(logger *logrus.Entry) { + bufferLen *= 2 + } + buf = buf[:stackSize] ++ ioutil.WriteFile(fmt.Sprintf(stacksLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1)), buf, 0600) + logger.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) + } + +-- +2.7.4.3 + diff --git a/patch/0019-restore-check-shim-alive-when-containerd-is-r.patch b/patch/0019-restore-check-shim-alive-when-containerd-is-r.patch new file mode 100644 index 0000000000000000000000000000000000000000..4557c56fb78dc49cd1d0c89e00e71d4797597ebc --- /dev/null +++ b/patch/0019-restore-check-shim-alive-when-containerd-is-r.patch @@ -0,0 +1,47 @@ +From 112c2ef89b1085e95959285ce5328af5d74ba8db Mon Sep 17 00:00:00 2001 +From: xueshaojia +Date: Thu, 14 Feb 2019 10:48:14 +0800 +Subject: [PATCH 19/27] restore: check shim alive when containerd is + restarted + +reason: fix docker_containerd-shim:testCE_docker_containerd_shim_ABN.021.sh + When containerd is restarted, it will load all tasks.In some cases, the + containerd-shim is killed and the sock file will exist for a while. + Containerd should check the containerd-shim is available using the sock file. + If the containerd-shim server not responses, do r.cleanupAfterDeadShim + +Change-Id: I448c8caefa8c1252bd5cdcff79deb8eff1005903 +Signed-off-by: xueshaojia +--- + runtime/v1/linux/runtime.go | 15 +++++++++++++++ + 1 file changed, 15 insertions(+) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 123d675..477cda0 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -343,6 +343,21 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + } + continue + } ++ ctxContact, cancel := context.WithTimeout(ctx, 5*time.Second) ++ defer cancel() ++ alive, err := s.IsAlive(ctxContact) ++ if !alive { ++ log.G(ctx).WithError(err).WithFields(logrus.Fields{ ++ "id": id, ++ "namespace": ns, ++ }).Error("contacting to shim") ++ err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid) ++ if err != nil { ++ log.G(ctx).WithError(err).WithField("bundle", bundle.path). ++ Error("cleaning up after dead shim") ++ } ++ continue ++ } + + logDirPath := filepath.Join(r.root, ns, id) + +-- +2.7.4.3 + diff --git a/patch/0020-events-resend-pending-exit-events-on-restore.patch b/patch/0020-events-resend-pending-exit-events-on-restore.patch new file mode 100644 index 0000000000000000000000000000000000000000..8d2b63b669fbaf0b7204e9dbc8dfc135917fcdd8 --- /dev/null +++ b/patch/0020-events-resend-pending-exit-events-on-restore.patch @@ -0,0 +1,357 @@ +From 27762e8d75c00c8898c725873c17a23105ba5b7c Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Tue, 12 Feb 2019 17:03:11 +0800 +Subject: [PATCH 20/27] events: resend pending exit events on restore + +reason: fix exit event may lost. +testCE_docker_containerd_ABN.026.sh + +Change-Id: I5bcdf06ad4ee7b8a0ca782e610186f52e3d79bbd +Signed-off-by: jingrui +--- + events/events.go | 13 +++++ + events/exchange/exchange.go | 12 +++++ + events/exit.go | 79 +++++++++++++++++++++++++++++ + runtime/v1/linux/runtime.go | 56 +++++++++++++++++--- + runtime/v1/linux/task.go | 10 ++-- + runtime/v1/shim/service.go | 2 + + vendor/github.com/docker/go-events/queue.go | 8 +++ + 7 files changed, 167 insertions(+), 13 deletions(-) + create mode 100644 events/exit.go + +diff --git a/events/events.go b/events/events.go +index b7eb86f..aa07236 100644 +--- a/events/events.go ++++ b/events/events.go +@@ -22,6 +22,7 @@ import ( + + "github.com/containerd/typeurl" + "github.com/gogo/protobuf/types" ++ apievents "github.com/containerd/containerd/api/events" + ) + + // Envelope provides the packaging for an event. +@@ -32,6 +33,18 @@ type Envelope struct { + Event *types.Any + } + ++func (e *Envelope) ExitFile() string { ++ decoded, err := typeurl.UnmarshalAny(e.Event) ++ if err != nil { ++ return "" ++ } ++ ++ if e, ok := decoded.(*apievents.TaskExit); ok { ++ return ExitFile(e.ContainerID, e.Pid, e.ExitStatus) ++ } ++ ++ return "" ++} + // Field returns the value for the given fieldpath as a string, if defined. + // If the value is not defined, the second value will be false. + func (e *Envelope) Field(fieldpath []string) (string, bool) { +diff --git a/events/exchange/exchange.go b/events/exchange/exchange.go +index 95d21b7..540f180 100644 +--- a/events/exchange/exchange.go ++++ b/events/exchange/exchange.go +@@ -49,6 +49,11 @@ func NewExchange() *Exchange { + var _ events.Publisher = &Exchange{} + var _ events.Forwarder = &Exchange{} + var _ events.Subscriber = &Exchange{} ++var mobySubcribed = false ++ ++func MobySubscribed() bool { ++ return mobySubcribed ++} + + // Forward accepts an envelope to be direcly distributed on the exchange. + // +@@ -161,6 +166,13 @@ func (e *Exchange) Subscribe(ctx context.Context, fs ...string) (ch <-chan *even + } + + e.broadcaster.Add(dst) ++ logrus.Infof("subscribe ctx=%v fs=%v", ctx, fs) ++ for _, s := range fs { ++ if !MobySubscribed() && s == "namespace==moby,topic~=|^/tasks/|" { ++ queue.Namespace = "moby" ++ mobySubcribed = true ++ } ++ } + + go func() { + defer closeAll() +diff --git a/events/exit.go b/events/exit.go +new file mode 100644 +index 0000000..e1ce089 +--- /dev/null ++++ b/events/exit.go +@@ -0,0 +1,79 @@ ++package events ++ ++import ( ++ "fmt" ++ "io/ioutil" ++ "os" ++ "path/filepath" ++ "strconv" ++ "strings" ++ "github.com/sirupsen/logrus" ++) ++ ++const ExitDir = "/var/run/docker/containerd/exit" ++const ExitStatusDefault = 137 ++ ++func ExitFile(cid string, pid uint32, status uint32) string { ++ return fmt.Sprintf("%s.%d.%d", cid, pid, status) ++} ++ ++func ExitInfo(ef string) (string, uint32, uint32) { ++ s := strings.Split(ef, ".") ++ if len(s) != 3 { ++ return "", 0, 0 ++ } ++ ++ cid := s[0] ++ pid, err := strconv.ParseUint(s[1], 10, 32) ++ if err != nil { ++ return "", 0, 0 ++ } ++ status, err := strconv.ParseUint(s[2], 10, 32) ++ if err != nil { ++ return "", 0, 0 ++ } ++ ++ return cid, uint32(pid), uint32(status) ++} ++ ++func ExitAddFile(ns string, ef string, reason string) { ++ os.MkdirAll(filepath.Join(ExitDir, ns), 0700) ++ err := ioutil.WriteFile(filepath.Join(ExitDir, ns, ef), []byte{}, 0600) ++ logrus.Infof("exit-add %s/%s [reason: %s] error=%v", ns, ef, reason, err) ++} ++ ++func ExitDelFile(ns string, ef string) { ++ err := os.RemoveAll(filepath.Join(ExitDir, ns, ef)) ++ logrus.Infof("exit-del %s/%s error=%v", ns, ef, err) ++} ++ ++func ExitGetFile(ns string, cid string, pid uint32, status uint32) string { ++ ef := ExitFile(cid, pid, status) ++ if _, err := os.Stat(filepath.Join(ExitDir, ns, ef)); err == nil { ++ return ef ++ } ++ return "" ++} ++ ++func ExitGetFiles(ns string) []string { ++ files, err := ioutil.ReadDir(filepath.Join(ExitDir, ns)) ++ if err != nil { ++ return []string{} ++ } ++ ++ names := []string{} ++ for _, f := range files { ++ names = append(names, f.Name()) ++ } ++ ++ return names ++} ++ ++func ExitPending(ns string, cid string, pid uint32) bool { ++ for _, ef := range ExitGetFiles(ns) { ++ if strings.Contains(ef, fmt.Sprintf("%s.%d", cid, pid)) { ++ return true ++ } ++ } ++ return false ++} +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 477cda0..add4d52 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -31,6 +31,7 @@ import ( + "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/errdefs" ++ "github.com/containerd/containerd/events" + "github.com/containerd/containerd/events/exchange" + "github.com/containerd/containerd/identifiers" + "github.com/containerd/containerd/log" +@@ -129,6 +130,7 @@ func New(ic *plugin.InitContext) (interface{}, error) { + return nil, err + } + } ++ go r.resendExitEvents(ic.Context, "moby") + return r, nil + } + +@@ -175,7 +177,8 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts + } + defer func() { + if err != nil { +- bundle.Delete() ++ errd := bundle.Delete() ++ log.G(ctx).WithError(err).Errorf("revert: delete bundle error=%v", errd) + } + }() + +@@ -218,9 +221,8 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts + } + defer func() { + if err != nil { +- if kerr := s.KillShim(ctx); kerr != nil { +- log.G(ctx).WithError(err).Error("failed to kill shim") +- } ++ kerr := s.KillShim(ctx) ++ log.G(ctx).WithError(err).Errorf("revert: kill shim error=%v", kerr) + } + }() + +@@ -305,6 +307,41 @@ func (r *Runtime) Get(ctx context.Context, id string) (runtime.Task, error) { + return r.tasks.Get(ctx, id) + } + ++func (r *Runtime) resendExitEvents(ctx context.Context, ns string) { ++ for { ++ time.Sleep(time.Second) ++ efs := events.ExitGetFiles(ns) ++ if len(efs) == 0 { ++ break ++ } ++ ++ if !exchange.MobySubscribed() { ++ logrus.Infof("waiting moby event stream ...") ++ continue ++ } ++ time.Sleep(time.Second) ++ ++ for _, ef := range efs { ++ cid, pid, status := events.ExitInfo(ef) ++ if cid == "" { ++ continue ++ } ++ ++ e := &eventstypes.TaskExit{ ++ ContainerID: cid, ++ ID: cid, ++ ExitStatus: status, ++ ExitedAt: time.Now().UTC(), ++ Pid: uint32(pid), ++ } ++ ++ ctx := namespaces.WithNamespace(context.Background(), ns) ++ err := r.events.Publish(ctx, runtime.TaskExitEventTopic, e) ++ logrus.Infof("resend exit event %v error=%v", e, err) ++ } ++ } ++} ++ + func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + dir, err := ioutil.ReadDir(filepath.Join(r.state, ns)) + if err != nil { +@@ -388,13 +425,16 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + log.G(ctx).WithError(err).Error("loading task type") + continue + } +- if pid == -1 { +- _, err := t.DeleteForce(ctx) +- log.G(ctx).Warnf("delete force %s Pid=-1 error=%v", id, err) ++ if pid <= 0 { ++ _, err := t.DeleteForce(ctx, 0) ++ log.G(ctx).Warnf("delete force %s Pid=%d error=%v", id, pid, err) + continue + } + if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil { +- _, err := t.DeleteForce(ctx) ++ if !events.ExitPending(ns, t.id, uint32(pid)) { ++ events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task") ++ } ++ _, err := t.DeleteForce(ctx, uint32(pid)) + log.G(ctx).Warnf("delete force %s Pid=%d(exiting) error=%v", id, pid, err) + continue + } +diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go +index 6995156..b692ae7 100644 +--- a/runtime/v1/linux/task.go ++++ b/runtime/v1/linux/task.go +@@ -88,7 +88,7 @@ func (t *Task) Namespace() string { + } + + // Delete the task and return the exit status +-func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) { ++func (t *Task) delete(ctx context.Context, force bool, pid uint32) (*runtime.Exit, error) { + rsp, err := t.shim.Delete(ctx, empty) + if err != nil { + log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force) +@@ -108,7 +108,7 @@ func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) { + rsp = &shim.DeleteResponse{} + rsp.ExitStatus = 128 + uint32(unix.SIGKILL) + rsp.ExitedAt = time.Now().UTC() +- rsp.Pid = 0 ++ rsp.Pid = pid + } + + t.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{ +@@ -126,11 +126,11 @@ func (t *Task) delete(ctx context.Context, force bool) (*runtime.Exit, error) { + + // Delete the task and return the exit status + func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) { +- return t.delete(ctx, false) ++ return t.delete(ctx, false, 0) + } + +-func (t *Task) DeleteForce(ctx context.Context) (*runtime.Exit, error) { +- return t.delete(ctx, true) ++func (t *Task) DeleteForce(ctx context.Context, pid uint32) (*runtime.Exit, error) { ++ return t.delete(ctx, true, pid) + } + + // Start the task +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index 8c7984f..a2eb35b 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -505,6 +505,8 @@ func (s *Service) checkProcesses(e runc.Exit) { + for _, p := range s.processes { + if p.Pid() == e.Pid { + if ip, ok := p.(*proc.Init); ok { ++ ns := filepath.Base(filepath.Dir(ip.Bundle)) ++ events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited") + ioutil.WriteFile(filepath.Join(ip.Bundle, proc.InitExit), []byte(fmt.Sprintf("%d", e.Pid)), 0600) + } + if shouldKillAll { +diff --git a/vendor/github.com/docker/go-events/queue.go b/vendor/github.com/docker/go-events/queue.go +index 4bb770a..0608e7e 100644 +--- a/vendor/github.com/docker/go-events/queue.go ++++ b/vendor/github.com/docker/go-events/queue.go +@@ -5,12 +5,14 @@ import ( + "sync" + + "github.com/sirupsen/logrus" ++ topevents "github.com/containerd/containerd/events" + ) + + // Queue accepts all messages into a queue for asynchronous consumption + // by a sink. It is unbounded and thread safe but the sink must be reliable or + // events will be dropped. + type Queue struct { ++ Namespace string + dst Sink + events *list.List + cond *sync.Cond +@@ -83,6 +85,12 @@ func (eq *Queue) run() { + "event": event, + "sink": eq.dst, + }).WithError(err).Debug("eventqueue: dropped event") ++ } else { ++ if e, ok := event.(*topevents.Envelope); ok { ++ if ef := e.ExitFile(); ef != "" { ++ topevents.ExitDelFile(eq.Namespace, ef) ++ } ++ } + } + } + } +-- +2.7.4.3 + diff --git a/patch/0021-containerd-Update-the-version-info-of-contain.patch b/patch/0021-containerd-Update-the-version-info-of-contain.patch new file mode 100644 index 0000000000000000000000000000000000000000..c1db6686804f5cf20e0d2697e207d4ee55fc111b --- /dev/null +++ b/patch/0021-containerd-Update-the-version-info-of-contain.patch @@ -0,0 +1,59 @@ +From 818ef5fe43d3b9b4c53301800d545ce4c775afff Mon Sep 17 00:00:00 2001 +From: lixiang172 +Date: Tue, 12 Feb 2019 11:37:37 +0800 +Subject: [PATCH 21/27] containerd: Update the version info of + containerd + +reason: Update the version info after type "containerd -v" +The version info now is defined by "containerd.spec" rather than +"version.go" + +Change-Id: I04c6b78737e09f93a3e84a100c88be19294a5c4f +Signed-off-by: lixiang172 +--- + Makefile | 8 ++++---- + version/version.go | 2 +- + 2 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index 35021fd..e38dfb3 100644 +--- a/Makefile ++++ b/Makefile +@@ -20,8 +20,8 @@ ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + DESTDIR=/usr/local + + # Used to populate variables in version package. +-VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always) +-REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) ++VERSION=$(shell echo version:)$(shell grep '^Version' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/').$(shell grep '^Release:' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') ++REVISION=$(shell echo commit:)$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) + + ifneq "$(strip $(shell command -v go 2>/dev/null))" "" + GOOS ?= $(shell go env GOOS) +@@ -77,8 +77,8 @@ MANPAGES=ctr.1 containerd.1 containerd-config.1 containerd-config.toml.5 + # Build tags seccomp and apparmor are needed by CRI plugin. + BUILDTAGS ?= seccomp apparmor + GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",) +-GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) $(EXTRA_LDFLAGS)' +-SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) -extldflags "-static"' ++GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' ++SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' + + #Replaces ":" (*nix), ";" (windows) with newline for easy parsing + GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n") +diff --git a/version/version.go b/version/version.go +index b2874bf..04b7097 100644 +--- a/version/version.go ++++ b/version/version.go +@@ -18,7 +18,7 @@ package version + + var ( + // Package is filled at linking time +- Package = "github.com/containerd/containerd" ++ Package = "" + + // Version holds the complete version number. Filled in at linking time. + Version = "1.2.0+unknown" +-- +2.7.4.3 + diff --git a/patch/0022-containerd-bump-version-1.2.0.4.patch b/patch/0022-containerd-bump-version-1.2.0.4.patch new file mode 100644 index 0000000000000000000000000000000000000000..515b1f83022eccba52eceb7fdff8af90be538e80 --- /dev/null +++ b/patch/0022-containerd-bump-version-1.2.0.4.patch @@ -0,0 +1,29 @@ +From bea413085725db89439817284b63bb4061e62753 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Wed, 13 Feb 2019 22:03:08 +0800 +Subject: [PATCH 22/27] containerd: bump version 1.2.0.4 + +reason: bump version + +Change-Id: Iee2348e931a723929ccfe63b3539c812514acc90 +Signed-off-by: jingrui +--- + hack/containerd.spec | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/hack/containerd.spec b/hack/containerd.spec +index 462d35e..f8d9084 100644 +--- a/hack/containerd.spec ++++ b/hack/containerd.spec +@@ -3,7 +3,7 @@ + Version: 1.2.0 + + Name: containerd +-Release: 3%{?dist} ++Release: 4%{?dist} + Summary: An industry-standard container runtime + License: ASL 2.0 + URL: https://containerd.io +-- +2.7.4.3 + diff --git a/patch/0023-containerd-set-create-and-exec-timeout.patch b/patch/0023-containerd-set-create-and-exec-timeout.patch new file mode 100644 index 0000000000000000000000000000000000000000..f99410947aa79d0b548ed1b7c99381d810d320ce --- /dev/null +++ b/patch/0023-containerd-set-create-and-exec-timeout.patch @@ -0,0 +1,218 @@ +From 006bc6d0a9e0c233d0d14de53de0b18799c67081 Mon Sep 17 00:00:00 2001 +From: xiadanni +Date: Fri, 15 Feb 2019 06:00:52 +0800 +Subject: [PATCH 23/27] containerd: set create and exec timeout + +reason:set create and exec timeout to avild block when command failed + +Change-Id: I6bc55f4ccc953bdc1d926ab940f0900811d68760 +Signed-off-by: xiadanni +--- + hack/containerd.spec | 2 +- + runtime/v1/shim/reaper.go | 50 +++++++++++++++++++++++++ + runtime/v2/shim/reaper_unix.go | 4 ++ + vendor/github.com/containerd/go-runc/monitor.go | 6 +++ + vendor/github.com/containerd/go-runc/runc.go | 31 +++++++++++++-- + 5 files changed, 88 insertions(+), 5 deletions(-) + +diff --git a/hack/containerd.spec b/hack/containerd.spec +index f8d9084..f39c57a 100644 +--- a/hack/containerd.spec ++++ b/hack/containerd.spec +@@ -3,7 +3,7 @@ + Version: 1.2.0 + + Name: containerd +-Release: 4%{?dist} ++Release: 5%{?dist} + Summary: An industry-standard container runtime + License: ASL 2.0 + URL: https://containerd.io +diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go +index 10d5c30..a2b90fe 100644 +--- a/runtime/v1/shim/reaper.go ++++ b/runtime/v1/shim/reaper.go +@@ -19,8 +19,13 @@ + package shim + + import ( ++ "io/ioutil" + "os/exec" ++ "path/filepath" ++ "strconv" ++ "strings" + "sync" ++ "syscall" + "time" + + "github.com/containerd/containerd/sys" +@@ -100,6 +105,34 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) { + return -1, ErrNoSuchProcess + } + ++// WaitTimeout is used to skip the blocked command and kill the left process. ++func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) { ++ sch := make(chan int) ++ ech := make(chan error) ++ go func() { ++ for e := range ec { ++ if e.Pid == c.Process.Pid { ++ // make sure we flush all IO ++ c.Wait() ++ m.Unsubscribe(ec) ++ sch <- e.Status ++ return ++ } ++ } ++ }() ++ select { ++ case <-time.After(time.Duration(sec) * time.Second): ++ if SameProcess(c, c.Process.Pid) { ++ syscall.Kill(c.Process.Pid, syscall.SIGKILL) ++ } ++ return 0, errors.Errorf("timeout %ds for cmd(pid= %d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) ++ case status := <-sch: ++ return status, nil ++ case err := <-ech: ++ return -1, err ++ } ++} ++ + // Subscribe to process exit changes + func (m *Monitor) Subscribe() chan runc.Exit { + c := make(chan runc.Exit, bufferSize) +@@ -116,3 +149,20 @@ func (m *Monitor) Unsubscribe(c chan runc.Exit) { + close(c) + m.Unlock() + } ++ ++func SameProcess(cmd *exec.Cmd, pid int) bool { ++ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline")) ++ if err != nil { ++ return false ++ } ++ for i := range bytes { ++ if bytes[i] == 0 { ++ bytes[i] = 32 ++ } ++ } ++ cmdline := string(bytes) ++ if strings.EqualFold(cmdline, strings.Join(cmd.Args, " ")+" ") { ++ return true ++ } ++ return false ++} +diff --git a/runtime/v2/shim/reaper_unix.go b/runtime/v2/shim/reaper_unix.go +index 10d5c30..8bd7dd1 100644 +--- a/runtime/v2/shim/reaper_unix.go ++++ b/runtime/v2/shim/reaper_unix.go +@@ -100,6 +100,10 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) { + return -1, ErrNoSuchProcess + } + ++func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, error) { ++ return m.Wait(c, ec) ++} ++ + // Subscribe to process exit changes + func (m *Monitor) Subscribe() chan runc.Exit { + c := make(chan runc.Exit, bufferSize) +diff --git a/vendor/github.com/containerd/go-runc/monitor.go b/vendor/github.com/containerd/go-runc/monitor.go +index ff06a3f..2c184d2 100644 +--- a/vendor/github.com/containerd/go-runc/monitor.go ++++ b/vendor/github.com/containerd/go-runc/monitor.go +@@ -40,6 +40,7 @@ type Exit struct { + type ProcessMonitor interface { + Start(*exec.Cmd) (chan Exit, error) + Wait(*exec.Cmd, chan Exit) (int, error) ++ WaitTimeout(*exec.Cmd, chan Exit, int64) (int, error) + } + + type defaultMonitor struct { +@@ -74,3 +75,8 @@ func (m *defaultMonitor) Wait(c *exec.Cmd, ec chan Exit) (int, error) { + e := <-ec + return e.Status, nil + } ++ ++func (m *defaultMonitor) WaitTimeout(c *exec.Cmd, ec chan Exit, sec int64) (int, error) { ++ e := <-ec ++ return e.Status, nil ++} +\ No newline at end of file +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index e688881..fc64e8a 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -52,6 +52,8 @@ const ( + Text Format = "text" + // DefaultCommand is the default command for Runc + DefaultCommand = "runc" ++ execTimeout = 30 ++ createTimeout = 120 + ) + + // Runc is the client to the runc cli +@@ -155,7 +157,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp + cmd.ExtraFiles = opts.ExtraFiles + + if cmd.Stdout == nil && cmd.Stderr == nil { +- data, err := cmdOutput(cmd, true) ++ data, err := cmdOutputTimeout(cmd, true, createTimeout) + if err != nil { + return fmt.Errorf("%s: %s", err, data) + } +@@ -172,7 +174,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp + } + } + } +- status, err := Monitor.Wait(cmd, ec) ++ status, err := Monitor.WaitTimeout(cmd, ec, createTimeout) + if err == nil && status != 0 { + err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) + } +@@ -234,7 +236,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts + opts.Set(cmd) + } + if cmd.Stdout == nil && cmd.Stderr == nil { +- data, err := cmdOutput(cmd, true) ++ data, err := cmdOutputTimeout(cmd, true, execTimeout) + if err != nil { + return fmt.Errorf("%s: %s", err, data) + } +@@ -251,7 +253,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts + } + } + } +- status, err := Monitor.Wait(cmd, ec) ++ status, err := Monitor.WaitTimeout(cmd, ec, execTimeout) + if err == nil && status != 0 { + err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) + } +@@ -707,3 +709,24 @@ func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) { + + return b.Bytes(), err + } ++ ++func cmdOutputTimeout(cmd *exec.Cmd, combined bool, timeout int64) ([]byte, error) { ++ b := getBuf() ++ defer putBuf(b) ++ ++ cmd.Stdout = b ++ if combined { ++ cmd.Stderr = b ++ } ++ ec, err := Monitor.Start(cmd) ++ if err != nil { ++ return nil, err ++ } ++ ++ status, err := Monitor.WaitTimeout(cmd, ec, timeout) ++ if err == nil && status != 0 { ++ err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) ++ } ++ ++ return b.Bytes(), err ++} +-- +2.7.4.3 + diff --git a/patch/0024-create-cleanup-runc-dirty-files-on-start.patch b/patch/0024-create-cleanup-runc-dirty-files-on-start.patch new file mode 100644 index 0000000000000000000000000000000000000000..c951ceae72231074324945ccc2a1ed6a1bec0ec6 --- /dev/null +++ b/patch/0024-create-cleanup-runc-dirty-files-on-start.patch @@ -0,0 +1,54 @@ +From f96039fcd94c5bc75dcec297668418811d60e785 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Tue, 19 Feb 2019 11:53:41 +0800 +Subject: [PATCH 24/27] create: cleanup runc dirty files on start + +reason: add check before cleanup runtime dirty files. + +Change-Id: I6f218fd8d19ed65d8b13ae1ea744b80574279f83 +Signed-off-by: jingrui +--- + hack/containerd.spec | 2 +- + vendor/github.com/containerd/go-runc/runc.go | 6 +++++- + 2 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/hack/containerd.spec b/hack/containerd.spec +index f39c57a..869012a 100644 +--- a/hack/containerd.spec ++++ b/hack/containerd.spec +@@ -3,7 +3,7 @@ + Version: 1.2.0 + + Name: containerd +-Release: 5%{?dist} ++Release: 6%{?dist} + Summary: An industry-standard container runtime + License: ASL 2.0 + URL: https://containerd.io +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index fc64e8a..e66ea5b 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -30,6 +30,7 @@ import ( + "strings" + "syscall" + "time" ++ "github.com/sirupsen/logrus" + + specs "github.com/opencontainers/runtime-spec/specs-go" + ) +@@ -140,7 +141,10 @@ func (o *CreateOpts) args() (out []string, err error) { + + // Create creates a new container and returns its pid if it was created successfully + func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error { +- r.Delete(context, id, &DeleteOpts{Force: true}) ++ if _, err := os.Stat(filepath.Join(r.Root, id)); err == nil { ++ logrus.Warnf("cleanup residue runtime with bundle %s root=%s", bundle, r.Root) ++ r.Delete(context, id, &DeleteOpts{Force: true}) ++ } + + args := []string{"create", "--bundle", bundle} + if opts != nil { +-- +2.7.4.3 + diff --git a/patch/0025-restore-skip-load-task-in-creating.patch b/patch/0025-restore-skip-load-task-in-creating.patch new file mode 100644 index 0000000000000000000000000000000000000000..143f1c4e466d080c45ebe5f1951cb458f443df5c --- /dev/null +++ b/patch/0025-restore-skip-load-task-in-creating.patch @@ -0,0 +1,74 @@ +From 869ceecb455640da5e90f7827f75275665e93e95 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sat, 23 Feb 2019 15:51:24 +0800 +Subject: [PATCH 25/27] restore: skip load task in creating + +load task in creating will stuck containerd restore process. + +Change-Id: I2f8b77a88d78597ef2be5122708fc8ab16fad956 +Signed-off-by: jingrui +--- + runtime/v1/linux/runtime.go | 5 ++--- + runtime/v1/shim/service.go | 6 ++++++ + 2 files changed, 8 insertions(+), 3 deletions(-) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index add4d52..5647f94 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -353,7 +353,6 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + continue + } + id := path.Name() +- log.G(ctx).Infof("load-task %s", id) + bundle := loadBundle( + id, + filepath.Join(r.state, ns, id), +@@ -361,6 +360,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + ) + ctx = namespaces.WithNamespace(ctx, ns) + pid, _ := runc.ReadPidFile(filepath.Join(bundle.path, proc.InitPidFile)) ++ log.G(ctx).Infof("load-task %s/%s/%s Pid=%d", r.state, ns, id, pid) + s, err := bundle.NewShimClient(ctx, ns, ShimConnect(r.config, func() { + err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid) + if err != nil { +@@ -426,8 +426,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + continue + } + if pid <= 0 { +- _, err := t.DeleteForce(ctx, 0) +- log.G(ctx).Warnf("delete force %s Pid=%d error=%v", id, pid, err) ++ log.G(ctx).Warnf("skip load task in creating %s", id) + continue + } + if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil { +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index a2eb35b..d7fdcaf 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -26,6 +26,7 @@ import ( + "os" + "path/filepath" + "sync" ++ "time" + + "github.com/containerd/console" + eventstypes "github.com/containerd/containerd/api/events" +@@ -140,9 +141,14 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ * + rootfs := filepath.Join(r.Bundle, "rootfs") + defer func() { + if err != nil { ++ logrus.Errorf("create init %s failed error=%v", r.ID, err) + if err2 := mount.UnmountAll(rootfs, 0); err2 != nil { + log.G(ctx).WithError(err2).Warn("Failed to cleanup rootfs mount") + } ++ go func() { ++ time.Sleep(10*time.Second) ++ os.Exit(0) ++ }() + } + }() + for _, rm := range mounts { +-- +2.7.4.3 + diff --git a/patch/0026-exit-optimize-init.exit-record.patch b/patch/0026-exit-optimize-init.exit-record.patch new file mode 100644 index 0000000000000000000000000000000000000000..80eaae2a207b3a7fda5ca31efbb65880c265137c --- /dev/null +++ b/patch/0026-exit-optimize-init.exit-record.patch @@ -0,0 +1,96 @@ +From c26316153098e72a9b30668befc36fcfcba3b76f Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sat, 23 Feb 2019 15:55:21 +0800 +Subject: [PATCH 26/27] exit: optimize init.exit record + +Change-Id: If1319f7d87defed16d1113337957f36b7320e9b9 +Signed-off-by: jingrui +--- + events/exit.go | 21 +++++++++++++++++++++ + runtime/v1/linux/proc/init.go | 1 - + runtime/v1/linux/runtime.go | 2 +- + runtime/v1/shim/service.go | 2 +- + 4 files changed, 23 insertions(+), 3 deletions(-) + +diff --git a/events/exit.go b/events/exit.go +index e1ce089..772dc24 100644 +--- a/events/exit.go ++++ b/events/exit.go +@@ -7,11 +7,13 @@ import ( + "path/filepath" + "strconv" + "strings" ++ + "github.com/sirupsen/logrus" + ) + + const ExitDir = "/var/run/docker/containerd/exit" + const ExitStatusDefault = 137 ++const InitExit = "init.exit" + + func ExitFile(cid string, pid uint32, status uint32) string { + return fmt.Sprintf("%s.%d.%d", cid, pid, status) +@@ -77,3 +79,22 @@ func ExitPending(ns string, cid string, pid uint32) bool { + } + return false + } ++ ++func InitExitWrite(bundle string, pid int) { ++ if _, err := os.Stat(bundle); err != nil { ++ logrus.Infof("skip write init.exit %s error=%v", bundle, err) ++ return ++ } ++ err := ioutil.WriteFile(filepath.Join(bundle, InitExit), []byte(fmt.Sprintf("%d", pid)), 0600) ++ if err != nil { ++ logrus.Infof("failed write init.exit error=%s", bundle, err) ++ } ++} ++ ++func InitExitExist(bundle string) bool { ++ if _, err := os.Stat(filepath.Join(bundle, InitExit)); err == nil { ++ return true ++ } ++ return false ++} ++ +diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go +index caa31c3..5b23671 100644 +--- a/runtime/v1/linux/proc/init.go ++++ b/runtime/v1/linux/proc/init.go +@@ -43,7 +43,6 @@ import ( + + // InitPidFile name of the file that contains the init pid + const InitPidFile = "init.pid" +-const InitExit = "init.exit" + + // Init represents an initial process for a container + type Init struct { +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 5647f94..e92904e 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -429,7 +429,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + log.G(ctx).Warnf("skip load task in creating %s", id) + continue + } +- if _, err := os.Stat(filepath.Join(bundle.path, proc.InitExit)); err == nil { ++ if events.InitExitExist(bundle.path) { + if !events.ExitPending(ns, t.id, uint32(pid)) { + events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task") + } +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index d7fdcaf..f421fde 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -513,7 +513,7 @@ func (s *Service) checkProcesses(e runc.Exit) { + if ip, ok := p.(*proc.Init); ok { + ns := filepath.Base(filepath.Dir(ip.Bundle)) + events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited") +- ioutil.WriteFile(filepath.Join(ip.Bundle, proc.InitExit), []byte(fmt.Sprintf("%d", e.Pid)), 0600) ++ events.InitExitWrite(ip.Bundle, e.Pid) + } + if shouldKillAll { + if ip, ok := p.(*proc.Init); ok { +-- +2.7.4.3 + diff --git a/patch/0027-log-make-tester-happy.patch b/patch/0027-log-make-tester-happy.patch new file mode 100644 index 0000000000000000000000000000000000000000..f3e9295492002463ffeb8f7cf2f1ef53cdb5fea8 --- /dev/null +++ b/patch/0027-log-make-tester-happy.patch @@ -0,0 +1,48 @@ +From a275b359b2e85d8f353eab12d538a94609171918 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sat, 23 Feb 2019 18:32:00 +0800 +Subject: [PATCH 27/27] log: make tester happy + +reason: make tester happy ++ check_docker_error /tmp/tmp_11955/log2 b3357887148bc59212d30dba46d3eea9490cfe94594fa00aa7706c7addb92d91 ++ grep docker /tmp/tmp_11955/log2 ++ grep error ++ grep b3357887148bc59212d30dba46d3eea9490cfe94594fa00aa7706c7addb92d91 ++ grep -w 'container did not start before the specified timeout' + +Change-Id: Iddd40bd42212bf09f52c17f28119a6b5364f4de7 +Signed-off-by: jingrui +--- + hack/containerd.spec | 2 +- + runtime/v1/shim/reaper.go | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/hack/containerd.spec b/hack/containerd.spec +index 869012a..05f68c7 100644 +--- a/hack/containerd.spec ++++ b/hack/containerd.spec +@@ -3,7 +3,7 @@ + Version: 1.2.0 + + Name: containerd +-Release: 6%{?dist} ++Release: 7%{?dist} + Summary: An industry-standard container runtime + License: ASL 2.0 + URL: https://containerd.io +diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go +index a2b90fe..529a533 100644 +--- a/runtime/v1/shim/reaper.go ++++ b/runtime/v1/shim/reaper.go +@@ -125,7 +125,7 @@ func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, e + if SameProcess(c, c.Process.Pid) { + syscall.Kill(c.Process.Pid, syscall.SIGKILL) + } +- return 0, errors.Errorf("timeout %ds for cmd(pid= %d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) ++ return 0, errors.Errorf("container did not start before the specified timeout %ds for cmd(pid=%d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) + case status := <-sch: + return status, nil + case err := <-ech: +-- +2.7.4.3 + diff --git a/patch/0028-restore-delete-task-in-containerd-restoring.patch b/patch/0028-restore-delete-task-in-containerd-restoring.patch new file mode 100644 index 0000000000000000000000000000000000000000..e5b36a121f93de21e42253e8e67607930d399861 --- /dev/null +++ b/patch/0028-restore-delete-task-in-containerd-restoring.patch @@ -0,0 +1,33 @@ +From 1130a0bc101c3f59c99eb850b24d0799c216d677 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Fri, 22 Mar 2019 21:22:08 +0800 +Subject: [PATCH] restore: delete task in containerd restoring + +reason: delete task quickly when containerd is restoring to avoid container restart fail. + +Change-Id: Ide5e8c9bbd873addc6c35b9604e4cda03ca78b5e +Signed-off-by: xiadanni1 +--- + runtime/v1/linux/runtime.go | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index e92904e..2a45aaa 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -426,7 +426,11 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + continue + } + if pid <= 0 { +- log.G(ctx).Warnf("skip load task in creating %s", id) ++ go func() { ++ log.G(ctx).Infof("del task in creating %s", id) ++ t.DeleteForce(ctx, uint32(pid)) ++ log.G(ctx).Infof("del task in creating %s done", id) ++ }() + continue + } + if events.InitExitExist(bundle.path) { +-- +1.8.3.1 + diff --git a/patch/0029-restore-delete-task-asynchronously.patch b/patch/0029-restore-delete-task-asynchronously.patch new file mode 100644 index 0000000000000000000000000000000000000000..2d3dc90dda0b62347a1d65545baf89cb80b68512 --- /dev/null +++ b/patch/0029-restore-delete-task-asynchronously.patch @@ -0,0 +1,35 @@ +From de14f9d00033a9596823e0ea953437f5f244cb74 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Sat, 23 Mar 2019 07:18:57 +0800 +Subject: [PATCH] restore: delete task asynchronously + +reason: set delete task to asynchronous to avoid containerd be killed when delete is blocking. + testCE_docker_hook_spec_ABN.059.sh + +Change-Id: I5fae8e60987b9617a835ea07710ca3c842efab14 +Signed-off-by: xiadanni1 +--- + runtime/v1/linux/runtime.go | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 2a45aaa..cca72fe 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -437,8 +437,11 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + if !events.ExitPending(ns, t.id, uint32(pid)) { + events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task") + } +- _, err := t.DeleteForce(ctx, uint32(pid)) +- log.G(ctx).Warnf("delete force %s Pid=%d(exiting) error=%v", id, pid, err) ++ go func(){ ++ log.G(ctx).Infof("delete force %s start, Pid=%d(exiting)", id, pid) ++ _, err := t.DeleteForce(ctx, uint32(pid)) ++ log.G(ctx).Infof("delete force %s done, Pid=%d(exiting) error=%v", id, pid, err) ++ }() + continue + } + log.G(ctx).Infof("load-task %s Pid=%d done", id, pid) +-- +1.8.3.1 + diff --git a/patch/0030-event-fix-events-lost-when-loadTask-failed.patch b/patch/0030-event-fix-events-lost-when-loadTask-failed.patch new file mode 100644 index 0000000000000000000000000000000000000000..959460d2b6d00a533ed6566133081dd15aad8307 --- /dev/null +++ b/patch/0030-event-fix-events-lost-when-loadTask-failed.patch @@ -0,0 +1,45 @@ +From 375689497320d105aa2ed026710e20d9b0bd2a72 Mon Sep 17 00:00:00 2001 +From: jiangpengfei9 +Date: Mon, 1 Apr 2019 13:08:50 -0400 +Subject: [PATCH] event: fix events lost when loadTask failed + +reason: If containerd-shim and containerd process is killed, container will exit, +however containerd exit event which generates when containerd restart to reload +tasks can not publish to dockerd, because at the time of loading tasks the connection +between dockerd and containerd isn't established. + +So we add this unpublish exit event to file and resend this event after grpc connection +is established. + +Signed-off-by: jiangpengfei9 +--- + runtime/v1/linux/runtime.go | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index cca72fe..af823b2 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -373,6 +373,9 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + "id": id, + "namespace": ns, + }).Error("connecting to shim") ++ if !events.ExitPending(ns, id, uint32(pid)) { ++ events.ExitAddFile(ns, events.ExitFile(id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task") ++ } + err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid) + if err != nil { + log.G(ctx).WithError(err).WithField("bundle", bundle.path). +@@ -388,6 +391,9 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + "id": id, + "namespace": ns, + }).Error("contacting to shim") ++ if !events.ExitPending(ns, id, uint32(pid)) { ++ events.ExitAddFile(ns, events.ExitFile(id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task") ++ } + err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid) + if err != nil { + log.G(ctx).WithError(err).WithField("bundle", bundle.path). +-- +1.8.3.1 + diff --git a/patch/0031-containerd-enable-relro-flags.patch b/patch/0031-containerd-enable-relro-flags.patch new file mode 100644 index 0000000000000000000000000000000000000000..2ee4f4b6294a90ee58cfc7acf89966571c9a3b7d --- /dev/null +++ b/patch/0031-containerd-enable-relro-flags.patch @@ -0,0 +1,28 @@ +From 2db6e4cda2e042fab327493c0fa095723d7c0352 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Mon, 15 Apr 2019 10:58:07 +0800 +Subject: [PATCH] containerd: enable relro flags + +Change-Id: I5f32e7bf794842a14e1644f7aa3115a65b1bc698 +Signed-off-by: jingrui +--- + Makefile | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/Makefile b/Makefile +index e38dfb38..921b2d50 100644 +--- a/Makefile ++++ b/Makefile +@@ -77,7 +77,8 @@ MANPAGES=ctr.1 containerd.1 containerd-config.1 containerd-config.toml.5 + # Build tags seccomp and apparmor are needed by CRI plugin. + BUILDTAGS ?= seccomp apparmor + GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",) +-GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' ++GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' \ ++ -ldflags=-extldflags=-zrelro -ldflags=-extldflags=-znow + SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' + + #Replaces ":" (*nix), ";" (windows) with newline for easy parsing +-- +2.17.1 + diff --git a/patch/0032-containerd-enable-bep-ldflags.patch b/patch/0032-containerd-enable-bep-ldflags.patch new file mode 100644 index 0000000000000000000000000000000000000000..e557af1c8b8c34dbb88daf8612cbce5b6b7064a3 --- /dev/null +++ b/patch/0032-containerd-enable-bep-ldflags.patch @@ -0,0 +1,45 @@ +From da6ea77f9f47c740fe85e7e4d34889e131135b81 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Mon, 15 Apr 2019 23:44:55 +0800 +Subject: [PATCH] containerd: enable bep ldflags + +Change-Id: I820b100aa1420fc399878a905de14fb6a25ca1a4 +Signed-off-by: jingrui +--- + Makefile | 12 ++++++++---- + 1 file changed, 8 insertions(+), 4 deletions(-) + +diff --git a/Makefile b/Makefile +index 921b2d50..612330b4 100644 +--- a/Makefile ++++ b/Makefile +@@ -77,9 +77,12 @@ MANPAGES=ctr.1 containerd.1 containerd-config.1 containerd-config.toml.5 + # Build tags seccomp and apparmor are needed by CRI plugin. + BUILDTAGS ?= seccomp apparmor + GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",) +-GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' \ +- -ldflags=-extldflags=-zrelro -ldflags=-extldflags=-znow +-SHIM_GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' ++ ++BEP_DIR=/tmp/containerd-build-bep ++BEP_FLAGS=-tmpdir=/tmp/containerd-build-bep ++ ++GO_LDFLAGS=-ldflags '-s -w -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' ++SHIM_GO_LDFLAGS=-ldflags '-s -w $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' + + #Replaces ":" (*nix), ";" (windows) with newline for easy parsing + GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n") +@@ -166,8 +169,9 @@ FORCE: + + # Build a binary from a cmd. + bin/%: cmd/% FORCE ++ mkdir -p $(BEP_DIR) + @echo "$(WHALE) $@${BINARY_SUFFIX}" +- @go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@${BINARY_SUFFIX} ${GO_LDFLAGS} ${GO_TAGS} ./$< ++ go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@${BINARY_SUFFIX} ${GO_LDFLAGS} ${GO_TAGS} ./$< + + bin/containerd-shim: cmd/containerd-shim FORCE # set !cgo and omit pie for a static shim build: https://github.com/golang/go/issues/17789#issuecomment-258542220 + @echo "$(WHALE) bin/containerd-shim" +-- +2.17.1 + diff --git a/patch/0033-containerd-fix-opened-file-not-close.patch b/patch/0033-containerd-fix-opened-file-not-close.patch new file mode 100644 index 0000000000000000000000000000000000000000..8ba547cc5f0dd79f1f74d49902b9c09bdfb58162 --- /dev/null +++ b/patch/0033-containerd-fix-opened-file-not-close.patch @@ -0,0 +1,28 @@ +From b5806942e2938d4800298df276f1a095b859bacb Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Fri, 19 Apr 2019 22:05:18 +0800 +Subject: [PATCH] containerd: fix opened file not close + +reason: fix opened file not close + +Change-Id: I69f53255eabd3dd2e87a61ba963fa8027870e014 +Signed-off-by: xiadanni1 +--- + runtime/v1/linux/proc/utils.go | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/runtime/v1/linux/proc/utils.go b/runtime/v1/linux/proc/utils.go +index 3d0334c..ab9f5fa 100644 +--- a/runtime/v1/linux/proc/utils.go ++++ b/runtime/v1/linux/proc/utils.go +@@ -41,6 +41,7 @@ func getLastRuntimeError(r *runc.Runc) (string, error) { + if err != nil { + return "", err + } ++ defer f.Close() + + var ( + errMsg string +-- +1.8.3.1 + diff --git a/patch/0034-containerd-add-buildid-in-Makefile.patch b/patch/0034-containerd-add-buildid-in-Makefile.patch new file mode 100644 index 0000000000000000000000000000000000000000..8d92ee0d6811d610b052353893718b771d14858d --- /dev/null +++ b/patch/0034-containerd-add-buildid-in-Makefile.patch @@ -0,0 +1,28 @@ +From e61f2c1664c91b5c8a8cb48641002c7c471c1d45 Mon Sep 17 00:00:00 2001 +From: zhangyu235 +Date: Tue, 23 Apr 2019 12:24:50 +0800 +Subject: [PATCH] containerd: add buildid in Makefile + +Change-Id: I1c2ff035db2a02d125139b9ff170f91e81181541 +--- + Makefile | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Makefile b/Makefile +index 612330b..a400899 100644 +--- a/Makefile ++++ b/Makefile +@@ -81,8 +81,8 @@ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",) + BEP_DIR=/tmp/containerd-build-bep + BEP_FLAGS=-tmpdir=/tmp/containerd-build-bep + +-GO_LDFLAGS=-ldflags '-s -w -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' +-SHIM_GO_LDFLAGS=-ldflags '-s -w $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' ++GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' ++SHIM_GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' + + #Replaces ":" (*nix), ";" (windows) with newline for easy parsing + GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n") +-- +2.7.4.3 + diff --git a/patch/0035-containerd-fix-the-path-of-containerd.spec-in.patch b/patch/0035-containerd-fix-the-path-of-containerd.spec-in.patch new file mode 100644 index 0000000000000000000000000000000000000000..0f7659a2bdf021bd3ac38a1f709608acc71c1ee3 --- /dev/null +++ b/patch/0035-containerd-fix-the-path-of-containerd.spec-in.patch @@ -0,0 +1,82 @@ +From 8f97c7a7353c05a8b64ef9ee522ee62fba66a608 Mon Sep 17 00:00:00 2001 +From: zhangyu235 +Date: Sun, 5 May 2019 19:50:56 +0800 +Subject: [PATCH] containerd: fix the path of containerd.spec in + Makefile + +Change-Id: I4ec87e5ddf256574513f977e53e4bdf050e0169c +Signed-off-by: zhangyu235 +--- + Makefile | 2 +- + hack/containerd.spec | 46 ---------------------------------------------- + 2 files changed, 1 insertion(+), 47 deletions(-) + delete mode 100644 hack/containerd.spec + +diff --git a/Makefile b/Makefile +index a400899..5de5cf7 100644 +--- a/Makefile ++++ b/Makefile +@@ -20,7 +20,7 @@ ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + DESTDIR=/usr/local + + # Used to populate variables in version package. +-VERSION=$(shell echo version:)$(shell grep '^Version' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/').$(shell grep '^Release:' ${ROOTDIR}/hack/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') ++VERSION=$(shell echo version:)$(shell grep '^Version' ${ROOTDIR}/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/').$(shell grep '^Release:' ${ROOTDIR}/containerd.spec | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') + REVISION=$(shell echo commit:)$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) + + ifneq "$(strip $(shell command -v go 2>/dev/null))" "" +diff --git a/hack/containerd.spec b/hack/containerd.spec +deleted file mode 100644 +index 05f68c7..0000000 +--- a/hack/containerd.spec ++++ /dev/null +@@ -1,46 +0,0 @@ +-%global goipath github.com/containerd/containerd +-%global debug_package %{nil} +-Version: 1.2.0 +- +-Name: containerd +-Release: 7%{?dist} +-Summary: An industry-standard container runtime +-License: ASL 2.0 +-URL: https://containerd.io +-Source0: containerd-1.2.0.tar.gz +- +-BuildRequires: golang glibc-static make +-BuildRequires: btrfs-progs-devel +- +- +-%description +-containerd is an industry-standard container runtime with an emphasis on +-simplicity, robustness and portability. It is available as a daemon for Linux +-and Windows, which can manage the complete container lifecycle of its host +-system: image transfer and storage, container execution and supervision, +-low-level storage and network attachments, etc. +- +- +-%prep +-%setup -c -n containerd +- +-%build +-GO_BUILD_PATH=$PWD/_build +-install -m 0755 -vd $(dirname $GO_BUILD_PATH/src/%{goipath}) +-ln -fs $PWD $GO_BUILD_PATH/src/%{goipath} +-cd $GO_BUILD_PATH/src/%{goipath} +-export GOPATH=$GO_BUILD_PATH:%{gopath} +-export BUILDTAGS="no_btrfs no_cri" +-make +- +-%install +-install -d $RPM_BUILD_ROOT/%{_bindir} +-install -p -m 755 bin/containerd $RPM_BUILD_ROOT/%{_bindir}/containerd +-install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim +- +-%files +-%{_bindir}/containerd +-%{_bindir}/containerd-shim +- +- +-%changelog +-- +2.7.4.3 + diff --git a/patch/0036-containerd-support-container-start-timeout-se.patch b/patch/0036-containerd-support-container-start-timeout-se.patch new file mode 100644 index 0000000000000000000000000000000000000000..40c531000df970cdbeb258890a9ce075c18e085e --- /dev/null +++ b/patch/0036-containerd-support-container-start-timeout-se.patch @@ -0,0 +1,69 @@ +From 1980e34108cf2fab407c4e0b45cb07fc06e15642 Mon Sep 17 00:00:00 2001 +From: lixiang172 +Date: Thu, 9 May 2019 21:36:56 +0800 +Subject: [PATCH] containerd: support container start timeout setting + +Change-Id: I8c958a1c16ed6c7a86e4c6299ad1ef81c7476120 +Signed-off-by: lixiang172 +--- + vendor/github.com/containerd/go-runc/runc.go | 24 ++++++++++++++++++++++-- + 1 file changed, 22 insertions(+), 2 deletions(-) + +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index e66ea5b..6323bf2 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -30,9 +30,9 @@ import ( + "strings" + "syscall" + "time" +- "github.com/sirupsen/logrus" + + specs "github.com/opencontainers/runtime-spec/specs-go" ++ "github.com/sirupsen/logrus" + ) + + // Format is the type of log formatting options avaliable +@@ -54,7 +54,10 @@ const ( + // DefaultCommand is the default command for Runc + DefaultCommand = "runc" + execTimeout = 30 +- createTimeout = 120 ++) ++ ++var ( ++ createTimeout int64 = 120 + ) + + // Runc is the client to the runc cli +@@ -72,6 +75,15 @@ type Runc struct { + Rootless *bool // nil stands for "auto" + } + ++func init() { ++ runtimeTimeout, err := convertTime(os.Getenv("DOCKER_RUNTIME_START_TIMEOUT")) ++ if err != nil { ++ logrus.Warnf("init error, wrong runtimeTimeout format: %v", err) ++ } else { ++ createTimeout = runtimeTimeout ++ } ++} ++ + // List returns all containers created inside the provided runc root directory + func (r *Runc) List(context context.Context) ([]*Container, error) { + data, err := cmdOutput(r.command(context, "list", "--format=json"), false) +@@ -734,3 +746,11 @@ func cmdOutputTimeout(cmd *exec.Cmd, combined bool, timeout int64) ([]byte, erro + + return b.Bytes(), err + } ++ ++func convertTime(timeout string) (int64, error) { ++ timeDura, err := time.ParseDuration(timeout) ++ if err != nil { ++ return 0, err ++ } ++ return timeDura.Nanoseconds() / 1e9, nil ++} +-- +1.8.3.1 + diff --git a/patch/0037-containerd-Fix-fd-leak-of-shim-log.patch b/patch/0037-containerd-Fix-fd-leak-of-shim-log.patch new file mode 100644 index 0000000000000000000000000000000000000000..a62144650d9d1ba92c47a7cede77601cb6c6cd7e --- /dev/null +++ b/patch/0037-containerd-Fix-fd-leak-of-shim-log.patch @@ -0,0 +1,55 @@ +From 26c6307f1cab31105583ef22c2da8fe44a8d45e4 Mon Sep 17 00:00:00 2001 +From: zhangyu235 +Date: Fri, 17 May 2019 16:52:06 +0800 +Subject: [PATCH] containerd: Fix fd leak of shim log + +reason:Open shim v2 log with the flag `O_RDWR` will cause the `Read()` block +forever even if the pipe has been closed on the shim side. Then the +`io.Copy()` would never return and lead to a fd leak. +Fix typo when closing shim v1 log which causes the `stdouLog` leak. +Update `numPipes` function in test case to get the opened FIFO +correctly. + +Cherry-pick from upstream cf6e00854 +Reference from https://github.com/containerd/containerd/pull/3266 + +Change-Id: If83a4ca9b9ec0079ac0f0015d1f6768581571030 +Signed-off-by: Li Yuxuan +Signed-off-by: zhangyu235 +--- + container_linux_test.go | 2 +- + runtime/v1/shim/client/client.go | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/container_linux_test.go b/container_linux_test.go +index fa764d7..fdf6349 100644 +--- a/container_linux_test.go ++++ b/container_linux_test.go +@@ -329,7 +329,7 @@ func TestShimDoesNotLeakPipes(t *testing.T) { + } + + func numPipes(pid int) (int, error) { +- cmd := exec.Command("sh", "-c", fmt.Sprintf("lsof -p %d | grep pipe", pid)) ++ cmd := exec.Command("sh", "-c", fmt.Sprintf("lsof -p %d | grep FIFO", pid)) + + var stdout bytes.Buffer + cmd.Stdout = &stdout +diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go +index ef74030..a819be6 100644 +--- a/runtime/v1/shim/client/client.go ++++ b/runtime/v1/shim/client/client.go +@@ -96,9 +96,9 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa + cmd.Wait() + exitHandler() + if stdoutLog != nil { +- stderrLog.Close() ++ stdoutLog.Close() + } +- if stdoutLog != nil { ++ if stderrLog != nil { + stderrLog.Close() + } + }() +-- +2.7.4.3 + diff --git a/patch/0037-containerd-fix-shim-std-logs-not-close-after-.patch b/patch/0037-containerd-fix-shim-std-logs-not-close-after-.patch new file mode 100644 index 0000000000000000000000000000000000000000..77aa6ca986033b87f5b74c4a7f29e0bbec0bbe0c --- /dev/null +++ b/patch/0037-containerd-fix-shim-std-logs-not-close-after-.patch @@ -0,0 +1,59 @@ +From d13733a390a987006bd5febb7d28a2d1c7873af2 Mon Sep 17 00:00:00 2001 +From: zhangyu235 +Date: Thu, 30 May 2019 09:27:00 +0800 +Subject: [PATCH] containerd: fix shim std logs not close after shim + exit + +reason:fix shim std logs not close after shim exit + +Change-Id: I980fb17b1d46de099b81529ea46681cf9f4bf09c +Signed-off-by: zhangyu235 +--- + runtime/v1/linux/runtime.go | 16 +++++++++++++++- + 1 file changed, 15 insertions(+), 1 deletion(-) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index af823b2..66914fe 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -361,7 +361,9 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + ctx = namespaces.WithNamespace(ctx, ns) + pid, _ := runc.ReadPidFile(filepath.Join(bundle.path, proc.InitPidFile)) + log.G(ctx).Infof("load-task %s/%s/%s Pid=%d", r.state, ns, id, pid) ++ shimExit := make(chan struct{}) + s, err := bundle.NewShimClient(ctx, ns, ShimConnect(r.config, func() { ++ close(shimExit) + err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid) + if err != nil { + log.G(ctx).WithError(err).WithField("bundle", bundle.path). +@@ -426,6 +428,18 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + } + go io.Copy(os.Stderr, shimStderrLog) + ++ go func() { ++ select { ++ case <-shimExit: ++ if shimStdoutLog != nil { ++ shimStdoutLog.Close() ++ } ++ if shimStderrLog != nil { ++ shimStderrLog.Close() ++ } ++ } ++ }() ++ + t, err := newTask(id, ns, pid, s, r.events, r.tasks, bundle) + if err != nil { + log.G(ctx).WithError(err).Error("loading task type") +@@ -443,7 +457,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + if !events.ExitPending(ns, t.id, uint32(pid)) { + events.ExitAddFile(ns, events.ExitFile(t.id, uint32(pid), uint32(events.ExitStatusDefault)), "cleanup dirty task") + } +- go func(){ ++ go func() { + log.G(ctx).Infof("delete force %s start, Pid=%d(exiting)", id, pid) + _, err := t.DeleteForce(ctx, uint32(pid)) + log.G(ctx).Infof("delete force %s done, Pid=%d(exiting) error=%v", id, pid, err) +-- +2.7.4.3 + diff --git a/patch/0038-containerd-support-kill-D-state-container.patch b/patch/0038-containerd-support-kill-D-state-container.patch new file mode 100644 index 0000000000000000000000000000000000000000..bc94d1f487565867f281bbdc844c2ffcd190c7a0 --- /dev/null +++ b/patch/0038-containerd-support-kill-D-state-container.patch @@ -0,0 +1,65 @@ +From 8ab02b5aecb0fa04ad747988d838e1c4de535222 Mon Sep 17 00:00:00 2001 +From: Jing Rui +Date: Tue, 18 Jun 2019 00:12:41 +0800 +Subject: [PATCH] containerd: support kill D state container + +Change-Id: I057553f2b8d3f57b71e5ea79930067bb7071e524 +Signed-off-by: Jing Rui +--- + runtime/v1/shim/service.go | 21 +++++++++++++++++++++ + 1 file changed, 21 insertions(+) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index f421fdef..8adaf35b 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -26,6 +26,7 @@ import ( + "os" + "path/filepath" + "sync" ++ "syscall" + "time" + + "github.com/containerd/console" +@@ -366,11 +367,30 @@ func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, e + + // Kill a process with the provided signal + func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Empty, error) { ++ delayKill := func(p rproc.Process) { ++ if s.id != p.ID() || r.Signal != uint32(syscall.SIGKILL) { ++ return ++ } ++ ++ for i := 1; i < 5; i++ { ++ time.Sleep(10 * time.Second) ++ err := p.Kill(ctx, r.Signal, r.All) ++ logrus.Infof("delay kill %s retry %d error=%v", s.id, i, err) ++ } ++ ++ logrus.Infof("force exit shim %s ...", s.id) ++ p.SetExited(137) ++ err := p.Delete(ctx) ++ logrus.Infof("force exit shim %s error=%v", s.id, err) ++ os.Exit(0) ++ } ++ + if r.ID == "" { + p, err := s.getInitProcess() + if err != nil { + return nil, err + } ++ go delayKill(p) + if err := p.Kill(ctx, r.Signal, r.All); err != nil { + return nil, errdefs.ToGRPC(err) + } +@@ -381,6 +401,7 @@ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Emp + if err != nil { + return nil, err + } ++ go delayKill(p) + if err := p.Kill(ctx, r.Signal, r.All); err != nil { + return nil, errdefs.ToGRPC(err) + } +-- +2.17.1 + diff --git a/patch/0039-containerd-modify-containerd-shim-to-ad.patch b/patch/0039-containerd-modify-containerd-shim-to-ad.patch new file mode 100644 index 0000000000000000000000000000000000000000..e475ee3ec0132917910c88370365b110da85b992 --- /dev/null +++ b/patch/0039-containerd-modify-containerd-shim-to-ad.patch @@ -0,0 +1,51 @@ +From 5eef82c3c41eabb532cd7520acf7e8587b76d8b5 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 10 Jul 2019 15:07:46 -0400 +Subject: [PATCH] containerd: modify containerd-shim to adapt runv + runtime + +reason: containerd-shim pass a too long runtime root path to runv runtime, which cause hyperstartgrpc.sock +file absolute path exceed the max length of Unix Socket(max length is 108). + +Signed-off-by: jiangpengfei +--- + runtime/v1/linux/proc/init.go | 11 ++++++++++- + 1 file changed, 10 insertions(+), 1 deletion(-) + +diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go +index 5b23671..d464147 100644 +--- a/runtime/v1/linux/proc/init.go ++++ b/runtime/v1/linux/proc/init.go +@@ -44,6 +44,9 @@ import ( + // InitPidFile name of the file that contains the init pid + const InitPidFile = "init.pid" + ++// Default runv runtime root dir ++const defaultRunvRoot = "/run/runv" ++ + // Init represents an initial process for a container + type Init struct { + wg sync.WaitGroup +@@ -83,12 +86,18 @@ func NewRunc(root, path, namespace, runtime, criu string, systemd bool) *runc.Ru + if root == "" { + root = RuncRoot + } ++ ++ rootPath := filepath.Join(root, namespace) ++ if strings.Contains(runtime, "runv") { ++ rootPath = defaultRunvRoot ++ } ++ + return &runc.Runc{ + Command: runtime, + Log: filepath.Join(path, "log.json"), + LogFormat: runc.JSON, + PdeathSignal: syscall.SIGKILL, +- Root: filepath.Join(root, namespace), ++ Root: rootPath, + Criu: criu, + SystemdCgroup: systemd, + } +-- +1.8.3.1 + diff --git a/patch/0040-containerd-add-shim-exit-when-bundle-dir-does.patch b/patch/0040-containerd-add-shim-exit-when-bundle-dir-does.patch new file mode 100644 index 0000000000000000000000000000000000000000..b266393a4fc44c766ca879f8b592edea8ca33aef --- /dev/null +++ b/patch/0040-containerd-add-shim-exit-when-bundle-dir-does.patch @@ -0,0 +1,47 @@ +From 07605707cce769e4f4c79b700586b5c59ec0b15a Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Sat, 13 Jul 2019 06:32:54 +0800 +Subject: [PATCH] containerd: add shim exit when bundle dir does not + exist + +reason: when bundle dir is deleted, containerd-shim should exit to avoid +shim.sock is occupied when container restart next time. + +Change-Id: I956412598e17d15f25b91afe1cbb9e24463f04be +Signed-off-by: xiadanni1 +--- + runtime/v1/shim/service.go | 12 +++++++++++- + 1 file changed, 11 insertions(+), 1 deletion(-) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index 8adaf35..ac545ea 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -141,13 +141,23 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ * + } + rootfs := filepath.Join(r.Bundle, "rootfs") + defer func() { ++ go func() { ++ for i := 0; i < 60; i++ { ++ time.Sleep(time.Second) ++ _, err := os.Stat(r.Bundle) ++ if os.IsNotExist(err) { ++ logrus.Errorf("bundle dir: %v does not exist, containerd-shim exit", r.Bundle) ++ os.Exit(0) ++ } ++ } ++ }() + if err != nil { + logrus.Errorf("create init %s failed error=%v", r.ID, err) + if err2 := mount.UnmountAll(rootfs, 0); err2 != nil { + log.G(ctx).WithError(err2).Warn("Failed to cleanup rootfs mount") + } + go func() { +- time.Sleep(10*time.Second) ++ time.Sleep(10 * time.Second) + os.Exit(0) + }() + } +-- +1.8.3.1 + diff --git a/patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch b/patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch new file mode 100644 index 0000000000000000000000000000000000000000..910cb601853196ba1b16fd26aad9e20c122d5bc2 --- /dev/null +++ b/patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch @@ -0,0 +1,80 @@ +From be9c04e9a90be92437c12ce90c8ff6d4ec1d83b3 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 18 Jul 2019 07:57:52 -0400 +Subject: [PATCH] containerd: fix containerd call runv delete directly + use wrong --root parameters + +reason: When containerd-shim process is killed abnormaly, containerd will exec runv +delete command directly, however it will use the wrong --root parameters which is not +compatible with runv runtime. + +Signed-off-by: jiangpengfei +--- + runtime/v1/linux/proc/init.go | 4 ++-- + runtime/v1/linux/runtime.go | 10 +++++++++- + 2 files changed, 11 insertions(+), 3 deletions(-) + +diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go +index d464147..44d3f58 100644 +--- a/runtime/v1/linux/proc/init.go ++++ b/runtime/v1/linux/proc/init.go +@@ -45,7 +45,7 @@ import ( + const InitPidFile = "init.pid" + + // Default runv runtime root dir +-const defaultRunvRoot = "/run/runv" ++const DefaultRunvRoot = "/run/runv" + + // Init represents an initial process for a container + type Init struct { +@@ -89,7 +89,7 @@ func NewRunc(root, path, namespace, runtime, criu string, systemd bool) *runc.Ru + + rootPath := filepath.Join(root, namespace) + if strings.Contains(runtime, "runv") { +- rootPath = defaultRunvRoot ++ rootPath = DefaultRunvRoot + } + + return &runc.Runc{ +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 66914fe..f8e3074 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -25,6 +25,7 @@ import ( + "io/ioutil" + "os" + "path/filepath" ++ "strings" + "time" + + eventstypes "github.com/containerd/containerd/api/events" +@@ -506,6 +507,7 @@ func (r *Runtime) terminate(ctx context.Context, bundle *bundle, ns, id string) + if err != nil { + return err + } ++ + if err := rt.Delete(ctx, id, &runc.DeleteOpts{ + Force: true, + }); err != nil { +@@ -539,11 +541,17 @@ func (r *Runtime) getRuntime(ctx context.Context, ns, id string) (*runc.Runc, er + } + } + ++ rootPath := filepath.Join(root, ns) ++ ++ if strings.Contains(cmd, "runv") { ++ rootPath = proc.DefaultRunvRoot ++ } ++ + return &runc.Runc{ + Command: cmd, + LogFormat: runc.JSON, + PdeathSignal: unix.SIGKILL, +- Root: filepath.Join(root, ns), ++ Root: rootPath, + Debug: r.config.ShimDebug, + }, nil + } +-- +1.8.3.1 + diff --git a/patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch b/patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch new file mode 100644 index 0000000000000000000000000000000000000000..73af8e46aee4efa1745a8c5c3a06c2cf95e3630c --- /dev/null +++ b/patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch @@ -0,0 +1,34 @@ +From dcef6fcbdc78f7e9c14bdcd58e79d3eac8bc1c1b Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 18 Jul 2019 15:44:12 -0400 +Subject: [PATCH] containerd: close inherit shim.sock fd to adapt runv + +reason: runv create prcess is created by containerd-shim process and will +inherit the abstract unix socket shim.sock fd from containerd-shim. +If pause container restart, qemu and runv-proxy process are still running, +and shim.sock fd doesn't close, so pause container can not reuse the shim.sock +path and restart failed! + +Signed-off-by: jiangpengfei +--- + cmd/containerd-shim/main_unix.go | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go +index 38b3eb4..89f6be9 100644 +--- a/cmd/containerd-shim/main_unix.go ++++ b/cmd/containerd-shim/main_unix.go +@@ -189,6 +189,10 @@ func serve(ctx context.Context, server *ttrpc.Server, path string) error { + ) + if path == "" { + l, err = net.FileListener(os.NewFile(3, "socket")) ++ _, _, errnoValue := unix.Syscall(unix.SYS_FCNTL, 3, uintptr(unix.F_SETFD), unix.FD_CLOEXEC) ++ if errnoValue != 0 { ++ logrus.Errorf("SYS_FCNTL set fd 3 FD_CLOEXEC flag failed: %v", errnoValue) ++ } + path = "[inherited from parent]" + } else { + if len(path) > 106 { +-- +1.8.3.1 + diff --git a/patch/0043-containerd-run-state-with-timeout-10s.patch b/patch/0043-containerd-run-state-with-timeout-10s.patch new file mode 100644 index 0000000000000000000000000000000000000000..cfe0706287f9f60c91cd14f955aa31419f1075c2 --- /dev/null +++ b/patch/0043-containerd-run-state-with-timeout-10s.patch @@ -0,0 +1,77 @@ +From 7b9e8a793fa6c0ec67effac0bc53d55c275e13be Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Thu, 25 Jul 2019 19:29:50 +0800 +Subject: [PATCH] containerd: run state with timeout 10s + +Change-Id: Idf55f750c2e7c6a9268318f519f1c8bc1595e09e +Signed-off-by: jingrui +--- + Makefile | 4 ++-- + runtime/v1/linux/task.go | 3 --- + services/tasks/local.go | 11 +++++++++++ + 3 files changed, 13 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index 5de5cf75..9e7f3ae3 100644 +--- a/Makefile ++++ b/Makefile +@@ -81,8 +81,8 @@ GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",) + BEP_DIR=/tmp/containerd-build-bep + BEP_FLAGS=-tmpdir=/tmp/containerd-build-bep + +-GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' +-SHIM_GO_LDFLAGS=-ldflags '-s -w -buildid=IdByIsula $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' ++GO_LDFLAGS=-ldflags ' -buildid=IdByIsula -extldflags=-zrelro -extldflags=-znow $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(EXTRA_LDFLAGS)' ++SHIM_GO_LDFLAGS=-ldflags ' -buildid=IdByIsula $(BEP_FLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -extldflags "-static"' + + #Replaces ":" (*nix), ";" (windows) with newline for easy parsing + GOPATHS=$(shell echo ${GOPATH} | tr ":" "\n" | tr ";" "\n") +diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go +index b692ae78..d2bbb764 100644 +--- a/runtime/v1/linux/task.go ++++ b/runtime/v1/linux/task.go +@@ -92,9 +92,6 @@ func (t *Task) delete(ctx context.Context, force bool, pid uint32) (*runtime.Exi + rsp, err := t.shim.Delete(ctx, empty) + if err != nil { + log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force) +- if !force { +- return nil, errdefs.FromGRPC(err) +- } + } + t.tasks.Delete(ctx, t.id) + if err := t.shim.KillShim(ctx); err != nil { +diff --git a/services/tasks/local.go b/services/tasks/local.go +index ce9ee59d..990e8411 100644 +--- a/services/tasks/local.go ++++ b/services/tasks/local.go +@@ -47,6 +47,7 @@ import ( + ptypes "github.com/gogo/protobuf/types" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +@@ -185,9 +186,19 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. + if err := l.monitor.Monitor(c); err != nil { + return nil, errors.Wrap(err, "monitor task") + } ++ ++ ctx, cancel := context.WithTimeout(ctx, 20*time.Second) ++ defer cancel() ++ + state, err := c.State(ctx) + if err != nil { + log.G(ctx).Error(err) ++ go func() { ++ ctx, cancel := context.WithTimeout(context.Background(), time.Second) ++ defer cancel() ++ _, err := c.Delete(ctx) ++ logrus.Errorf("failed get pid, delete force error=%v", err) ++ }() + } + return &api.CreateTaskResponse{ + ContainerID: r.ContainerID, +-- +2.17.1 + diff --git a/patch/0044-containerd-add-copyright.patch b/patch/0044-containerd-add-copyright.patch new file mode 100644 index 0000000000000000000000000000000000000000..f40d27227836c78751425852a3974404feaa5492 --- /dev/null +++ b/patch/0044-containerd-add-copyright.patch @@ -0,0 +1,38 @@ +From 80972f7d142540b886068d67a49794aaa7232fb5 Mon Sep 17 00:00:00 2001 +From: lixiang +Date: Fri, 6 Sep 2019 15:16:21 +0800 +Subject: [PATCH] containerd: add copyright + +reason: add copyright + +Change-Id: I93ef565c6bf10d6f8cb66d956dddbfbd14477138 +Signed-off-by: lixiang +--- + events/exit.go | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/events/exit.go b/events/exit.go +index 772dc24..d3b3027 100644 +--- a/events/exit.go ++++ b/events/exit.go +@@ -1,3 +1,12 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++Use of this source code is governed by Apache-2.0 ++license that can be found in the LICENSE file ++Description: common functions ++Author: jingrui ++Create: 2019-02-12 ++*/ ++ + package events + + import ( +@@ -97,4 +106,3 @@ func InitExitExist(bundle string) bool { + } + return false + } +- +-- +1.8.3.1 + diff --git a/series.conf b/series.conf new file mode 100644 index 0000000000000000000000000000000000000000..b268de2c4df846590679825462783a6c7f6629b4 --- /dev/null +++ b/series.conf @@ -0,0 +1,46 @@ +patch/0006-shim-optimize-shim-lock-in-runtime-v1.patch +patch/0007-shim-Increase-reaper-buffer-size-and-non-bl.patch +patch/0008-runtime-Use-named-pipes-for-shim-logs.patch +patch/0009-runtime-fix-pipe-in-broken-may-cause-shim-l.patch +patch/0010-runtime-fix-pipe-in-broken-may-cause-shim-l.patch +patch/0011-runtime-Add-timeout-and-cancel-to-shim-fifo.patch +patch/0037-containerd-Fix-fd-leak-of-shim-log.patch + +patch/0001-grpc-vendor-grpc-fix-grpc-map-panic.patch +patch/0002-sys-sys-count-steal-time-when-calculating-Sys.patch +patch/0003-oci-oci-add-files-cgroups-support.patch +patch/0004-runv-vendor-runv-compatibility.patch +patch/0005-containerd-add-spec-for-build.patch +patch/0012-bump-bump-containerd-to-1.2.0.2.patch +patch/0013-log-support-log-init-pid-to-start-event-log.patch +patch/0014-event-resend-exit-event-when-detect-container.patch +patch/0015-restore-cleanup-container-pid-1.patch +patch/0016-create-runc-delete-force-before-create.patch +patch/0017-exit-using-init.exit-indicate-container-is-ex.patch +patch/0018-containerd-shim-Dump-log-to-file-when-docker-.patch +patch/0019-restore-check-shim-alive-when-containerd-is-r.patch +patch/0020-events-resend-pending-exit-events-on-restore.patch +patch/0021-containerd-Update-the-version-info-of-contain.patch +patch/0022-containerd-bump-version-1.2.0.4.patch +patch/0023-containerd-set-create-and-exec-timeout.patch +patch/0024-create-cleanup-runc-dirty-files-on-start.patch +patch/0025-restore-skip-load-task-in-creating.patch +patch/0026-exit-optimize-init.exit-record.patch +patch/0027-log-make-tester-happy.patch +patch/0028-restore-delete-task-in-containerd-restoring.patch +patch/0029-restore-delete-task-asynchronously.patch +patch/0030-event-fix-events-lost-when-loadTask-failed.patch +patch/0031-containerd-enable-relro-flags.patch +patch/0032-containerd-enable-bep-ldflags.patch +patch/0033-containerd-fix-opened-file-not-close.patch +patch/0034-containerd-add-buildid-in-Makefile.patch +patch/0035-containerd-fix-the-path-of-containerd.spec-in.patch +patch/0036-containerd-support-container-start-timeout-se.patch +patch/0037-containerd-fix-shim-std-logs-not-close-after-.patch +patch/0038-containerd-support-kill-D-state-container.patch +patch/0039-containerd-modify-containerd-shim-to-ad.patch +patch/0040-containerd-add-shim-exit-when-bundle-dir-does.patch +patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch +patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch +patch/0043-containerd-run-state-with-timeout-10s.patch +patch/0044-containerd-add-copyright.patch