diff --git a/containerd.spec b/containerd.spec index 70ad3cf78158fae52420a12b15d7544108be634a..e392a20490aadc8349d69852ad5ba6b749d4e29d 100644 --- a/containerd.spec +++ b/containerd.spec @@ -2,7 +2,7 @@ %global debug_package %{nil} Version: 1.2.0 Name: containerd -Release: 101 +Release: 102 Summary: An industry-standard container runtime License: ASL 2.0 URL: https://containerd.io @@ -41,3 +41,20 @@ install -p -m 755 bin/containerd-shim $RPM_BUILD_ROOT/%{_bindir}/containerd-shim %{_bindir}/containerd-shim %changelog +* Wed Nov 25 2020 xiadanni - 1.2.0-102 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:sync patches 0059-0063 + +* Thu Mar 5 2020 xiadanni - 1.2.0-101 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:sync patches 0038-0057 + +* Wed Jan 1 2020 xiadanni - 1.2.0-100 +- Type:requirement +- ID:NA +- SUG:NA +- DESC:package init diff --git a/patch/0059-containerd-add-GO_GCFLAGS-to-containerd-shim-making.patch b/patch/0059-containerd-add-GO_GCFLAGS-to-containerd-shim-making.patch new file mode 100644 index 0000000000000000000000000000000000000000..7f0bf1c2f52875f22700cf31229474b55ef41c38 --- /dev/null +++ b/patch/0059-containerd-add-GO_GCFLAGS-to-containerd-shim-making.patch @@ -0,0 +1,26 @@ +From 44079d9ee81c215d39ed81e39eb2ae31cf0ad453 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Tue, 11 Aug 2020 05:55:59 +0800 +Subject: [PATCH] add GO_GCFLAGS to containerd-shim making + +Signed-off-by: xiadanni1 +--- + Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makefile b/Makefile +index 6011aa1..ba512ef 100644 +--- a/Makefile ++++ b/Makefile +@@ -175,7 +175,7 @@ bin/%: cmd/% FORCE + + bin/containerd-shim: cmd/containerd-shim FORCE # set !cgo and omit pie for a static shim build: https://github.com/golang/go/issues/17789#issuecomment-258542220 + @echo "$(WHALE) bin/containerd-shim" +- @CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o bin/containerd-shim ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./cmd/containerd-shim ++ @CGO_ENABLED=0 go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o bin/containerd-shim ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./cmd/containerd-shim + + bin/containerd-shim-runc-v1: cmd/containerd-shim-runc-v1 FORCE # set !cgo and omit pie for a static shim build: https://github.com/golang/go/issues/17789#issuecomment-258542220 + @echo "$(WHALE) bin/containerd-shim-runc-v1" +-- +1.8.3.1 + diff --git a/patch/0060-containerd-do-not-disable-cgo-in-containerd-shim-mak.patch b/patch/0060-containerd-do-not-disable-cgo-in-containerd-shim-mak.patch new file mode 100644 index 0000000000000000000000000000000000000000..baf09bd00bb1a933fe135f8b28d18de217f70115 --- /dev/null +++ b/patch/0060-containerd-do-not-disable-cgo-in-containerd-shim-mak.patch @@ -0,0 +1,28 @@ +From 6523d7e39a9bb45be632ff114c64329f43e1499a Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Wed, 12 Aug 2020 01:52:16 +0800 +Subject: [PATCH] containerd: do not disable cgo in containerd-shim making + +reason: for debuginfo + +Signed-off-by: xiadanni1 +--- + Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makefile b/Makefile +index ba512ef..f69559b 100644 +--- a/Makefile ++++ b/Makefile +@@ -175,7 +175,7 @@ bin/%: cmd/% FORCE + + bin/containerd-shim: cmd/containerd-shim FORCE # set !cgo and omit pie for a static shim build: https://github.com/golang/go/issues/17789#issuecomment-258542220 + @echo "$(WHALE) bin/containerd-shim" +- @CGO_ENABLED=0 go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o bin/containerd-shim ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./cmd/containerd-shim ++ go build ${GO_BUILD_FLAGS} -o bin/containerd-shim ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./cmd/containerd-shim + + bin/containerd-shim-runc-v1: cmd/containerd-shim-runc-v1 FORCE # set !cgo and omit pie for a static shim build: https://github.com/golang/go/issues/17789#issuecomment-258542220 + @echo "$(WHALE) bin/containerd-shim-runc-v1" +-- +1.8.3.1 + diff --git a/patch/0061-containerd-check-if-bundle-exists-before-create-bund.patch b/patch/0061-containerd-check-if-bundle-exists-before-create-bund.patch new file mode 100644 index 0000000000000000000000000000000000000000..2969e088bdaff0ef68d2fafac87c00ed37bf2f40 --- /dev/null +++ b/patch/0061-containerd-check-if-bundle-exists-before-create-bund.patch @@ -0,0 +1,66 @@ +From c56df3dd08d709e8ee81675661527aac47a7cba2 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Fri, 6 Nov 2020 10:19:26 +0800 +Subject: [PATCH] containerd: check if bundle exists before create bundle + +reason: If container starts following tightly the last stop, bundle +directory may be deleted by the not yet completed stop, which may cause +container start fail. So we add bundle check during start to avoid this, +if bundle exists, wait for it to clean up. + +Signed-off-by: xiadanni1 +--- + runtime/v1/linux/bundle.go | 17 ++++++++++++++++- + 1 file changed, 16 insertions(+), 1 deletion(-) + +diff --git a/runtime/v1/linux/bundle.go b/runtime/v1/linux/bundle.go +index d73866a..b4f7b4c 100644 +--- a/runtime/v1/linux/bundle.go ++++ b/runtime/v1/linux/bundle.go +@@ -23,12 +23,14 @@ import ( + "io/ioutil" + "os" + "path/filepath" ++ "time" + + "github.com/containerd/containerd/events/exchange" + "github.com/containerd/containerd/runtime/linux/runctypes" + "github.com/containerd/containerd/runtime/v1/shim" + "github.com/containerd/containerd/runtime/v1/shim/client" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + // loadBundle loads an existing bundle from disk +@@ -46,6 +48,20 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) { + return nil, err + } + path = filepath.Join(path, id) ++ workDir = filepath.Join(workDir, id) ++ ++ for waitTime := 10 * time.Millisecond; ; waitTime *= 2 { ++ if _, err = os.Stat(workDir); err != nil { ++ break ++ } ++ logrus.Debugf("bundle-check: wait time %v", waitTime) ++ if waitTime > 2*time.Second { ++ logrus.Warnf("bundle-check: waiting cleanup bundle timeout, start anyway") ++ break ++ } ++ time.Sleep(waitTime) ++ } ++ + if err := os.Mkdir(path, 0711); err != nil { + return nil, err + } +@@ -54,7 +70,6 @@ func newBundle(id, path, workDir string, spec []byte) (b *bundle, err error) { + os.RemoveAll(path) + } + }() +- workDir = filepath.Join(workDir, id) + if err := os.MkdirAll(workDir, 0711); err != nil { + return nil, err + } +-- +1.8.3.1 + diff --git a/patch/0062-containerd-use-path-based-socket-for-shims.patch b/patch/0062-containerd-use-path-based-socket-for-shims.patch new file mode 100644 index 0000000000000000000000000000000000000000..b96cd00bc5673508ef69759e61d532ed3aa54073 --- /dev/null +++ b/patch/0062-containerd-use-path-based-socket-for-shims.patch @@ -0,0 +1,318 @@ +From 4185b832a4f89e671e6ecf201d21b75d866a48e4 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Sat, 14 Nov 2020 15:55:30 +0800 +Subject: [PATCH] use path based socket for shims + +Signed-off-by: jingrui +--- + cmd/containerd-shim/main_unix.go | 16 +++-- + cmd/ctr/commands/shim/shim.go | 2 + + runtime/v1/linux/bundle.go | 37 +++++++++- + runtime/v1/shim/client/client.go | 118 ++++++++++++++++++++++++++++--- + 4 files changed, 159 insertions(+), 14 deletions(-) + +diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go +index e9c14263b..3a5bb6170 100644 +--- a/cmd/containerd-shim/main_unix.go ++++ b/cmd/containerd-shim/main_unix.go +@@ -66,7 +66,7 @@ var ( + func init() { + flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs") + flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim") +- flag.StringVar(&socketFlag, "socket", "", "abstract socket path to serve") ++ flag.StringVar(&socketFlag, "socket", "", "socket path to serve") + flag.StringVar(&addressFlag, "address", "", "grpc address back to main containerd") + flag.StringVar(&workdirFlag, "workdir", "", "path used to storge large temporary data") + flag.StringVar(&runtimeRootFlag, "runtime-root", proc.RuncRoot, "root directory for the runtime") +@@ -190,10 +190,18 @@ func serve(ctx context.Context, server *ttrpc.Server, path string) error { + } + path = "[inherited from parent]" + } else { +- if len(path) > 106 { +- return errors.Errorf("%q: unix socket path too long (> 106)", path) ++ const ( ++ abstractSocketPrefix = "\x00" ++ socketPathLimit = 106 ++ ) ++ p := strings.TrimPrefix(path, "unix://") ++ if len(p) == len(path) { ++ p = abstractSocketPrefix + p + } +- l, err = net.Listen("unix", "\x00"+path) ++ if len(p) > socketPathLimit { ++ return errors.Errorf("%q: unix socket path too long (> %d)", p, socketPathLimit) ++ } ++ l, err = net.Listen("unix", p) + } + if err != nil { + return err +diff --git a/cmd/ctr/commands/shim/shim.go b/cmd/ctr/commands/shim/shim.go +index ec08cc68b..8ef068292 100644 +--- a/cmd/ctr/commands/shim/shim.go ++++ b/cmd/ctr/commands/shim/shim.go +@@ -23,6 +23,7 @@ import ( + "fmt" + "io/ioutil" + "net" ++ "strings" + + "github.com/containerd/console" + "github.com/containerd/containerd/cmd/ctr/commands" +@@ -231,6 +232,7 @@ func getTaskService(context *cli.Context) (task.TaskService, error) { + return nil, errors.New("socket path must be specified") + } + ++ bindSocket = strings.TrimPrefix(bindSocket, "unix://") + conn, err := net.Dial("unix", "\x00"+bindSocket) + if err != nil { + return nil, err +diff --git a/runtime/v1/linux/bundle.go b/runtime/v1/linux/bundle.go +index ef4200b29..0442246f9 100644 +--- a/runtime/v1/linux/bundle.go ++++ b/runtime/v1/linux/bundle.go +@@ -20,6 +20,7 @@ package linux + + import ( + "context" ++ "fmt" + "io/ioutil" + "os" + "path/filepath" +@@ -117,7 +118,7 @@ func ShimLocal(c *Config, exchange *exchange.Exchange) ShimOpt { + // ShimConnect is a ShimOpt for connecting to an existing remote shim + func ShimConnect(c *Config, onClose func()) ShimOpt { + return func(b *bundle, ns string, ropts *runctypes.RuncOptions) (shim.Config, client.Opt) { +- return b.shimConfig(ns, c, ropts), client.WithConnect(b.shimAddress(ns), onClose) ++ return b.shimConfig(ns, c, ropts), client.WithConnect(b.decideShimAddress(ns), onClose) + } + } + +@@ -129,6 +130,11 @@ func (b *bundle) NewShimClient(ctx context.Context, namespace string, getClientO + + // Delete deletes the bundle from disk + func (b *bundle) Delete() error { ++ address, _ := b.loadAddress() ++ if address != "" { ++ // we don't care about errors here ++ client.RemoveSocket(address) ++ } + err := os.RemoveAll(b.path) + if err == nil { + return os.RemoveAll(b.workDir) +@@ -141,10 +147,37 @@ func (b *bundle) Delete() error { + return errors.Wrapf(err, "Failed to remove both bundle and workdir locations: %v", err2) + } + +-func (b *bundle) shimAddress(namespace string) string { ++func (b *bundle) legacyShimAddress(namespace string) string { + return filepath.Join(string(filepath.Separator), "containerd-shim", namespace, b.id, "shim.sock") + } + ++const socketRoot = "/run/containerd" ++ ++func (b *bundle) shimAddress(namespace string) string { ++ return fmt.Sprintf("unix://%s", b.shimSock()) ++} ++ ++func (b *bundle) shimSock() string { ++ return filepath.Join(socketRoot, "s", b.id) ++} ++ ++func (b *bundle) loadAddress() (string, error) { ++ addressPath := filepath.Join(b.path, "address") ++ data, err := ioutil.ReadFile(addressPath) ++ if err != nil { ++ return "", err ++ } ++ return string(data), nil ++} ++ ++func (b *bundle) decideShimAddress(namespace string) string { ++ address, err := b.loadAddress() ++ if err != nil { ++ return b.legacyShimAddress(namespace) ++ } ++ return address ++} ++ + func (b *bundle) shimConfig(namespace string, c *Config, runcOptions *runctypes.RuncOptions) shim.Config { + var ( + criuPath string +diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go +index a4669d33c..06453b35a 100644 +--- a/runtime/v1/shim/client/client.go ++++ b/runtime/v1/shim/client/client.go +@@ -20,11 +20,14 @@ package client + + import ( + "context" ++ "fmt" + "io" + "net" + "os" + "os/exec" ++ "path/filepath" + "runtime" ++ "strconv" + "strings" + "sync" + "syscall" +@@ -55,9 +58,17 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa + return func(ctx context.Context, config shim.Config) (_ shimapi.ShimService, _ io.Closer, err error) { + socket, err := newSocket(address) + if err != nil { +- return nil, nil, err ++ if !eaddrinuse(err) { ++ return nil, nil, err ++ } ++ if err := RemoveSocket(address); err != nil { ++ return nil, nil, errors.Wrap(err, "remove already used socket") ++ } ++ if socket, err = newSocket(address); err != nil { ++ return nil, nil, err ++ } + } +- defer socket.Close() ++ + f, err := socket.File() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get fd for socket %s", address) +@@ -102,12 +113,22 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa + if stderrLog != nil { + stderrLog.Close() + } ++ socket.Close() ++ RemoveSocket(address) + }() + log.G(ctx).WithFields(logrus.Fields{ + "pid": cmd.Process.Pid, + "address": address, + "debug": debug, + }).Infof("shim %s started", binary) ++ ++ if err := writeFile(filepath.Join(config.Path, "address"), address); err != nil { ++ return nil, nil, err ++ } ++ if err := writeFile(filepath.Join(config.Path, "shim.pid"), strconv.Itoa(cmd.Process.Pid)); err != nil { ++ return nil, nil, err ++ } ++ + // set shim in cgroup if it is provided + if cgroup != "" { + if err := setCgroup(cgroup, cmd); err != nil { +@@ -170,25 +191,106 @@ func newCommand(binary, daemonAddress string, debug bool, config shim.Config, so + return cmd, nil + } + ++// writeFile writes a address file atomically ++func writeFile(path, address string) error { ++ path, err := filepath.Abs(path) ++ if err != nil { ++ return err ++ } ++ tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path))) ++ f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) ++ if err != nil { ++ return err ++ } ++ _, err = f.WriteString(address) ++ f.Close() ++ if err != nil { ++ return err ++ } ++ return os.Rename(tempPath, path) ++} ++ ++const ( ++ abstractSocketPrefix = "\x00" ++ socketPathLimit = 106 ++) ++ ++func eaddrinuse(err error) bool { ++ cause := errors.Cause(err) ++ netErr, ok := cause.(*net.OpError) ++ if !ok { ++ return false ++ } ++ if netErr.Op != "listen" { ++ return false ++ } ++ syscallErr, ok := netErr.Err.(*os.SyscallError) ++ if !ok { ++ return false ++ } ++ errno, ok := syscallErr.Err.(syscall.Errno) ++ if !ok { ++ return false ++ } ++ return errno == syscall.EADDRINUSE ++} ++ ++type socket string ++ ++func (s socket) isAbstract() bool { ++ return !strings.HasPrefix(string(s), "unix://") ++} ++ ++func (s socket) path() string { ++ path := strings.TrimPrefix(string(s), "unix://") ++ // if there was no trim performed, we assume an abstract socket ++ if len(path) == len(s) { ++ path = abstractSocketPrefix + path ++ } ++ return path ++} ++ + func newSocket(address string) (*net.UnixListener, error) { +- if len(address) > 106 { +- return nil, errors.Errorf("%q: unix socket path too long (> 106)", address) ++ if len(address) > socketPathLimit { ++ return nil, errors.Errorf("%q: unix socket path too long (> %d)", address, socketPathLimit) ++ } ++ var ( ++ sock = socket(address) ++ path = sock.path() ++ ) ++ if !sock.isAbstract() { ++ if err := os.MkdirAll(filepath.Dir(path), 0600); err != nil { ++ return nil, errors.Wrapf(err, "%s", path) ++ } + } +- l, err := net.Listen("unix", "\x00"+address) ++ l, err := net.Listen("unix", path) + if err != nil { +- return nil, errors.Wrapf(err, "failed to listen to abstract unix socket %q", address) ++ return nil, errors.Wrapf(err, "failed to listen to unix socket %q (abstract: %t)", address, sock.isAbstract()) ++ } ++ if err := os.Chmod(path, 0600); err != nil { ++ l.Close() ++ return nil, err + } + + return l.(*net.UnixListener), nil + } + ++// RemoveSocket removes the socket at the specified address if ++// it exists on the filesystem ++func RemoveSocket(address string) error { ++ sock := socket(address) ++ if !sock.isAbstract() { ++ return os.Remove(sock.path()) ++ } ++ return nil ++} ++ + func connect(address string, d func(string, time.Duration) (net.Conn, error)) (net.Conn, error) { + return d(address, 100*time.Second) + } + + func annonDialer(address string, timeout time.Duration) (net.Conn, error) { +- address = strings.TrimPrefix(address, "unix://") +- return net.DialTimeout("unix", "\x00"+address, timeout) ++ return net.DialTimeout("unix", socket(address).path(), timeout) + } + + // WithConnect connects to an existing shim +-- +2.17.1 + diff --git a/patch/0063-containerd-kill-init-directly-if-runtime-kill-failed.patch b/patch/0063-containerd-kill-init-directly-if-runtime-kill-failed.patch new file mode 100644 index 0000000000000000000000000000000000000000..a3e9bfcd69947ce46f6683e04972deb9fbe577ff --- /dev/null +++ b/patch/0063-containerd-kill-init-directly-if-runtime-kill-failed.patch @@ -0,0 +1,37 @@ +From 3ec035244d33b4cb64adacb8133ae3e204cae55f Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Thu, 19 Nov 2020 15:49:53 +0800 +Subject: [PATCH] containerd: kill init directly if runtime kill failed + +Change-Id: I80a1c0c4f88530fe9732e6e9a2d1fb222ece118c +Signed-off-by: jingrui +--- + runtime/v1/shim/service.go | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index beb0ed8d5..7e07ab011 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -49,6 +49,7 @@ import ( + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ++ "golang.org/x/sys/unix" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + ) +@@ -390,6 +391,10 @@ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Emp + time.Sleep(10 * time.Second) + err := p.Kill(ctx, r.Signal, r.All) + logrus.Infof("delay kill %s retry %d error=%v", s.id, i, err) ++ if err != nil { ++ err := unix.Kill(p.Pid(), syscall.SIGKILL) ++ logrus.Infof("delay kill-direct %s retry %d error=%v", s.id, i, err) ++ } + } + + logrus.Infof("force exit shim %s ...", s.id) +-- +2.17.1 + diff --git a/series.conf b/series.conf index 05c09a330a82ac67161d26fd7abc1bda712ff66c..a0357e71b193f1a88997cab247c8b4b57232c9ec 100644 --- a/series.conf +++ b/series.conf @@ -60,3 +60,8 @@ patch/0054-containerd-clean-up-residual-container.patch patch/0055-containerd-add-LLT-for-containerd-shim-timeou.patch patch/0056-containerd-save-dumpstack-to-file.patch patch/0057-containerd-add-timeout-for-delete-command.patch +patch/0059-containerd-add-GO_GCFLAGS-to-containerd-shim-making.patch +patch/0060-containerd-do-not-disable-cgo-in-containerd-shim-mak.patch +patch/0061-containerd-check-if-bundle-exists-before-create-bund.patch +patch/0062-containerd-use-path-based-socket-for-shims.patch +patch/0063-containerd-kill-init-directly-if-runtime-kill-failed.patch