diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..75390fd316504c0fd8164bb94c97a6e53a40f86d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# IluvatarCorex Container Toolkit Changelog + +## v1.0.0 + +- [ix-ctk] Support configure docker, containerd and cri-o runtime +- [ix-ctk] Support generate CDI +- [ix-container-runtime] Support mount GPU devices using `IX_VISIBLE_DEVICES` environment variable diff --git a/Makefile b/Makefile index 166975c5b406005b23b1b1ca5ee2db516e21a082..045d62d10e92bfdbe4c6763cd0f40fb16f472ad9 100644 --- a/Makefile +++ b/Makefile @@ -13,13 +13,5 @@ uninstall: rm -f /usr/local/bin/ix-container-runtime rm -f /usr/local/bin/ix-ctk -generate-deps: all - mkdir -p build/deps - cp install.sh build/deps/ - cp build/ix-container-runtime build/deps/ - cp build/ix-ctk build/deps/ - cp binary/* build/deps/ - makeself build/deps build/ix-runtime-installer.run "IxRuntime" ./install.sh - clean: rm -rf build diff --git a/README.md b/README.md index 006402242d8788620b4dffe188ea3e707bc9f3b7..247756c655363f9d84f23880c950f0578355e4a6 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,11 @@ - [Configuring](#configuring) - [Configuring Docker](#configuring-docker) - [Configuring Containerd](#configuring-containerd) + - [Configuring Crio](#configuring-crio) - [Running Samples](#running-samples) - - [Running Samples with Docker](#running-samples-with-docker) - - [Running Samples with Containerd](#running-samples-with-containerd) - - [Running Samples with Podman](#running-samples-with-podman) + - [Running a Sample Workload with Docker](#running-a-sample-workload-with-docker) + - [Running a Sample Workload with Containerd/Crio(for kubernetes 1.22+)](#running-a-sample-workload-with-containerd/crio(for-kubernetes-1.22+)) + - [Running a Sample Workload with Podman](#running-a-sample-workload-with-podman) - [License](#license) ## Introduction @@ -63,6 +64,23 @@ sudo systemctl daemon-reload sudo systemctl restart containerd ``` +### Configuring Crio (for Kubernetes) + +1. Run the ix-ctk + +```shell +sudo ix-ctk runtime configure --runtime crio --ix-set-as-default +``` + +Run the `runtime configure --runtime crio` command will automatically add the configuration of `ix-container-runtime` to `/etc/crio/crio.conf`, The default configuration is generated in the config.yaml `/etc/iluvatarcorex/ix-container-runtime/` file, and the containerd service needs to be restarted for the configuration to take effect. + +2. Restart crio service + +```shell +sudo systemctl daemon-reload +sudo systemctl restart crio +``` + ## Running Samples ### Running a Sample Workload with Docker @@ -94,7 +112,7 @@ Your output should resemble the following output: +-----------------------------------------------------------------------------+ ``` -### Running a Sample Workload with Containerd(for kubernetes 1.24+) +### Running a Sample Workload with Containerd/Crio(for kubernetes 1.22+) After you install and configure the toolkit and install Iluvatar GPU Driver and SDK, you can verify your installation by running a sample workload with kubernetes. diff --git a/cmd/ix-ctk/runtime/configure/configure.go b/cmd/ix-ctk/runtime/configure/configure.go index 287db11984b33913cc7c2a8fe83ec8e638894afc..0dfc99c476a0abceceeb95859f8fba157187c2f5 100644 --- a/cmd/ix-ctk/runtime/configure/configure.go +++ b/cmd/ix-ctk/runtime/configure/configure.go @@ -23,6 +23,7 @@ import ( "gitee.com/deep-spark/ix-container-runtime/internal/config/engine" "gitee.com/deep-spark/ix-container-runtime/internal/config/engine/containerd" + "gitee.com/deep-spark/ix-container-runtime/internal/config/engine/crio" "gitee.com/deep-spark/ix-container-runtime/internal/config/engine/docker" "github.com/urfave/cli/v2" ) @@ -131,6 +132,10 @@ func ConfigureRuntime(c config) error { cfg, err = containerd.New( containerd.WithPath(configFilePath), ) + case "crio": + cfg, err = crio.New( + crio.WithPath(configFilePath), + ) case "docker": cfg, err = docker.New( docker.WithPath(configFilePath), diff --git a/install.sh b/install.sh deleted file mode 100755 index 61d70df8356227713b8d44cf6352a893be8bd9cd..0000000000000000000000000000000000000000 --- a/install.sh +++ /dev/null @@ -1,268 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -docker_tmp_name="docker_tmp.json" -containerd_tmp_name="containerd_tmp.toml" -docker_backup_name="docker_backup.json" -containerd_backup_name="containerd_backup.toml" -iluvatar_tmp_name="iluvatar_config_tmp.yaml" -iluvatar_backup_name="iluvatar_config_backup.yaml" - -iluvatar_config_dir="/etc/iluvatarcorex/ix-container-runtime" -iluvatar_config_name="config.yaml" -iluvatar_config_path="${iluvatar_config_dir}/${iluvatar_config_name}" -out_name="ix-container-runtime" - -iluvatar_log_dir="/var/log/iluvatarcorex/ix-container-toolkit/" - -check_runtime_install() { - isExist="$(which $1)" - if [ -z "${isExist}" ];then - return 0 - else - return 1 - fi -} - -configure_docker_env() { - work_dir="$1" - override="$2" - isInstalled="$3" - - temp_config_path="${work_dir}/${docker_tmp_name}" - backup_config_path="${work_dir}/${docker_backup_name}" - docker_config_path="/etc/docker/daemon.json" - docker_config_content="" - check_runtime_install docker - docker_installed="$?" - - if [ "${docker_installed}" -eq 1 ];then - mkdir -p $(dirname ${docker_config_path}) - - if [ -f "${docker_config_path}" ];then - docker_config_content=$(cat ${docker_config_path}) - cp ${docker_config_path} ${backup_config_path} - fi - if [ "${isInstalled}" = true ]; then - printf "%s\n" "${docker_config_content}" | ./yq_linux_amd64 '.runtimes += {"iluvatar": {"args": [], "path":"/usr/local/bin/ix-container-runtime"}}|."default-runtime"="iluvatar"' -o json > ${temp_config_path} - else - printf "%s\n" "${docker_config_content}" | ./yq_linux_amd64 'del(."runtimes"."iluvatar", ."default-runtime")' -o json > ${temp_config_path} - fi - - if [ "${override}" = true ];then - cp ${temp_config_path} ${docker_config_path} - fi - fi -} - -configure_containerd_env() { - work_dir="$1" - override="$2" - isInstalled="$3" - - temp_config_path="${work_dir}/${containerd_tmp_name}" - backup_config_path="${work_dir}/${containerd_backup_name}" - containerd_config_path="/etc/containerd/config.toml" - containerd_config_content=$(containerd config default) - check_runtime_install containerd - containerd_installed="$?" - - if [ "${containerd_installed}" -eq 1 ];then - mkdir -p $(dirname ${containerd_config_path}) - - if [ -f "${containerd_config_path}" ];then - containerd_config_content=$(cat ${containerd_config_path}) - cp ${containerd_config_path} ${backup_config_path} - fi - - if [ "${isInstalled}" = true ]; then - printf "%s\n" "${containerd_config_content}"| ./yq_linux_amd64 --input-format=toml '.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.iluvatar += {"runtime_type": "io.containerd.runc.v1", "options": {"BinaryName":"/usr/local/bin/ix-container-runtime"}}'|./dasel_linux_amd64 -r yaml -w toml > ${temp_config_path} - else - printf "%s\n" "${containerd_config_content}"| ./yq_linux_amd64 --input-format=toml 'del(.plugins."io.containerd.grpc.v1.cri".containerd.runtimes.iluvatar)' |./dasel_linux_amd64 -r yaml -w toml > ${temp_config_path} - fi - - if [ "${override}" = true ];then - cp ${temp_config_path} ${containerd_config_path} - fi - fi -} - -configure_iluvatar_env() { - work_dir="$1" - override="$2" - isInstalled="$3" - temp_config_path="${work_dir}/${iluvatar_tmp_name}" - backup_config_path="${work_dir}/${iluvatar_backup_name}" - iluvatar_config_content="librarypath: /usr/local/corex/lib64/libixml.so" - - mkdir -p ${iluvatar_config_dir} - - - if [ -f "${iluvatar_config_path}" ];then - iluvatar_config_content=$(cat ${iluvatar_config_path}) - cp ${iluvatar_config_path} ${backup_config_path} - fi - - if [ "${isInstalled}" = true ]; then - printf "%s\n" "${iluvatar_config_content}"| ./yq_linux_amd64 --input-format=yaml '.librarypath = "/usr/local/corex/lib64/libixml.so"|.sdksocketpath = "/run/ix-sdk-manager/ix-sdk.sock"' -o yaml > ${temp_config_path} - - if [ "${override}" = true ];then - cp ${temp_config_path} ${iluvatar_config_path} - fi - else - if [ "${override}" = true ];then - rm -rf ${iluvatar_config_dir} - fi - fi -} - -install() { - local override=false - local rm=false - while [[ "$1" != "" ]];do - case "$1" in - --override) - shift - override=true - ;; - --rm) - shift - override=true - rm=true - ;; - *) - help - exit 1 - ;; - esac - done - - echo "Start to Install ix-Runtime" - temp_dir=$(mktemp -d) - if [ ! -d "${temp_dir}" ];then - echo "Failed to create temp directory" - exit 1 - else - echo "Working temp directory is ${temp_dir}" - fi - - mkdir -p ${iluvatar_log_dir} - cp ix-container-runtime /usr/local/bin - cp ix-ctk /usr/local/bin - chmod 755 /usr/local/bin/ix-ctk - chmod 755 /usr/local/bin/ix-container-runtime - configure_docker_env ${temp_dir} ${override} true - configure_containerd_env ${temp_dir} ${override} true - configure_iluvatar_env ${temp_dir} ${override} true - - if [ "${override}" = true ];then - echo "----------------" - echo "Install Finished" - echo "Please restart your service to update your environment" - if [ "${rm}" = true ]; then - rm -rf ${temp_dir} - else - echo "Please delete the temporary directory ${temp_dir} once you have confirmed they are correct." - fi - else - echo "********************************" - echo "Generated Configuration Finished" - echo "Please enter ${temp_dir} to configure the environment" - echo "The file ${docker_tmp_name} used for configrue the docker config file" - echo "The file ${containerd_tmp_name} used for configrue the containerd config file" - echo "The file ${iluvatar_tmp_name} used for configrue the iluvatar config file, install " - echo "mannual by put this file under ${iluvatar_config_dir} and renamed to ${iluvatar_config_name}" - fi -} - -uninstall() { - local override=false - local rm=false - while [[ "$1" != "" ]];do - case "$1" in - --override) - shift - override=true - ;; - --rm) - shift - override=true - rm=true - ;; - *) - help - exit 1 - ;; - esac - done - echo "Start to Uninstall ix-Runtime" - temp_dir=$(mktemp -d) - if [ ! -d "${temp_dir}" ];then - echo "Failed to create temp directory" - exit 1 - else - echo "Working temp directory is ${temp_dir}" - fi - - - configure_docker_env ${temp_dir} ${override} false - configure_containerd_env ${temp_dir} ${override} false - configure_iluvatar_env ${temp_dir} ${override} false - if [ "${override}" = true ];then - rm /usr/local/bin/ix-container-runtime - echo "----------------" - echo "Uninstall Finished" - echo "Please restart your service to update your environment" - if [ "${rm}" = true ]; then - rm -rf ${temp_dir} - else - echo "Please delete the temporary directory ${temp_dir} once you have confirmed they are correct." - fi - else - echo "********************************" - echo "Generated Configuration Finished" - echo "Please enter ${temp_dir} to configure the environment" - echo "The file ${docker_tmp_name} used for configrue the docker config file" - echo "The file ${containerd_tmp_name} used for configrue the containerd config file" - fi -} - -help() { - echo "Usage: ix-runtime-installer.run {install|uninstall} --override --rm" - echo "--override will override the configuration file(default is false)" - echo "--rm not only override the configuration file, but also remove the workspace(default is false)" - exit 1 -} - -if [ -z "$1" ]; then - help -fi - -case "$1" in - install) - shift - install "$@" - ;; - uninstall) - shift - uninstall "$@" - ;; - *) - help - ;; -esac diff --git a/internal/config/config.go b/internal/config/config.go index 0127d868ee208e5b7f83a4d9731243b2e962c6b9..86118e2fc50cbbe9084defc0012e6b5a071dde5b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "path/filepath" "runtime" log "github.com/sirupsen/logrus" @@ -97,6 +98,13 @@ func (c *Config) update() error { } log.SetLevel(level) + if dir := filepath.Dir(c.LogPath); dir != "" { + err := os.MkdirAll(dir, 0755) + if err != nil { + return fmt.Errorf("unable to create directory %v: %v", dir, err) + } + } + reader, err := os.OpenFile(c.LogPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) if err != nil { return fmt.Errorf("error opening config file: %v", err) diff --git a/internal/config/engine/crio/crio.go b/internal/config/engine/crio/crio.go new file mode 100644 index 0000000000000000000000000000000000000000..26eaf90311fcd7f5647be14390a2baf4da3621a2 --- /dev/null +++ b/internal/config/engine/crio/crio.go @@ -0,0 +1,118 @@ +/** +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package crio + +import ( + "fmt" + + "gitee.com/deep-spark/ix-container-runtime/internal/config/engine" + "github.com/pelletier/go-toml" +) + +// Config represents the cri-o config +type Config struct { + *toml.Tree +} + +var _ engine.Interface = (*Config)(nil) + +// New creates a cri-o config with the specified options +func New(opts ...Option) (engine.Interface, error) { + b := &builder{} + for _, opt := range opts { + opt(b) + } + + return b.build() +} + +// AddRuntime adds a new runtime to the crio config +func (c *Config) AddRuntime(name string, path string, setAsDefault bool, _ ...map[string]interface{}) error { + if c == nil { + return fmt.Errorf("config is nil") + } + + config := *c.Tree + + if runc, ok := config.GetPath([]string{"cri", "runtime", "runtimes", "runc"}).(*toml.Tree); ok { + runc, _ = toml.Load(runc.String()) + config.SetPath([]string{"crio", "runtime", "runtimes", name}, runc) + } + + config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_path"}, path) + config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_type"}, "oci") + + if setAsDefault { + config.SetPath([]string{"crio", "runtime", "default_runtime"}, name) + } + + *c.Tree = config + return nil +} + +// Save writes the config to the specified path +func (c Config) Save(path string) (int64, error) { + config := c.Tree + output, err := config.Marshal() + if err != nil { + return 0, fmt.Errorf("unable to convert to TOML: %v", err) + } + + n, err := engine.Config(path).Write(output) + return int64(n), err +} + +// DefaultRuntime returns the default runtime for the cri-o config +func (c *Config) DefaultRuntime() string { + if c == nil || c.Tree == nil { + return "" + } + if runtime, ok := c.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok { + return runtime + } + return "" +} + +// RemoveRuntime removes a runtime from the cri-o config +func (c *Config) RemoveRuntime(name string) error { + if c == nil { + return nil + } + + config := *c.Tree + if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok { + if runtime == name { + config.DeletePath([]string{"crio", "runtime", "default_runtime"}) + } + } + + runtimeClassPath := []string{"crio", "runtime", "runtimes", name} + config.DeletePath(runtimeClassPath) + for i := 0; i < len(runtimeClassPath); i++ { + remainingPath := runtimeClassPath[:len(runtimeClassPath)-i] + if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok { + if len(entry.Keys()) != 0 { + break + } + config.DeletePath(remainingPath) + } + } + + *c.Tree = config + return nil +} diff --git a/internal/config/engine/crio/option.go b/internal/config/engine/crio/option.go new file mode 100644 index 0000000000000000000000000000000000000000..4e5c81f35318217dd734ee0383fb0271fdaab628 --- /dev/null +++ b/internal/config/engine/crio/option.go @@ -0,0 +1,79 @@ +/** +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package crio + +import ( + "fmt" + "os" + + "gitee.com/deep-spark/ix-container-runtime/internal/config/engine" + "github.com/pelletier/go-toml" + log "github.com/sirupsen/logrus" +) + +type builder struct { + path string +} + +// Option defines a function that can be used to configure the config builder +type Option func(*builder) + +// WithPath sets the path for the config builder +func WithPath(path string) Option { + return func(b *builder) { + b.path = path + } +} + +func (b *builder) build() (engine.Interface, error) { + if b.path == "" { + return nil, fmt.Errorf("config path is empty") + } + + config, err := b.loadConfig(b.path) + if err != nil { + return nil, fmt.Errorf("failed to load config: %v", err) + } + + return config, nil +} + +// loadConfig loads the containerd config from disk +func (b *builder) loadConfig(config string) (*Config, error) { + info, err := os.Stat(config) + if os.IsExist(err) && info.IsDir() { + return nil, fmt.Errorf("config file is a directory") + } + + if os.IsNotExist(err) { + log.Infof("Config file does not exist; using empty config") + config = "/dev/null" + } else { + log.Infof("Loading config from %v", config) + } + + tomlConfig, err := toml.LoadFile(config) + if err != nil { + return nil, err + } + + cfg := Config{ + Tree: tomlConfig, + } + return &cfg, nil +} diff --git a/internal/modifier/graphics.go b/internal/modifier/graphics.go index 431ffbed3163f41f9989b29666ed09b73bcff751..519d971ee557699592d235a5d03de43f0477fed3 100644 --- a/internal/modifier/graphics.go +++ b/internal/modifier/graphics.go @@ -137,7 +137,7 @@ func searchDevice() map[int]specs.LinuxDevice { return ret } -func generate_dev_from_string(devmap map[uint]IndexDevice, val string, mountIdx int) *specs.LinuxDevice { +func generate_dev_from_string(devmap map[uint]IndexDevice, val string) *specs.LinuxDevice { var ret specs.LinuxDevice i, err := strconv.Atoi(val) if err != nil { @@ -151,7 +151,7 @@ func generate_dev_from_string(devmap map[uint]IndexDevice, val string, mountIdx return nil } ret = dev.LinuxDevice - strIdx := strconv.Itoa(mountIdx) + strIdx := strconv.Itoa(int(dev.Minor)) ret.Path = devicePath + "/" + deviceName + strIdx return &ret } @@ -174,7 +174,7 @@ func getdevice(devmap map[uint]IndexDevice, cudaImage image.CUDA) []specs.LinuxD case "none": return nil default: - dev := generate_dev_from_string(devmap, val, 0) + dev := generate_dev_from_string(devmap, val) if dev != nil { ret = append(ret, *dev) return ret @@ -184,8 +184,8 @@ func getdevice(devmap map[uint]IndexDevice, cudaImage image.CUDA) []specs.LinuxD } } - for mountIdx, v := range devices.List() { - dev := generate_dev_from_string(devmap, v, mountIdx) + for _, v := range devices.List() { + dev := generate_dev_from_string(devmap, v) if dev != nil { ret = append(ret, *dev) }