diff --git a/cli/test/dt_go/build.sh b/cli/test/dt_go/build.sh index 8eb0feefe39bcf323b8188ae8e711d1e7fc97537..a61cd5ab62586134dc61b87bdbcaa1d64d3cfb55 100644 --- a/cli/test/dt_go/build.sh +++ b/cli/test/dt_go/build.sh @@ -18,25 +18,23 @@ set -e umask 077 CUR_DIR=$(dirname "$(readlink -f $0)") TOP_DIR=$(realpath "${CUR_DIR}"/../../..) -RUNTIME_DIR=${TOP_DIR}/CODE/mindxcheckutils -export GOPATH="${TOP_DIR}/CODE/opensource" export PATH="${GOPATH}/bin/;$PATH" export GO111MODULE=on export GONOSUMDB="*" function execute_test() { - cd ${RUNTIME_DIR} + cd ${TOP_DIR} go mod tidy go install github.com/axw/gocov/gocov@v1.0.0 go install github.com/matm/gocov-html@latest go install gotest.tools/gotestsum@latest - if ! (go test -mod=mod -gcflags=all=-l -v -race -coverprofile cov.out ${RUNTIME_DIR} >./$file_input); then + if ! (go test -mod=mod -gcflags=all=-l -v -race -coverprofile cov.out ${TOP_DIR}/... >./$file_input); then echo '****** go test cases error! ******' exit 1 else echo ${file_detail_output} ${GOPATH}/bin/gocov convert cov.out | ${GOPATH}/bin/gocov-html >${file_detail_output} - ${GOPATH}/bin/gotestsum --junitfile "${TOP_DIR}"/test/unit-tests.xml "${RUNTIME_DIR}"/... + ${GOPATH}/bin/gotestsum --junitfile "${TOP_DIR}"/test/unit-tests.xml "${TOP_DIR}"/... fi } diff --git a/runtime/dcmi/dcmi_api_test.go b/runtime/dcmi/dcmi_api_test.go index e609d6fceafc932ba68eec15979f0e08de981056..1a9ef5576d75944a9a396d377fb870c7a038c144 100644 --- a/runtime/dcmi/dcmi_api_test.go +++ b/runtime/dcmi/dcmi_api_test.go @@ -16,9 +16,11 @@ package dcmi import ( + "context" "testing" "github.com/opencontainers/runtime-spec/specs-go" + "huawei.com/npu-exporter/v5/common-utils/hwlog" ) const mockDeviceID = 100 @@ -37,7 +39,7 @@ func (w *mockWorker) ShutDown() { // CreateVDevice create v device func (w *mockWorker) CreateVDevice(_, _ int32, _ string) (int32, error) { - return int32(mockDeviceId), nil + return int32(mockDeviceID), nil } // DestroyVDevice destroy virtual device @@ -50,32 +52,58 @@ func (w *mockWorker) FindDevice(_ int32) (int32, int32, error) { return 0, 0, nil } +// GetProductType gets product type +func (w *mockWorker) GetProductType(cardID, deviceID int32) (string, error) { + return "", nil +} + +// GetChipInfo gets chip info +func (w *mockWorker) GetChipInfo(cardID, deviceID int32) (*ChipInfo, error) { + return &ChipInfo{}, nil +} + +// TestCreateVDevice tests the function CreateVDevice func TestCreateVDevice(t *testing.T) { t.Log("TestCreateVDevice start") process := specs.Process{} spec := specs.Spec{Process: &process} spec.Process.Env = []string{} + var deviceIdList []int + backups := 2 + logMaxAge := 365 + fileMaxSize := 2 + runLogConfig := hwlog.LogConfig{ + LogFileName: "./test/run.log", + LogLevel: 0, + MaxBackups: backups, + FileMaxSize: fileMaxSize, + MaxAge: logMaxAge, + } + if err := hwlog.InitRunLogger(&runLogConfig, context.Background()); err != nil { + t.Fatalf("hwlog init failed, error is %v", err) + } // no split, all ok - vdevice, err := CreateVDevice(&mockWorker{}, &spec) + vdevice, err := CreateVDevice(&mockWorker{}, &spec, deviceIdList) if err != nil { t.Fatalf("%v %v", vdevice, err) } // no npu assigin for split spec.Process.Env = []string{"ASCEND_VNPU_SPECS=vir04"} - vdevice, err = CreateVDevice(&mockWorker{}, &spec) + vdevice, err = CreateVDevice(&mockWorker{}, &spec, deviceIdList) if err == nil { t.Fatalf("%v %v", vdevice, err) } // split ok + deviceIdList = []int{0} spec.Process.Env = []string{"ASCEND_VNPU_SPECS=vir04", "ASCEND_VISIBLE_DEVICES=0"} - vdevice, err = CreateVDevice(&mockWorker{}, &spec) + vdevice, err = CreateVDevice(&mockWorker{}, &spec, deviceIdList) if err != nil { t.Fatalf("%v %v", vdevice, err) } - if vdevice.VdeviceID != mockDeviceId { + if vdevice.VdeviceID != mockDeviceID { t.Fatalf("%v %v", vdevice, err) } diff --git a/runtime/main.go b/runtime/main.go index f3dc0e7969daea81725d01cb1872df444e77e7d7..0f341fd678a05726f886579ae7e6b286fcbbfb28 100644 --- a/runtime/main.go +++ b/runtime/main.go @@ -17,664 +17,17 @@ package main import ( "context" - "encoding/json" "fmt" - "io/ioutil" "log" "os" - "os/exec" - "path" - "path/filepath" - "regexp" - "sort" - "strconv" "strings" - "syscall" - "github.com/containerd/containerd/oci" - "github.com/opencontainers/runtime-spec/specs-go" "huawei.com/npu-exporter/v5/common-utils/hwlog" "ascend-docker-runtime/mindxcheckutils" - "ascend-docker-runtime/runtime/dcmi" + "ascend-docker-runtime/runtime/process" ) -const ( - runLogPath = "/var/log/ascend-docker-runtime/runtime-run.log" - hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook" - - maxCommandLength = 65535 - hookCli = "ascend-docker-hook" - destroyHookCli = "ascend-docker-destroy" - dockerRuncFile = "docker-runc" - runcFile = "runc" - envLength = 2 - kvPairSize = 2 - borderNum = 2 - - // ENV for device-plugin to identify ascend-docker-runtime - useAscendDocker = "ASCEND_DOCKER_RUNTIME=True" - devicePlugin = "ascend-device-plugin" - ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES" - ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS" - - // void indicates that the NPU card does not need to be mounted - void = "void" -) - -var ( - hookCliPath = hookCli - hookDefaultFile = hookDefaultFilePath - dockerRuncName = dockerRuncFile - runcName = runcFile -) - -const ( - // Atlas200ISoc Product name - Atlas200ISoc = "Atlas 200I SoC A1" - // Atlas200 Product name - Atlas200 = "Atlas 200 Model 3000" - // Ascend310 ascend 310 chip - Ascend310 = "Ascend310" - // Ascend310P ascend 310P chip - Ascend310P = "Ascend310P" - // Ascend310B ascend 310B chip - Ascend310B = "Ascend310B" - // Ascend910 ascend 910 chip - Ascend910 = "Ascend910" - ascend = "Ascend" - - devicePath = "/dev/" - davinciName = "davinci" - virtualDavinciName = "vdavinci" - davinciManager = "davinci_manager" - davinciManagerDocker = "davinci_manager_docker" - notRenameDeviceType = "" - devmmSvm = "devmm_svm" - hisiHdc = "hisi_hdc" - svm0 = "svm0" - tsAisle = "ts_aisle" - upgrade = "upgrade" - sys = "sys" - vdec = "vdec" - vpc = "vpc" - pngd = "pngd" - venc = "venc" - dvppCmdList = "dvpp_cmdlist" - logDrv = "log_drv" - acodec = "acodec" - ai = "ai" - ao = "ao" - vo = "vo" - hdmi = "hdmi" -) - -type args struct { - bundleDirPath string - cmd string -} - -// GetDeviceTypeByChipName get device type by chipName -func GetDeviceTypeByChipName(chipName string) string { - if strings.Contains(chipName, "310B") { - return Ascend310B - } - if strings.Contains(chipName, "310P") { - return Ascend310P - } - if strings.Contains(chipName, "310") { - return Ascend310 - } - if strings.Contains(chipName, "910") { - return Ascend910 - } - return "" -} - -func getArgs() (*args, error) { - args := &args{} - - for i, param := range os.Args { - if param == "--bundle" || param == "-b" { - if len(os.Args)-i <= 1 { - return nil, fmt.Errorf("bundle option needs an argument") - } - args.bundleDirPath = os.Args[i+1] - } else if param == "create" { - args.cmd = param - } - } - - return args, nil -} - -func initLogModule(ctx context.Context) error { - const backups = 2 - const logMaxAge = 365 - const fileMaxSize = 2 - runLogConfig := hwlog.LogConfig{ - LogFileName: runLogPath, - LogLevel: 0, - MaxBackups: backups, - MaxAge: logMaxAge, - OnlyToFile: true, - FileMaxSize: fileMaxSize, - } - if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { - fmt.Printf("hwlog init failed, error is %v", err) - return err - } - return nil -} - -var execRunc = func() error { - tempRuncPath, err := exec.LookPath(dockerRuncName) - if err != nil { - tempRuncPath, err = exec.LookPath(runcName) - if err != nil { - return fmt.Errorf("failed to find the path of runc: %v", err) - } - } - runcPath, err := filepath.EvalSymlinks(tempRuncPath) - if err != nil { - return fmt.Errorf("failed to find realpath of runc %v", err) - } - if _, err := mindxcheckutils.RealFileChecker(runcPath, true, false, mindxcheckutils.DefaultSize); err != nil { - return err - } - - if err := mindxcheckutils.ChangeRuntimeLogMode("runtime-run-"); err != nil { - return err - } - if err = syscall.Exec(runcPath, append([]string{runcPath}, os.Args[1:]...), os.Environ()); err != nil { - return fmt.Errorf("failed to exec runc: %v", err) - } - - return nil -} - -func addAscendDockerEnv(spec *specs.Spec) { - if spec == nil || spec.Process == nil || spec.Process.Env == nil { - return - } - spec.Process.Env = append(spec.Process.Env, useAscendDocker) -} - -func addHook(spec *specs.Spec, deviceIdList *[]int) error { - if deviceIdList == nil { - return nil - } - currentExecPath, err := os.Executable() - if err != nil { - return fmt.Errorf("cannot get the path of ascend-docker-runtime: %v", err) - } - - hookCliPath = path.Join(path.Dir(currentExecPath), hookCli) - if _, err := mindxcheckutils.RealFileChecker(hookCliPath, true, false, mindxcheckutils.DefaultSize); err != nil { - return err - } - if _, err = os.Stat(hookCliPath); err != nil { - return fmt.Errorf("cannot find ascend-docker-hook executable file at %s: %v", hookCliPath, err) - } - - if spec.Hooks == nil { - spec.Hooks = &specs.Hooks{} - } - - needUpdate := true - if len(spec.Hooks.Prestart) > maxCommandLength { - return fmt.Errorf("too many items in Prestart ") - } - for _, hook := range spec.Hooks.Prestart { - if strings.Contains(hook.Path, hookCli) { - needUpdate = false - break - } - } - if needUpdate { - spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{ - Path: hookCliPath, - Args: []string{hookCliPath}, - }) - } - - if len(spec.Process.Env) > maxCommandLength { - return fmt.Errorf("too many items in Env ") - } - - if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { - return nil - } - - vdevice, err := dcmi.CreateVDevice(&dcmi.NpuWorker{}, spec, *deviceIdList) - if err != nil { - return err - } - hwlog.RunLog.Infof("vnpu split done: vdevice: %v", vdevice.VdeviceID) - - if vdevice.VdeviceID != -1 { - updateEnvAndPostHook(spec, vdevice, deviceIdList) - } - - return nil -} - -func removeDuplication(devices []int) []int { - list := make([]int, 0, len(devices)) - prev := -1 - - for _, device := range devices { - if device == prev { - continue - } - - list = append(list, device) - prev = device - } - - return list -} - -func parseDevices(visibleDevices string) ([]int, error) { - devices := make([]int, 0) - const maxDevice = 128 - - for _, d := range strings.Split(visibleDevices, ",") { - d = strings.TrimSpace(d) - if strings.Contains(d, "-") { - borders := strings.Split(d, "-") - if len(borders) != borderNum { - return nil, fmt.Errorf("invalid device range: %s", d) - } - - borders[0] = strings.TrimSpace(borders[0]) - borders[1] = strings.TrimSpace(borders[1]) - - left, err := strconv.Atoi(borders[0]) - if err != nil || left < 0 { - return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0]) - } - - right, err := strconv.Atoi(borders[1]) - if err != nil || right > maxDevice { - return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1]) - } - - if left > right { - return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right) - } - - for n := left; n <= right; n++ { - devices = append(devices, n) - } - } else { - n, err := strconv.Atoi(d) - if err != nil { - return nil, fmt.Errorf("invalid single device parameter: %s", d) - } - - devices = append(devices, n) - } - } - - sort.Slice(devices, func(i, j int) bool { return i < j }) - return removeDuplication(devices), nil -} - -func parseAscendDevices(visibleDevices string) ([]int, error) { - devicesList := strings.Split(visibleDevices, ",") - devices := make([]int, 0, len(devicesList)) - chipType := "" - - for _, d := range devicesList { - matchGroups := regexp.MustCompile(`^Ascend(910|310|310B|310P)-(\d+)$`).FindStringSubmatch(strings.TrimSpace(d)) - if matchGroups == nil { - return nil, fmt.Errorf("invalid device format: %s", d) - } - n, err := strconv.Atoi(matchGroups[2]) - if err != nil { - return nil, fmt.Errorf("invalid device id: %s", d) - } - - if chipType == "" { - chipType = matchGroups[1] - } - if chipType != "" && chipType != matchGroups[1] { - return nil, fmt.Errorf("invalid device chip type: %s", d) - } - - devices = append(devices, n) - - } - chipName, err := dcmi.GetChipName() - if err != nil { - return nil, fmt.Errorf("get chip name error: %v", err) - } - if ascend+chipType != GetDeviceTypeByChipName(chipName) { - return nil, fmt.Errorf("chip type not match really: %s", chipType) - } - - sort.Slice(devices, func(i, j int) bool { return i < j }) - return removeDuplication(devices), nil -} - -func getValueByKey(data []string, name string) string { - for _, envLine := range data { - words := strings.SplitN(envLine, "=", kvPairSize) - if len(words) != kvPairSize { - hwlog.RunLog.Error("environment error") - return "" - } - - if words[0] == name { - return words[1] - } - } - - return "" -} - -func getValueByDeviceKey(data []string) string { - res := "" - for i := len(data) - 1; i >= 0; i-- { - words := strings.SplitN(data[i], "=", kvPairSize) - if len(words) != kvPairSize { - hwlog.RunLog.Error("environment error") - return "" - } - - if words[0] == ascendVisibleDevices { - res = words[1] - break - } - } - if res == "" { - hwlog.RunLog.Error("ASCEND_VISIBLE_DEVICES env variable is empty, will not mount any ascend device") - } - return res -} - -func addDeviceToSpec(spec *specs.Spec, dPath string, deviceType string) error { - device, err := oci.DeviceFromPath(dPath) - if err != nil { - return fmt.Errorf("failed to get %s info : %#v", dPath, err) - } - - switch deviceType { - case virtualDavinciName: - vDeviceNumber := regexp.MustCompile("[0-9]+").FindAllString(dPath, -1) - if len(vDeviceNumber) != 1 { - return fmt.Errorf("invalid vdavinci path: %s", dPath) - } - device.Path = devicePath + davinciName + vDeviceNumber[0] - case davinciManagerDocker: - device.Path = devicePath + davinciManager - default: // do nothing - - } - - spec.Linux.Devices = append(spec.Linux.Devices, *device) - newDeviceCgroup := specs.LinuxDeviceCgroup{ - Allow: true, - Type: device.Type, - Major: &device.Major, - Minor: &device.Minor, - Access: "rwm", - } - spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, newDeviceCgroup) - return nil -} - -func addAscend310BManagerDevice(spec *specs.Spec) error { - var Ascend310BManageDevices = []string{ - svm0, - tsAisle, - upgrade, - sys, - vdec, - vpc, - pngd, - venc, - dvppCmdList, - logDrv, - acodec, - ai, - ao, - vo, - hdmi, - } - - for _, device := range Ascend310BManageDevices { - dPath := devicePath + device - if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { - hwlog.RunLog.Warnf("failed to add %s to spec : %#v", dPath, err) - } - } - - davinciManagerPath := devicePath + davinciManagerDocker - if _, err := os.Stat(davinciManagerPath); err != nil { - hwlog.RunLog.Warnf("failed to get davinci manager docker, err: %#v", err) - davinciManagerPath = devicePath + davinciManager - if _, err := os.Stat(davinciManagerPath); err != nil { - return fmt.Errorf("failed to get davinci manager, err: %#v", err) - } - } - return addDeviceToSpec(spec, davinciManagerPath, davinciManagerDocker) -} - -func addCommonManagerDevice(spec *specs.Spec) error { - var commonManagerDevices = []string{ - devmmSvm, - hisiHdc, - } - - for _, device := range commonManagerDevices { - dPath := devicePath + device - if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { - return fmt.Errorf("failed to add common manage device to spec : %#v", err) - } - } - - return nil -} - -func addManagerDevice(spec *specs.Spec) error { - chipName, err := dcmi.GetChipName() - if err != nil { - return fmt.Errorf("get chip name error: %#v", err) - } - devType := GetDeviceTypeByChipName(chipName) - hwlog.RunLog.Infof("device type is: %s", devType) - if devType == Ascend310B { - return addAscend310BManagerDevice(spec) - } - - if err := addDeviceToSpec(spec, devicePath+davinciManager, notRenameDeviceType); err != nil { - return fmt.Errorf("add davinci_manager to spec error: %#v", err) - } - - productType, err := dcmi.GetProductType(&dcmi.NpuWorker{}) - if err != nil { - return fmt.Errorf("parse product type error: %#v", err) - } - hwlog.RunLog.Infof("product type is %s", productType) - - switch productType { - // do nothing - case Atlas200ISoc, Atlas200: - default: - if err = addCommonManagerDevice(spec); err != nil { - return fmt.Errorf("add common manage device error: %#v", err) - } - } - - return nil -} - -func checkVisibleDevice(spec *specs.Spec) ([]int, error) { - visibleDevices := getValueByDeviceKey(spec.Process.Env) - if visibleDevices == "" || visibleDevices == void { - return nil, nil - } - - if strings.Contains(visibleDevices, ascend) { - devices, err := parseAscendDevices(visibleDevices) - if err != nil { - return nil, fmt.Errorf("failed to parse ascend device : %v", err) - } - hwlog.RunLog.Infof("ascend devices is: %v", devices) - return devices, err - } - devices, err := parseDevices(visibleDevices) - if err != nil { - return nil, fmt.Errorf("failed to parse device : %v", err) - } - hwlog.RunLog.Infof("devices is: %v", devices) - return devices, err -} - -func addDevice(spec *specs.Spec, deviceIdList []int) error { - deviceName := davinciName - if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { - deviceName = virtualDavinciName - } - for _, deviceId := range deviceIdList { - dPath := devicePath + deviceName + strconv.Itoa(deviceId) - if err := addDeviceToSpec(spec, dPath, deviceName); err != nil { - return fmt.Errorf("failed to add davinci device to spec: %v", err) - } - } - - if err := addManagerDevice(spec); err != nil { - return fmt.Errorf("failed to add Manager device to spec: %v", err) - } - - return nil -} - -func updateEnvAndPostHook(spec *specs.Spec, vdevice dcmi.VDeviceInfo, deviceIdList *[]int) { - if deviceIdList == nil { - return - } - newEnv := make([]string, 0, len(spec.Process.Env)+1) - needAddVirtualFlag := true - *deviceIdList = []int{int(vdevice.VdeviceID)} - for _, line := range spec.Process.Env { - words := strings.Split(line, "=") - if len(words) == envLength && strings.TrimSpace(words[0]) == ascendRuntimeOptions { - needAddVirtualFlag = false - if strings.Contains(words[1], "VIRTUAL") { - newEnv = append(newEnv, line) - continue - } else { - newEnv = append(newEnv, strings.TrimSpace(line)+",VIRTUAL") - continue - } - } - newEnv = append(newEnv, line) - } - if needAddVirtualFlag { - newEnv = append(newEnv, fmt.Sprintf("ASCEND_RUNTIME_OPTIONS=VIRTUAL")) - } - spec.Process.Env = newEnv - if currentExecPath, err := os.Executable(); err == nil { - postHookCliPath := path.Join(path.Dir(currentExecPath), destroyHookCli) - spec.Hooks.Poststop = append(spec.Hooks.Poststop, specs.Hook{ - Path: postHookCliPath, - Args: []string{postHookCliPath, fmt.Sprintf("%d", vdevice.CardID), fmt.Sprintf("%d", vdevice.DeviceID), - fmt.Sprintf("%d", vdevice.VdeviceID)}, - }) - } -} - -func modifySpecFile(path string) error { - stat, err := os.Stat(path) - if err != nil { - return fmt.Errorf("spec file doesnt exist %s: %v", path, err) - } - if _, err = mindxcheckutils.RealFileChecker(path, true, true, mindxcheckutils.DefaultSize); err != nil { - return err - } - - jsonFile, err := os.OpenFile(path, os.O_RDWR, stat.Mode()) - if err != nil { - return fmt.Errorf("cannot open oci spec file %s: %v", path, err) - } - - defer jsonFile.Close() - - jsonContent, err := ioutil.ReadAll(jsonFile) - if err != nil { - return fmt.Errorf("failed to read oci spec file %s: %v", path, err) - } - - if err = jsonFile.Truncate(0); err != nil { - return fmt.Errorf("failed to truncate: %v", err) - } - if _, err = jsonFile.Seek(0, 0); err != nil { - return fmt.Errorf("failed to seek: %v", err) - } - - var spec specs.Spec - if err = json.Unmarshal(jsonContent, &spec); err != nil { - return fmt.Errorf("failed to unmarshal oci spec file %s: %v", path, err) - } - - devices, err := checkVisibleDevice(&spec) - if err != nil { - hwlog.RunLog.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) - return fmt.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) - } - if len(devices) != 0 { - if err = addHook(&spec, &devices); err != nil { - hwlog.RunLog.Errorf("failed to inject hook, err: %v", err) - return fmt.Errorf("failed to inject hook, err: %v", err) - } - if err = addDevice(&spec, devices); err != nil { - return fmt.Errorf("failed to add device to env: %v", err) - } - } - - addAscendDockerEnv(&spec) - - jsonOutput, err := json.Marshal(spec) - if err != nil { - return fmt.Errorf("failed to marshal OCI spec file: %v", err) - } - - if _, err = jsonFile.WriteAt(jsonOutput, 0); err != nil { - return fmt.Errorf("failed to write OCI spec file: %v", err) - } - - return nil -} - -func doProcess() error { - args, err := getArgs() - if err != nil { - return fmt.Errorf("failed to get args: %v", err) - } - - if args.cmd != "create" { - return execRunc() - } - - if args.bundleDirPath == "" { - args.bundleDirPath, err = os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current working dir: %v", err) - } - } - - specFilePath := args.bundleDirPath + "/config.json" - - if err = modifySpecFile(specFilePath); err != nil { - return fmt.Errorf("failed to modify spec file %s: %v", specFilePath, err) - } - - return execRunc() -} - func main() { defer func() { if err := recover(); err != nil { @@ -682,7 +35,7 @@ func main() { } }() ctx, _ := context.WithCancel(context.Background()) - if err := initLogModule(ctx); err != nil { + if err := process.InitLogModule(ctx); err != nil { log.Fatal(err) } logPrefixWords, err := mindxcheckutils.GetLogPrefix() @@ -695,11 +48,11 @@ func main() { } }() if !mindxcheckutils.StringChecker(strings.Join(os.Args, " "), 0, - maxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { + process.MaxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { hwlog.RunLog.Errorf("%v ascend docker runtime args check failed", logPrefixWords) log.Fatal("command error") } - if err = doProcess(); err != nil { + if err = process.DoProcess(); err != nil { hwlog.RunLog.Errorf("%v docker runtime failed: %v", logPrefixWords, err) log.Fatal(err) } diff --git a/runtime/process/process.go b/runtime/process/process.go new file mode 100644 index 0000000000000000000000000000000000000000..d38d7656c240c47bc567963bfa34fd9873572f26 --- /dev/null +++ b/runtime/process/process.go @@ -0,0 +1,688 @@ +/* Copyright(C) 2022. Huawei Technologies Co.,Ltd. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package process does what ascend-docker-runtime is supposed to do before runc being executed. +package process + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "syscall" + + "github.com/containerd/containerd/oci" + "github.com/opencontainers/runtime-spec/specs-go" + "huawei.com/npu-exporter/v5/common-utils/hwlog" + + "ascend-docker-runtime/mindxcheckutils" + "ascend-docker-runtime/runtime/dcmi" +) + +const ( + runLogPath = "/var/log/ascend-docker-runtime/runtime-run.log" + hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook" + // MaxCommandLength is the max length of command. + MaxCommandLength = 65535 + hookCli = "ascend-docker-hook" + destroyHookCli = "ascend-docker-destroy" + dockerRuncFile = "docker-runc" + runcFile = "runc" + envLength = 2 + kvPairSize = 2 + borderNum = 2 + + // ENV for device-plugin to identify ascend-docker-runtime + useAscendDocker = "ASCEND_DOCKER_RUNTIME=True" + devicePlugin = "ascend-device-plugin" + ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES" + ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS" + + // void indicates that the NPU card does not need to be mounted + void = "void" +) + +var ( + hookCliPath = hookCli + hookDefaultFile = hookDefaultFilePath + dockerRuncName = dockerRuncFile + runcName = runcFile +) + +const ( + // Atlas200ISoc Product name + Atlas200ISoc = "Atlas 200I SoC A1" + // Atlas200 Product name + Atlas200 = "Atlas 200 Model 3000" + // Ascend310 ascend 310 chip + Ascend310 = "Ascend310" + // Ascend310P ascend 310P chip + Ascend310P = "Ascend310P" + // Ascend310B ascend 310B chip + Ascend310B = "Ascend310B" + // Ascend910 ascend 910 chip + Ascend910 = "Ascend910" + ascend = "Ascend" + + devicePath = "/dev/" + davinciName = "davinci" + virtualDavinciName = "vdavinci" + davinciManager = "davinci_manager" + davinciManagerDocker = "davinci_manager_docker" + notRenameDeviceType = "" + devmmSvm = "devmm_svm" + hisiHdc = "hisi_hdc" + svm0 = "svm0" + tsAisle = "ts_aisle" + upgrade = "upgrade" + sys = "sys" + vdec = "vdec" + vpc = "vpc" + pngd = "pngd" + venc = "venc" + dvppCmdList = "dvpp_cmdlist" + logDrv = "log_drv" + acodec = "acodec" + ai = "ai" + ao = "ao" + vo = "vo" + hdmi = "hdmi" +) + +type args struct { + bundleDirPath string + cmd string +} + +// GetDeviceTypeByChipName get device type by chipName +func GetDeviceTypeByChipName(chipName string) string { + if strings.Contains(chipName, "310B") { + return Ascend310B + } + if strings.Contains(chipName, "310P") { + return Ascend310P + } + if strings.Contains(chipName, "310") { + return Ascend310 + } + if strings.Contains(chipName, "910") { + return Ascend910 + } + return "" +} + +func getArgs() (*args, error) { + args := &args{} + + for i, param := range os.Args { + if param == "--bundle" || param == "-b" { + if len(os.Args)-i <= 1 { + return nil, fmt.Errorf("bundle option needs an argument") + } + args.bundleDirPath = os.Args[i+1] + } else if param == "create" { + args.cmd = param + } + } + + return args, nil +} + +// InitLogModule initializes some logging configuration. +func InitLogModule(ctx context.Context) error { + const backups = 2 + const logMaxAge = 365 + const fileMaxSize = 2 + runLogConfig := hwlog.LogConfig{ + LogFileName: runLogPath, + LogLevel: 0, + MaxBackups: backups, + MaxAge: logMaxAge, + OnlyToFile: true, + FileMaxSize: fileMaxSize, + } + if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { + fmt.Printf("hwlog init failed, error is %v", err) + return err + } + return nil +} + +var execRunc = func() error { + tempRuncPath, err := exec.LookPath(dockerRuncName) + if err != nil { + tempRuncPath, err = exec.LookPath(runcName) + if err != nil { + return fmt.Errorf("failed to find the path of runc: %v", err) + } + } + runcPath, err := filepath.EvalSymlinks(tempRuncPath) + if err != nil { + return fmt.Errorf("failed to find realpath of runc %v", err) + } + if _, err := mindxcheckutils.RealFileChecker(runcPath, true, false, mindxcheckutils.DefaultSize); err != nil { + return err + } + + if err := mindxcheckutils.ChangeRuntimeLogMode("runtime-run-"); err != nil { + return err + } + if err = syscall.Exec(runcPath, append([]string{runcPath}, os.Args[1:]...), os.Environ()); err != nil { + return fmt.Errorf("failed to exec runc: %v", err) + } + + return nil +} + +func addAscendDockerEnv(spec *specs.Spec) { + if spec == nil || spec.Process == nil || spec.Process.Env == nil { + return + } + spec.Process.Env = append(spec.Process.Env, useAscendDocker) +} + +func addHook(spec *specs.Spec, deviceIdList *[]int) error { + if deviceIdList == nil { + return nil + } + currentExecPath, err := os.Executable() + if err != nil { + return fmt.Errorf("cannot get the path of ascend-docker-runtime: %v", err) + } + + hookCliPath = path.Join(path.Dir(currentExecPath), hookCli) + if _, err := mindxcheckutils.RealFileChecker(hookCliPath, true, false, mindxcheckutils.DefaultSize); err != nil { + return err + } + if _, err = os.Stat(hookCliPath); err != nil { + return fmt.Errorf("cannot find ascend-docker-hook executable file at %s: %v", hookCliPath, err) + } + + if spec.Hooks == nil { + spec.Hooks = &specs.Hooks{} + } + + needUpdate := true + if len(spec.Hooks.Prestart) > MaxCommandLength { + return fmt.Errorf("too many items in Prestart ") + } + for _, hook := range spec.Hooks.Prestart { + if strings.Contains(hook.Path, hookCli) { + needUpdate = false + break + } + } + if needUpdate { + spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{ + Path: hookCliPath, + Args: []string{hookCliPath}, + }) + } + + if len(spec.Process.Env) > MaxCommandLength { + return fmt.Errorf("too many items in Env ") + } + + if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { + return nil + } + + vdevice, err := dcmi.CreateVDevice(&dcmi.NpuWorker{}, spec, *deviceIdList) + if err != nil { + return err + } + hwlog.RunLog.Infof("vnpu split done: vdevice: %v", vdevice.VdeviceID) + + if vdevice.VdeviceID != -1 { + updateEnvAndPostHook(spec, vdevice, deviceIdList) + } + + return nil +} + +func removeDuplication(devices []int) []int { + list := make([]int, 0, len(devices)) + prev := -1 + + for _, device := range devices { + if device == prev { + continue + } + + list = append(list, device) + prev = device + } + + return list +} + +func parseDevices(visibleDevices string) ([]int, error) { + devices := make([]int, 0) + + for _, value := range strings.Split(visibleDevices, ",") { + deviceFromValue, err := getDeviceListFromVisibleValue(value) + if err != nil { + hwlog.RunLog.Errorf("failed to get devices from ASCEND_VISIBLE_DEVICE value, error: %v", err) + return nil, err + } + devices = append(devices, deviceFromValue...) + } + + sort.Slice(devices, func(i, j int) bool { return i < j }) + return removeDuplication(devices), nil +} + +func getDeviceListFromVisibleValue(visibleValue string) ([]int, error) { + maxDevice := 128 + devices := make([]int, 0) + visibleValue = strings.TrimSpace(visibleValue) + if strings.Contains(visibleValue, "-") { + borders := strings.Split(visibleValue, "-") + if len(borders) != borderNum { + return nil, fmt.Errorf("invalid device range: %s", visibleValue) + } + + borders[0] = strings.TrimSpace(borders[0]) + borders[1] = strings.TrimSpace(borders[1]) + + left, err := strconv.Atoi(borders[0]) + if err != nil || left < 0 { + return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0]) + } + + right, err := strconv.Atoi(borders[1]) + if err != nil || right > maxDevice { + return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1]) + } + + if left > right { + return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right) + } + + for n := left; n <= right; n++ { + devices = append(devices, n) + } + return devices, nil + } + n, err := strconv.Atoi(visibleValue) + if err != nil { + return nil, fmt.Errorf("invalid single device parameter: %s", visibleValue) + } + devices = append(devices, n) + + return devices, nil +} + +func parseAscendDevices(visibleDevices string) ([]int, error) { + devicesList := strings.Split(visibleDevices, ",") + devices := make([]int, 0, len(devicesList)) + chipType := "" + + for _, d := range devicesList { + matchGroups := regexp.MustCompile(`^Ascend(910|310|310B|310P)-(\d+)$`).FindStringSubmatch(strings.TrimSpace(d)) + if matchGroups == nil { + return nil, fmt.Errorf("invalid device format: %s", d) + } + n, err := strconv.Atoi(matchGroups[2]) + if err != nil { + return nil, fmt.Errorf("invalid device id: %s", d) + } + + if chipType == "" { + chipType = matchGroups[1] + } + if chipType != "" && chipType != matchGroups[1] { + return nil, fmt.Errorf("invalid device chip type: %s", d) + } + + devices = append(devices, n) + + } + chipName, err := dcmi.GetChipName() + if err != nil { + return nil, fmt.Errorf("get chip name error: %v", err) + } + if ascend+chipType != GetDeviceTypeByChipName(chipName) { + return nil, fmt.Errorf("chip type not match really: %s", chipType) + } + + sort.Slice(devices, func(i, j int) bool { return i < j }) + return removeDuplication(devices), nil +} + +func getValueByKey(data []string, name string) string { + for _, envLine := range data { + words := strings.SplitN(envLine, "=", kvPairSize) + if len(words) != kvPairSize { + hwlog.RunLog.Error("environment error") + return "" + } + + if words[0] == name { + return words[1] + } + } + + return "" +} + +func getValueByDeviceKey(data []string) string { + res := "" + for i := len(data) - 1; i >= 0; i-- { + words := strings.SplitN(data[i], "=", kvPairSize) + if len(words) != kvPairSize { + hwlog.RunLog.Error("environment error") + return "" + } + + if words[0] == ascendVisibleDevices { + res = words[1] + break + } + } + if res == "" { + hwlog.RunLog.Error("ASCEND_VISIBLE_DEVICES env variable is empty, will not mount any ascend device") + } + return res +} + +func addDeviceToSpec(spec *specs.Spec, dPath string, deviceType string) error { + device, err := oci.DeviceFromPath(dPath) + if err != nil { + return fmt.Errorf("failed to get %s info : %#v", dPath, err) + } + + switch deviceType { + case virtualDavinciName: + vDeviceNumber := regexp.MustCompile("[0-9]+").FindAllString(dPath, -1) + if len(vDeviceNumber) != 1 { + return fmt.Errorf("invalid vdavinci path: %s", dPath) + } + device.Path = devicePath + davinciName + vDeviceNumber[0] + case davinciManagerDocker: + device.Path = devicePath + davinciManager + default: // do nothing + + } + + spec.Linux.Devices = append(spec.Linux.Devices, *device) + newDeviceCgroup := specs.LinuxDeviceCgroup{ + Allow: true, + Type: device.Type, + Major: &device.Major, + Minor: &device.Minor, + Access: "rwm", + } + spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, newDeviceCgroup) + return nil +} + +func addAscend310BManagerDevice(spec *specs.Spec) error { + var Ascend310BManageDevices = []string{ + svm0, + tsAisle, + upgrade, + sys, + vdec, + vpc, + pngd, + venc, + dvppCmdList, + logDrv, + acodec, + ai, + ao, + vo, + hdmi, + } + + for _, device := range Ascend310BManageDevices { + dPath := devicePath + device + if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { + hwlog.RunLog.Warnf("failed to add %s to spec : %#v", dPath, err) + } + } + + davinciManagerPath := devicePath + davinciManagerDocker + if _, err := os.Stat(davinciManagerPath); err != nil { + hwlog.RunLog.Warnf("failed to get davinci manager docker, err: %#v", err) + davinciManagerPath = devicePath + davinciManager + if _, err := os.Stat(davinciManagerPath); err != nil { + return fmt.Errorf("failed to get davinci manager, err: %#v", err) + } + } + return addDeviceToSpec(spec, davinciManagerPath, davinciManagerDocker) +} + +func addCommonManagerDevice(spec *specs.Spec) error { + var commonManagerDevices = []string{ + devmmSvm, + hisiHdc, + } + + for _, device := range commonManagerDevices { + dPath := devicePath + device + if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { + return fmt.Errorf("failed to add common manage device to spec : %#v", err) + } + } + + return nil +} + +func addManagerDevice(spec *specs.Spec) error { + chipName, err := dcmi.GetChipName() + if err != nil { + return fmt.Errorf("get chip name error: %#v", err) + } + devType := GetDeviceTypeByChipName(chipName) + hwlog.RunLog.Infof("device type is: %s", devType) + if devType == Ascend310B { + return addAscend310BManagerDevice(spec) + } + + if err := addDeviceToSpec(spec, devicePath+davinciManager, notRenameDeviceType); err != nil { + return fmt.Errorf("add davinci_manager to spec error: %#v", err) + } + + productType, err := dcmi.GetProductType(&dcmi.NpuWorker{}) + if err != nil { + return fmt.Errorf("parse product type error: %#v", err) + } + hwlog.RunLog.Infof("product type is %s", productType) + + switch productType { + // do nothing + case Atlas200ISoc, Atlas200: + default: + if err = addCommonManagerDevice(spec); err != nil { + return fmt.Errorf("add common manage device error: %#v", err) + } + } + + return nil +} + +func checkVisibleDevice(spec *specs.Spec) ([]int, error) { + visibleDevices := getValueByDeviceKey(spec.Process.Env) + if visibleDevices == "" || visibleDevices == void { + return nil, nil + } + + if strings.Contains(visibleDevices, ascend) { + devices, err := parseAscendDevices(visibleDevices) + if err != nil { + return nil, fmt.Errorf("failed to parse ascend device : %v", err) + } + hwlog.RunLog.Infof("ascend devices is: %v", devices) + return devices, err + } + devices, err := parseDevices(visibleDevices) + if err != nil { + return nil, fmt.Errorf("failed to parse device : %v", err) + } + hwlog.RunLog.Infof("devices is: %v", devices) + return devices, err +} + +func addDevice(spec *specs.Spec, deviceIdList []int) error { + deviceName := davinciName + if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { + deviceName = virtualDavinciName + } + for _, deviceId := range deviceIdList { + dPath := devicePath + deviceName + strconv.Itoa(deviceId) + if err := addDeviceToSpec(spec, dPath, deviceName); err != nil { + return fmt.Errorf("failed to add davinci device to spec: %v", err) + } + } + + if err := addManagerDevice(spec); err != nil { + return fmt.Errorf("failed to add Manager device to spec: %v", err) + } + + return nil +} + +func updateEnvAndPostHook(spec *specs.Spec, vdevice dcmi.VDeviceInfo, deviceIdList *[]int) { + if deviceIdList == nil { + return + } + newEnv := make([]string, 0, len(spec.Process.Env)+1) + needAddVirtualFlag := true + *deviceIdList = []int{int(vdevice.VdeviceID)} + for _, line := range spec.Process.Env { + words := strings.Split(line, "=") + if len(words) == envLength && strings.TrimSpace(words[0]) == ascendRuntimeOptions { + needAddVirtualFlag = false + if strings.Contains(words[1], "VIRTUAL") { + newEnv = append(newEnv, line) + continue + } else { + newEnv = append(newEnv, strings.TrimSpace(line)+",VIRTUAL") + continue + } + } + newEnv = append(newEnv, line) + } + if needAddVirtualFlag { + newEnv = append(newEnv, fmt.Sprintf("ASCEND_RUNTIME_OPTIONS=VIRTUAL")) + } + spec.Process.Env = newEnv + if currentExecPath, err := os.Executable(); err == nil { + postHookCliPath := path.Join(path.Dir(currentExecPath), destroyHookCli) + spec.Hooks.Poststop = append(spec.Hooks.Poststop, specs.Hook{ + Path: postHookCliPath, + Args: []string{postHookCliPath, fmt.Sprintf("%d", vdevice.CardID), fmt.Sprintf("%d", vdevice.DeviceID), + fmt.Sprintf("%d", vdevice.VdeviceID)}, + }) + } +} + +func modifySpecFile(path string) error { + stat, err := os.Stat(path) + if err != nil { + return fmt.Errorf("spec file doesnt exist %s: %v", path, err) + } + if _, err = mindxcheckutils.RealFileChecker(path, true, true, mindxcheckutils.DefaultSize); err != nil { + return err + } + + jsonFile, err := os.OpenFile(path, os.O_RDWR, stat.Mode()) + if err != nil { + return fmt.Errorf("cannot open oci spec file %s: %v", path, err) + } + + defer jsonFile.Close() + + jsonContent, err := ioutil.ReadAll(jsonFile) + if err != nil { + return fmt.Errorf("failed to read oci spec file %s: %v", path, err) + } + + if err = jsonFile.Truncate(0); err != nil { + return fmt.Errorf("failed to truncate: %v", err) + } + if _, err = jsonFile.Seek(0, 0); err != nil { + return fmt.Errorf("failed to seek: %v", err) + } + + var spec specs.Spec + if err = json.Unmarshal(jsonContent, &spec); err != nil { + return fmt.Errorf("failed to unmarshal oci spec file %s: %v", path, err) + } + + devices, err := checkVisibleDevice(&spec) + if err != nil { + hwlog.RunLog.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) + return fmt.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) + } + if len(devices) != 0 { + if err = addHook(&spec, &devices); err != nil { + hwlog.RunLog.Errorf("failed to inject hook, err: %v", err) + return fmt.Errorf("failed to inject hook, err: %v", err) + } + if err = addDevice(&spec, devices); err != nil { + return fmt.Errorf("failed to add device to env: %v", err) + } + } + + addAscendDockerEnv(&spec) + + jsonOutput, err := json.Marshal(spec) + if err != nil { + return fmt.Errorf("failed to marshal OCI spec file: %v", err) + } + + if _, err = jsonFile.WriteAt(jsonOutput, 0); err != nil { + return fmt.Errorf("failed to write OCI spec file: %v", err) + } + + return nil +} + +// DoProcess does what ascend-docker-runtime is supposed to do before runc being executed. +func DoProcess() error { + args, err := getArgs() + if err != nil { + return fmt.Errorf("failed to get args: %v", err) + } + + if args.cmd != "create" { + return execRunc() + } + + if args.bundleDirPath == "" { + args.bundleDirPath, err = os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working dir: %v", err) + } + } + + specFilePath := args.bundleDirPath + "/config.json" + + if err = modifySpecFile(specFilePath); err != nil { + return fmt.Errorf("failed to modify spec file %s: %v", specFilePath, err) + } + + return execRunc() +} diff --git a/runtime/main_test.go b/runtime/process/process_test.go similarity index 76% rename from runtime/main_test.go rename to runtime/process/process_test.go index 6721a4b7a2692ba0fd1f8e8a91eb94ca04c468da..cf852c0eee716db12ebbf40c350fb3b8f1e67abc 100644 --- a/runtime/main_test.go +++ b/runtime/process/process_test.go @@ -12,14 +12,14 @@ limitations under the License. */ -// Package main -package main +package process import ( "context" "fmt" "os" "reflect" + "strings" "testing" "github.com/agiledragon/gomonkey/v2" @@ -39,67 +39,67 @@ const ( fileMode0600 os.FileMode = 0600 fileMode0655 os.FileMode = 0655 needToMkdir = "./test" + fileExistErrorStr = "file exists" + bundleArgStr = "--bundle" + execStubLog = "execute stub" + configPath = "./test/config.json" ) var ( deviceList = []int{1} ) +// TestArgsIsCreate tests the function DoProcess func TestArgsIsCreate(t *testing.T) { - t.Log("进入测试用例") - - testArgs := []string{"create", "--bundle", "."} + testArgs := []string{"create", bundleArgStr, "."} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err := doProcess() + err := DoProcess() assert.NotNil(t, err) } +// TestArgsIsCreateCase1 tests the function DoProcess func TestArgsIsCreateCase1(t *testing.T) { - t.Log("进入测试用例") - - testArgs := []string{"create", "--bundle"} + testArgs := []string{"create", bundleArgStr} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err := doProcess() + err := DoProcess() assert.NotNil(t, err) } +// TestArgsIsCreateCase2 tests the function DoProcess func TestArgsIsCreateCase2(t *testing.T) { - t.Log("进入测试用例") - - testArgs := []string{"create", "--bundle", ""} + testArgs := []string{"create", bundleArgStr, ""} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err := doProcess() + err := DoProcess() assert.NotNil(t, err) } +// TestArgsIsCreateCase3 tests the function DoProcess func TestArgsIsCreateCase3(t *testing.T) { - t.Log("进入测试用例") - - if err := os.Mkdir(needToMkdir, fileMode0655); err != nil { + if err := os.Mkdir(needToMkdir, fileMode0655); err != nil && !strings.Contains(err.Error(), fileExistErrorStr) { t.Fatalf("failed to create file, error: %v", err) } - f, err := os.Create("./test/config.json") + f, err := os.Create(configPath) defer f.Close() if err != nil { t.Logf("create file error: %v", err) @@ -108,26 +108,25 @@ func TestArgsIsCreateCase3(t *testing.T) { if err != nil { t.Logf("chmod file error: %v", err) } - testArgs := []string{"create", "--bundle", needToMkdir} + testArgs := []string{"create", bundleArgStr, needToMkdir} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err = doProcess() + err = DoProcess() assert.NotNil(t, err) } +// TestArgsIsCreateCase4 tests the function DoProcess func TestArgsIsCreateCase4(t *testing.T) { - t.Log("进入测试用例") - - if err := os.Mkdir(needToMkdir, fileMode0655); err != nil { + if err := os.Mkdir(needToMkdir, fileMode0655); err != nil && !strings.Contains(err.Error(), fileExistErrorStr) { t.Fatalf("failed to create file, error: %v", err) } - f, err := os.Create("./test/config.json") + f, err := os.Create(configPath) defer f.Close() if err != nil { t.Logf("create file failed, error: %v", err) @@ -136,36 +135,43 @@ func TestArgsIsCreateCase4(t *testing.T) { if err != nil { t.Logf("chmod file failed, error: %v", err) } - testArgs := []string{"spec", "--bundle", needToMkdir} + testArgs := []string{"spec", bundleArgStr, needToMkdir} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err = doProcess() + err = DoProcess() assert.Nil(t, err) } +// TestModifySpecFile tests the function modifySpecFile func TestModifySpecFile(t *testing.T) { - err := modifySpecFile("./test/config.json") + err := modifySpecFile(configPath) assert.NotNil(t, err) } +// TestModifySpecFileCase1 tests the function modifySpecFile func TestModifySpecFileCase1(t *testing.T) { - if err := os.Mkdir(needToMkdir, fileMode0400); err != nil { + err := InitLogModule(context.Background()) + if err != nil { + t.Logf("init log failed, error: %v", err) + } + if err := os.Mkdir(needToMkdir, fileMode0400); err != nil && !strings.Contains(err.Error(), fileExistErrorStr) { t.Logf("mkdir error: %v", err) } - err := modifySpecFile(needToMkdir) + err = modifySpecFile(needToMkdir) assert.NotNil(t, err) if err := os.Remove(needToMkdir); err != nil { t.Logf("failed to remove dir, error: %v", err) } } +// TestModifySpecFileCase2 tests the function modifySpecFile func TestModifySpecFileCase2(t *testing.T) { file := "./test.json" f, err := os.Create(file) @@ -181,10 +187,11 @@ func TestModifySpecFileCase2(t *testing.T) { t.Log("run modifySpecFile failed") } if err := os.Remove(file); err != nil { - + t.Logf("remove file(%v) failed, error: %v", file, err) } } +// TestModifySpecFileCase3 tests the function modifySpecFile func TestModifySpecFileCase3(t *testing.T) { file := "./test_spec.json" if err := modifySpecFile(file); err != nil { @@ -192,6 +199,7 @@ func TestModifySpecFileCase3(t *testing.T) { } } +// TestAddHook tests the function addHook func TestAddHook(t *testing.T) { var specArgs = &specs.Spec{} if err := addHook(specArgs, &deviceList); err != nil { @@ -199,6 +207,7 @@ func TestAddHook(t *testing.T) { } } +// TestAddHookCase1 tests the function addHook func TestAddHookCase1(t *testing.T) { var specArgs = &specs.Spec{} stub := gomonkey.ApplyGlobalVar(&hookCliPath, ".") @@ -208,6 +217,7 @@ func TestAddHookCase1(t *testing.T) { assert.NotNil(t, err) } +// TestAddHookCase2 tests the function addHook func TestAddHookCase2(t *testing.T) { var specArgs = &specs.Spec{} stub := gomonkey.ApplyGlobalVar(&hookCliPath, ".") @@ -217,6 +227,7 @@ func TestAddHookCase2(t *testing.T) { assert.NotNil(t, err) } +// TestAddHookCase3 tests the function addHook func TestAddHookCase3(t *testing.T) { file := "/usr/local/bin/ascend-docker-hook" filenew := "/usr/local/bin/ascend-docker-hook-1" @@ -233,6 +244,7 @@ func TestAddHookCase3(t *testing.T) { } } +// TestExecRunc tests the function execRunc func TestExecRunc(t *testing.T) { stub := gomonkey.ApplyGlobalVar(&dockerRuncName, "abc-runc") stub.ApplyGlobalVar(&runcName, "runc123") @@ -242,6 +254,7 @@ func TestExecRunc(t *testing.T) { assert.NotNil(t, err) } +// TestParseDevicesCase1 tests the function parseDevices func TestParseDevicesCase1(t *testing.T) { visibleDevices := "0-3,5,7" expectVal := []int{0, 1, 2, 3, 5, 7} @@ -251,42 +264,49 @@ func TestParseDevicesCase1(t *testing.T) { } } +// TestParseDevicesCase2 tests the function parseDevices func TestParseDevicesCase2(t *testing.T) { visibleDevices := "0-3-4,5,7" _, err := parseDevices(visibleDevices) assert.NotNil(t, err) } +// TestParseDevicesCase3 tests the function parseDevices func TestParseDevicesCase3(t *testing.T) { visibleDevices := "0l-3,5,7" _, err := parseDevices(visibleDevices) assert.NotNil(t, err) } +// TestParseDevicesCase4 tests the function parseDevices func TestParseDevicesCase4(t *testing.T) { visibleDevices := "0-3o,5,7" _, err := parseDevices(visibleDevices) assert.NotNil(t, err) } +// TestParseDevicesCase5 tests the function parseDevices func TestParseDevicesCase5(t *testing.T) { visibleDevices := "4-3,5,7" _, err := parseDevices(visibleDevices) assert.NotNil(t, err) } +// TestParseDevicesCase6 tests the function parseDevices func TestParseDevicesCase6(t *testing.T) { visibleDevices := "3o,5,7" _, err := parseDevices(visibleDevices) assert.NotNil(t, err) } +// TestParseDevicesCase7 tests the function parseDevices func TestParseDevicesCase7(t *testing.T) { visibleDevices := "0=3,5,7" _, err := parseDevices(visibleDevices) assert.NotNil(t, err) } +// TestRemoveDuplication tests the function removeDuplication func TestRemoveDuplication(t *testing.T) { originList := []int{1, 2, 2, 4, 5, 5, 5, 6, 8, 8} targetList := []int{1, 2, 4, 5, 6, 8} @@ -295,6 +315,7 @@ func TestRemoveDuplication(t *testing.T) { assert.EqualValues(t, targetList, resultList) } +// TestAddEnvToDevicePlugin0 tests the function addAscendDockerEnv func TestAddEnvToDevicePlugin0(t *testing.T) { devicePluginHostName := devicePlugin + "pf2i6r" spec := specs.Spec{ @@ -309,6 +330,7 @@ func TestAddEnvToDevicePlugin0(t *testing.T) { assert.Contains(t, spec.Process.Env, useAscendDocker) } +// TestAddEnvToDevicePlugin1 tests the function addAscendDockerEnv func TestAddEnvToDevicePlugin1(t *testing.T) { devicePluginHostName := "pf2i6r" spec := specs.Spec{ @@ -323,36 +345,42 @@ func TestAddEnvToDevicePlugin1(t *testing.T) { assert.Contains(t, spec.Process.Env, useAscendDocker) } +// TestGetDeviceTypeByChipName0 tests the function GetDeviceTypeByChipName func TestGetDeviceTypeByChipName0(t *testing.T) { chipName := "310B" devType := GetDeviceTypeByChipName(chipName) assert.EqualValues(t, Ascend310B, devType) } +// TestGetDeviceTypeByChipName1 tests the function GetDeviceTypeByChipName func TestGetDeviceTypeByChipName1(t *testing.T) { chipName := "310P" devType := GetDeviceTypeByChipName(chipName) assert.EqualValues(t, Ascend310P, devType) } +// TestGetDeviceTypeByChipName2 tests the function GetDeviceTypeByChipName func TestGetDeviceTypeByChipName2(t *testing.T) { chipName := "310" devType := GetDeviceTypeByChipName(chipName) assert.EqualValues(t, Ascend310, devType) } +// TestGetDeviceTypeByChipName3 tests the function GetDeviceTypeByChipName func TestGetDeviceTypeByChipName3(t *testing.T) { chipName := "910" devType := GetDeviceTypeByChipName(chipName) assert.EqualValues(t, Ascend910, devType) } +// TestGetDeviceTypeByChipName4 tests the function GetDeviceTypeByChipName func TestGetDeviceTypeByChipName4(t *testing.T) { chipName := "980b" devType := GetDeviceTypeByChipName(chipName) assert.EqualValues(t, "", devType) } +// TestGetValueByKeyCase1 tests the function getValueByKey func TestGetValueByKeyCase1(t *testing.T) { data := []string{"ASCEND_VISIBLE_DEVICES=0-3,5,7"} word := "ASCEND_VISIBLE_DEVICES" @@ -361,6 +389,7 @@ func TestGetValueByKeyCase1(t *testing.T) { assert.EqualValues(t, expectVal, actualVal) } +// TestGetValueByKeyCase2 tests the function getValueByKey func TestGetValueByKeyCase2(t *testing.T) { data := []string{"ASCEND_VISIBLE_DEVICES"} word := "ASCEND_VISIBLE_DEVICES" @@ -374,6 +403,7 @@ func TestGetValueByKeyCase2(t *testing.T) { assert.EqualValues(t, expectVal, actualVal) } +// TestGetValueByKeyCase3 tests the function getValueByKey func TestGetValueByKeyCase3(t *testing.T) { data := []string{"ASCEND_VISIBLE_DEVICES=0-3,5,7"} word := "ASCEND_VISIBLE_DEVICE" @@ -382,6 +412,7 @@ func TestGetValueByKeyCase3(t *testing.T) { assert.EqualValues(t, expectVal, actualVal) } +// TestUpdateEnvAndPostHook tests the function updateEnvAndPostHook func TestUpdateEnvAndPostHook(t *testing.T) { defer func() { if e := recover(); e != nil { @@ -405,11 +436,12 @@ func TestUpdateEnvAndPostHook(t *testing.T) { } updateEnvAndPostHook(&spec, vdvice, &deviceList) - assert.Contains(t, spec.Process.Env, "ASCEND_VISIBLE_DEVICES=100") + assert.Contains(t, spec.Process.Env, "ASCEND_VISIBLE_DEVICES=0") assert.Contains(t, spec.Process.Env, "ASCEND_RUNTIME_OPTIONS=VIRTUAL") assert.Contains(t, spec.Hooks.Poststop[0].Path, destroyHookCli) } +// TestAddDeviceToSpec0 tests the function addDeviceToSpec func TestAddDeviceToSpec0(t *testing.T) { devPath := "/dev/davinci0" statStub := gomonkey.ApplyFunc(oci.DeviceFromPath, func(name string) (*specs.LinuxDevice, error) { @@ -433,6 +465,7 @@ func TestAddDeviceToSpec0(t *testing.T) { assert.Contains(t, spec.Linux.Devices[0].Path, devPath) } +// TestAddAscend310BManagerDevice tests the function addAscend310BManagerDevice func TestAddAscend310BManagerDevice(t *testing.T) { statStub := gomonkey.ApplyFunc(addDeviceToSpec, func(spec *specs.Spec, dPath string, deviceType string) error { return nil @@ -457,6 +490,7 @@ func TestAddAscend310BManagerDevice(t *testing.T) { assert.Nil(t, err) } +// TestAddCommonManagerDevice tests the function addCommonManagerDevice func TestAddCommonManagerDevice(t *testing.T) { statStub := gomonkey.ApplyFunc(addDeviceToSpec, func(spec *specs.Spec, dPath string, deviceType string) error { return nil @@ -476,6 +510,7 @@ func TestAddCommonManagerDevice(t *testing.T) { assert.Nil(t, err) } +// TestAddManagerDevice tests the function addManagerDevice func TestAddManagerDevice(t *testing.T) { devPath := "/dev/mockdevice" statStub := gomonkey.ApplyFunc(oci.DeviceFromPath, func(dPath string) (*specs.LinuxDevice, error) { @@ -504,12 +539,13 @@ func TestAddManagerDevice(t *testing.T) { }, } ctx, _ := context.WithCancel(context.Background()) - err := initLogModule(ctx) + err := InitLogModule(ctx) assert.Nil(t, err) err = addManagerDevice(&spec) assert.Nil(t, err) } +// TestAddDevice tests the function addDevice func TestAddDevice(t *testing.T) { devPath := "/dev/davinci1" statStub := gomonkey.ApplyFunc(oci.DeviceFromPath, func(name string) (*specs.LinuxDevice, error) { @@ -540,7 +576,7 @@ func TestAddDevice(t *testing.T) { } ctx, _ := context.WithCancel(context.Background()) - err := initLogModule(ctx) + err := InitLogModule(ctx) assert.Nil(t, err) err = addDevice(&spec, deviceList) assert.Nil(t, err)