From 325a0e318a389c278bf78c4665588118154389f3 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Sun, 25 Feb 2024 15:28:58 +0800 Subject: [PATCH] add `coreos-assembler remote-session` command [upstream] 3defcfa7639b46a2ac823324b7362714f74e7e51 e31e7f5ac659c771b8af7cd1bfbb1c31700ebdcd cea4d71f480d972e0441ace3e2e65d7649250d82 c98a6b8c9eff6b14c9c279d8ef69720ebdfc0044 93efb63dcbd63dc04a782e2c6c617ae0cd4a51c8 26c364ed03e481776bf5e184f5351992c64f953d 0083086c4720b602b8243effb85c0a1f73f013dd 0e8b784e2e43d0758c1f0d3fa296419d7d2f76f4 9b0e7e8382ed93a7a52d805a72dfd1b28b3b68d7 --- cmd/coreos-assembler.go | 14 +- cmd/remote-session.go | 273 +++++++++++++++++++++++++++++++++ docs/cosa.md | 2 +- docs/cosa/buildextend-secex.md | 43 ++++++ 4 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 cmd/remote-session.go create mode 100644 docs/cosa/buildextend-secex.md diff --git a/cmd/coreos-assembler.go b/cmd/coreos-assembler.go index 4f981f55..42a08b68 100644 --- a/cmd/coreos-assembler.go +++ b/cmd/coreos-assembler.go @@ -15,7 +15,7 @@ import ( var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean", "list"} var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container", "upload-oscontainer"} var buildextendCommands = []string{"aliyun", "aws", "azure", "digitalocean", "exoscale", "gcp", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"} -var utilityCommands = []string{"aws-replicate", "build-extensions-container", "compress", "generate-hashlist", "koji-upload", "kola", "remote-build-container", "remote-prune", "sign", "tag"} +var utilityCommands = []string{"aws-replicate", "build-extensions-container", "compress", "generate-hashlist", "koji-upload", "kola", "remote-build-container", "remote-prune", "remote-session", "sign", "tag"} var otherCommands = []string{"shell", "meta"} func init() { @@ -68,12 +68,24 @@ func run(argv []string) error { os.Exit(1) } + // if the COREOS_ASSEMBLER_REMOTE_SESSION environment variable is + // set then we "intercept" the command here and redirect it to + // `cosa remote-session exec`, which will execute the commands + // via `podman --remote` on a remote machine. + session, ok := os.LookupEnv("COREOS_ASSEMBLER_REMOTE_SESSION") + if ok && session != "" && cmd != "remote-session" { + argv = append([]string{"exec", "--", cmd}, argv...) + cmd = "remote-session" + } + // Manual argument parsing here for now; once we get to "phase 1" // of the Go conversion we can vendor cobra (and other libraries) // at the toplevel. switch cmd { case "clean": return runClean(argv) + case "remote-session": + return runRemoteSession(argv) case "build-extensions-container": return buildExtensionContainer() } diff --git a/cmd/remote-session.go b/cmd/remote-session.go new file mode 100644 index 00000000..2968a062 --- /dev/null +++ b/cmd/remote-session.go @@ -0,0 +1,273 @@ +package main + +import ( + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/spf13/cobra" +) + +type RemoteSessionOptions struct { + CreateImage string + CreateExpiration string + CreateWorkdir string + SyncQuiet bool +} + +var ( + remoteSessionOpts RemoteSessionOptions + + cmdRemoteSession = &cobra.Command{ + Use: "remote-session", + Short: "cosa remote-session [command]", + Long: "Initiate and use remote sessions for COSA execution.", + } + + cmdRemoteSessionCreate = &cobra.Command{ + Use: "create", + Short: "Create a remote session", + Long: "Create a remote session. This command will print an ID to " + + "STDOUT that should be set in COREOS_ASSEMBLER_REMOTE_SESSION " + + "environment variable for later commands to use.", + Args: cobra.ExactArgs(0), + PreRunE: preRunCheckEnv, + RunE: runCreate, + } + + cmdRemoteSessionDestroy = &cobra.Command{ + Use: "destroy", + Short: "Destroy a remote session", + Long: "Destroy a remote session. After running this command the " + + "COREOS_ASSEMBLER_REMOTE_SESSION should be unset.", + Args: cobra.ExactArgs(0), + PreRunE: preRunCheckEnv, + RunE: runDestroy, + } + + cmdRemoteSessionExec = &cobra.Command{ + Use: "exec", + Short: "Execute a cosa command in the remote session", + Long: "Execute a cosa command in the remote session.", + Args: cobra.MinimumNArgs(1), + PreRunE: preRunCheckEnv, + RunE: runExec, + } + + cmdRemoteSessionPS = &cobra.Command{ + Use: "ps", + Short: "Check if the remote session is running", + Long: "Check if the remote session is running.", + Args: cobra.ExactArgs(0), + PreRunE: preRunCheckEnv, + RunE: runPS, + } + + cmdRemoteSessionSync = &cobra.Command{ + Use: "sync", + Short: "sync files/directories to/from the remote", + Long: "sync files/directories to/from the remote. The symantics here " + + "are similar to rsync or scp. Provide `:from to` or `from :to`. " + + "The argument with the leading ':' will represent the remote.", + Args: cobra.MinimumNArgs(2), + PreRunE: preRunCheckEnv, + RunE: runSync, + } +) + +// Function to determine if stdin is a terminal or not. +func isatty() bool { + cmd := exec.Command("tty") + cmd.Stdin = os.Stdin + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + err := cmd.Run() + return err == nil +} + +// Function to check if a given environment variable exists +// and is non-empty. +func envVarIsSet(v string) bool { + val, ok := os.LookupEnv(v) + if !ok || val == "" { + return false + } else { + return true + } +} + +// Function to return an error object with appropriate text for an +// environment variable error based on the given inputs. +func envVarError(v string, required bool) error { + if required { + return fmt.Errorf("The env var %s must be defined and non-empty", v) + } else { + return fmt.Errorf("The env var %s must not be defined", v) + } +} + +// Function to check requisite environment variables. This is run +// before each subcommand to perform the checks. +func preRunCheckEnv(c *cobra.Command, args []string) error { + // We need to make sure that the CONTAINER_HOST env var + // is set for all commands. This is used for `podman --remote`. + // We could also check `CONTAINER_SSHKEY` key here but it's not + // strictly required (user could be using ssh-agent). + if !envVarIsSet("CONTAINER_HOST") { + return envVarError("CONTAINER_HOST", true) + } + // We need to check COREOS_ASSEMBLER_REMOTE_SESSION. For create + // we need to make sure it's not set. For all other commands we + // need to make sure it is set. + remoteSessionVarIsSet := envVarIsSet("COREOS_ASSEMBLER_REMOTE_SESSION") + if c.Use == "create" && remoteSessionVarIsSet { + return envVarError("COREOS_ASSEMBLER_REMOTE_SESSION", false) + } else if c.Use != "create" && !remoteSessionVarIsSet { + return envVarError("COREOS_ASSEMBLER_REMOTE_SESSION", true) + } + return nil +} + +// Creates a "remote session" on the remote. This just creates a +// container on the remote and prints to STDOUT the container ID. +// The user is then expected to store this ID in the +// COREOS_ASSEMBLER_REMOTE_SESSION environment variable. +func runCreate(c *cobra.Command, args []string) error { + podmanargs := []string{"--remote", "run", "--rm", "-d", + "--pull=always", "--privileged", "--security-opt=label=disable", + "--volume", remoteSessionOpts.CreateWorkdir, + "--workdir", remoteSessionOpts.CreateWorkdir, + // Mount required volume for buildextend-secex, it will be empty on + // non-s390x builders. + // See: https://github.com/coreos/coreos-assembler/blob/main/docs/cosa/buildextend-secex.md + "--volume=secex-data:/data.secex:ro", + "--uidmap=1000:0:1", "--uidmap=0:1:1000", "--uidmap=1001:1001:64536", + "--device=/dev/kvm", "--device=/dev/fuse", "--tmpfs=/tmp", + "--init", "--entrypoint=/usr/bin/sleep", + remoteSessionOpts.CreateImage, + remoteSessionOpts.CreateExpiration} + cmd := exec.Command("podman", podmanargs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// Destroys the "remote session". In reality it just deletes +// the container referenced by $COREOS_ASSEMBLER_REMOTE_SESSION. +func runDestroy(c *cobra.Command, args []string) error { + session := os.Getenv("COREOS_ASSEMBLER_REMOTE_SESSION") + podmanargs := []string{"--remote", "rm", "-f", session} + cmd := exec.Command("podman", podmanargs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// Executes a command in the "remote session". Mostly just a +// `podman --remote exec`. +func runExec(c *cobra.Command, args []string) error { + podmanargs := []string{"--remote", "exec", "-i"} + if isatty() { + podmanargs = append(podmanargs, "-t") + } + session := os.Getenv("COREOS_ASSEMBLER_REMOTE_SESSION") + podmanargs = append(podmanargs, session, "cosa") + podmanargs = append(podmanargs, args...) + cmd := exec.Command("podman", podmanargs...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + // If the command failed let's exit with the same exitcode + // as the remotely executed process. + os.Exit(exitError.ExitCode()) + } else { + return err + } + } + return nil +} + +// Executes a `podman --remote ps -a --filter id=` +// to show the status of the remote running cosa container. +func runPS(c *cobra.Command, args []string) error { + session := os.Getenv("COREOS_ASSEMBLER_REMOTE_SESSION") + podmanargs := []string{"--remote", "ps", "-a", + fmt.Sprintf("--filter=id=%s", session)} + cmd := exec.Command("podman", podmanargs...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// runSync provides an rsync-like interface that allows +// files to be copied to/from the remote. It uses +// `podman --remote exec` as the transport for rsync (see [1]) +// +// One of the arguments here must be prepended with a `:`. This +// argument will represent the path on the remote. This function +// will substitute `:` with `$COREOS_ASSEMBLER_REMOTE_SESSION:`. +// That environment var just contains the container ID on the remote. +// +// [1] https://github.com/moby/moby/issues/13660 +func runSync(c *cobra.Command, args []string) error { + // check arguments. Need one with pre-pended ':' + found := 0 + for index, arg := range args { + if strings.HasPrefix(arg, ":") { + args[index] = fmt.Sprintf("%s%s", + os.Getenv("COREOS_ASSEMBLER_REMOTE_SESSION"), arg) + found++ + } + } + if found != 1 { + return fmt.Errorf("Must pass in a single arg with `:` prepended") + } + // build command and execute + rsyncargs := []string{"-ah", "--no-owner", "--no-group", "--mkpath", "--blocking-io", + "--compress", "--rsh", "podman --remote exec -i"} + if !remoteSessionOpts.SyncQuiet { + rsyncargs = append(rsyncargs, "-v") + } + rsyncargs = append(rsyncargs, args...) + cmd := exec.Command("rsync", rsyncargs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func init() { + cmdRemoteSession.AddCommand(cmdRemoteSessionCreate) + cmdRemoteSession.AddCommand(cmdRemoteSessionDestroy) + cmdRemoteSession.AddCommand(cmdRemoteSessionExec) + cmdRemoteSession.AddCommand(cmdRemoteSessionPS) + cmdRemoteSession.AddCommand(cmdRemoteSessionSync) + + // cmdRemoteSessionCreate options + cmdRemoteSessionCreate.Flags().StringVarP( + &remoteSessionOpts.CreateImage, "image", "", + //TODO: to modify to the nestos's version + "quay.io/coreos-assembler/coreos-assembler:main", + "The COSA container image to use on the remote") + cmdRemoteSessionCreate.Flags().StringVarP( + &remoteSessionOpts.CreateExpiration, "expiration", "", "infinity", + "The amount of time before the remote-session auto-exits") + cmdRemoteSessionCreate.Flags().StringVarP( + &remoteSessionOpts.CreateWorkdir, "workdir", "", "/srv", + "The COSA working directory to use inside the container") + + // cmdRemoteSessionSync options + cmdRemoteSessionSync.Flags().BoolVarP( + &remoteSessionOpts.SyncQuiet, "quiet", "", false, + "Make the sync output less verbose") +} + +// execute the cmdRemoteSession cobra command +func runRemoteSession(argv []string) error { + cmdRemoteSession.SetArgs(argv) + return cmdRemoteSession.Execute() +} diff --git a/docs/cosa.md b/docs/cosa.md index 4d736e43..73020612 100644 --- a/docs/cosa.md +++ b/docs/cosa.md @@ -42,7 +42,7 @@ other platforms or cloud providers: | Name | Description | | ---- | ----------- | | [buildextend-live](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-buildextend-live) | Generate the Live ISO -| [buildextend-{dasd,metal,metal4k,qemu}](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-buildextend-metal) | Generate artifacts for the given platforms +| [buildextend-{dasd,metal,metal4k,qemu,secex}](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-buildextend-metal) | Generate artifacts for the given platforms | [buildextend-{aliyun,aws,azure,digitalocean,exoscale,gcp,vultr}](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-ore-wrapper) | Generate artifacts for the given platforms | [buildextend-{azurestack,ibmcloud,openstack,vmware}](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-artifact-disk) | Generate artifacts for the given platforms | [{aliyun,aws}-replicate](https://github.com/coreos/coreos-assembler/blob/main/src/cmd-ore-wrapper) | Replicate images on the platforms (AMIs for AWS) diff --git a/docs/cosa/buildextend-secex.md b/docs/cosa/buildextend-secex.md new file mode 100644 index 00000000..5eaa6a63 --- /dev/null +++ b/docs/cosa/buildextend-secex.md @@ -0,0 +1,43 @@ +--- +parent: CoreOS Assembler Command Line Reference +nav_order: 1 +--- + +# cosa buildextend-secex + +This buildextend command is used to build QEMU images that are enabled for IBM Secure Execution on IBM Z. +In order to build a QEMU image protected by IBM Secure Execution, you need to provide a host key to encrypt it. + +For more information on IBM Secure Execution on IBM Z, refer to the [IBM Documentation](https://www.ibm.com/docs/en/linux-on-systems?topic=ibmz-secure-execution). + +The command is intended to be used in the RHCOS CI together with the universal host key, such that the image can be booted on any IBM Z machine that supports IBM Secure Execution. +This results in a few specifics to note: +- The resulting image will only be encrypted with a single host key, to enable firstboot. +- The host key will not be written to the image. +- The host key(s) need to be provided later during firstboot through Ignition. + - The firstboot service will fail when no host key is provided, as the sdboot-image can not be recreated. + - Write the host key(s) to: `/etc/se-hostkeys/ibm-z-hostkey-.crt` + +To facilitate this, `buildextend-secex` can take 2 mutually exclusive additional arguments: `--genprotimgvm ` and `--hostkey `. +If none is provided, `--genprotimgvm` is used with default values. + +## `--genprotimgvm ` (default) + +Default Value: `/data.secex/genprotimgvm.qcow2` + +This path is the default behavior. It assumes that the host key is not directly available, but is supplied through an IBM Secure Execution protected VM only. + +The QEMU image will be built normally. However, it will not run `genprotimg` or `zipl`, but instead save the required input for the command to a temporary location. +After the build, the provided VM will run. The VM is used to isolate and protect the `genprotimg` command, so that the universal host key is not exposed. +A provided bash script is called before and after the `genprotimg` command, to fullfil the following steps: +1. Copy the required kernel, initramfs, and parmfile to the VM +2. Move the sdboot-image to the disk +3. Call `zipl`to make the image bootable. +This enables us to copy the required kernel, initramfs and parmfile to the VM and afterwards move the sdboot-image to the disk, as well as calling `zipl` to make the image bootable. + +## `--hostkey ` + +This path is intended for local development, but can be used for custom builds. The path takes a singe host key file, which is used to build the image. + +Instead of running `genprotimg` and `zipl` in a separate VM, they run during the build process. Otherwise, the build is identical to the `--genprotimgvm`. +Note: It is still assumed that the host key is provided via Ignition during firstboot. -- Gitee