From d40c9ad7de628f47acf8121772be45aafcb28e90 Mon Sep 17 00:00:00 2001 From: wangyueliang Date: Mon, 29 Jan 2024 15:55:31 +0800 Subject: [PATCH] Remove gangplank [upstream] 684c398fa93b4f10ac7549ba23f3b013eef6caa7 --- gangplank/.gitignore | 3 - gangplank/.vscode/settings.json | 6 - gangplank/Makefile | 88 --- gangplank/cmd/gangplank/commonflags.go | 50 -- gangplank/cmd/gangplank/generate.go | 168 ----- gangplank/cmd/gangplank/main.go | 195 ----- gangplank/cmd/gangplank/minio.go | 82 --- gangplank/cmd/gangplank/pod.go | 214 ------ gangplank/cmd/gangway/main.go | 33 - gangplank/internal/ocp/bc.go | 894 ----------------------- gangplank/internal/ocp/bc_ci_test.go | 34 - gangplank/internal/ocp/bc_test.go | 77 -- gangplank/internal/ocp/build.json | 86 --- gangplank/internal/ocp/client.go | 36 - gangplank/internal/ocp/conditions.go | 80 -- gangplank/internal/ocp/const.go | 28 - gangplank/internal/ocp/cosa-pod-s390x.go | 13 - gangplank/internal/ocp/cosa-pod.go | 624 ---------------- gangplank/internal/ocp/cosa-podman.go | 259 ------- gangplank/internal/ocp/cosa_init.go | 63 -- gangplank/internal/ocp/errors.go | 30 - gangplank/internal/ocp/filer.go | 686 ----------------- gangplank/internal/ocp/filer_test.go | 148 ---- gangplank/internal/ocp/hop.go | 134 ---- gangplank/internal/ocp/k8s.go | 231 ------ gangplank/internal/ocp/ocp.go | 98 --- gangplank/internal/ocp/pod.go | 276 ------- gangplank/internal/ocp/podman-vars.go | 34 - gangplank/internal/ocp/remotes.go | 119 --- gangplank/internal/ocp/remotes_test.go | 108 --- gangplank/internal/ocp/return.go | 280 ------- gangplank/internal/ocp/return_test.go | 93 --- gangplank/internal/ocp/sa_secrets.go | 233 ------ gangplank/internal/ocp/source_extract.go | 56 -- gangplank/internal/ocp/ssh.go | 230 ------ gangplank/internal/ocp/volumes.go | 260 ------- gangplank/internal/ocp/worker.go | 473 ------------ gangplank/internal/spec/cli.go | 131 ---- gangplank/internal/spec/clone.go | 53 -- gangplank/internal/spec/clouds.go | 210 ------ gangplank/internal/spec/jobspec.go | 327 --------- gangplank/internal/spec/jobspec_test.go | 116 --- gangplank/internal/spec/kola.go | 73 -- gangplank/internal/spec/override.go | 144 ---- gangplank/internal/spec/override_test.go | 88 --- gangplank/internal/spec/render_test.go | 94 --- gangplank/internal/spec/stage_test.go | 415 ----------- gangplank/internal/spec/stages.go | 649 ---------------- gangplank/internal/spec/tmpl.go | 107 --- 49 files changed, 8929 deletions(-) delete mode 100644 gangplank/.gitignore delete mode 100644 gangplank/.vscode/settings.json delete mode 100644 gangplank/Makefile delete mode 100644 gangplank/cmd/gangplank/commonflags.go delete mode 100644 gangplank/cmd/gangplank/generate.go delete mode 100644 gangplank/cmd/gangplank/main.go delete mode 100644 gangplank/cmd/gangplank/minio.go delete mode 100644 gangplank/cmd/gangplank/pod.go delete mode 100644 gangplank/cmd/gangway/main.go delete mode 100644 gangplank/internal/ocp/bc.go delete mode 100644 gangplank/internal/ocp/bc_ci_test.go delete mode 100644 gangplank/internal/ocp/bc_test.go delete mode 100644 gangplank/internal/ocp/build.json delete mode 100644 gangplank/internal/ocp/client.go delete mode 100644 gangplank/internal/ocp/conditions.go delete mode 100644 gangplank/internal/ocp/const.go delete mode 100644 gangplank/internal/ocp/cosa-pod-s390x.go delete mode 100644 gangplank/internal/ocp/cosa-pod.go delete mode 100644 gangplank/internal/ocp/cosa-podman.go delete mode 100644 gangplank/internal/ocp/cosa_init.go delete mode 100644 gangplank/internal/ocp/errors.go delete mode 100644 gangplank/internal/ocp/filer.go delete mode 100644 gangplank/internal/ocp/filer_test.go delete mode 100644 gangplank/internal/ocp/hop.go delete mode 100644 gangplank/internal/ocp/k8s.go delete mode 100644 gangplank/internal/ocp/ocp.go delete mode 100644 gangplank/internal/ocp/pod.go delete mode 100644 gangplank/internal/ocp/podman-vars.go delete mode 100644 gangplank/internal/ocp/remotes.go delete mode 100644 gangplank/internal/ocp/remotes_test.go delete mode 100644 gangplank/internal/ocp/return.go delete mode 100644 gangplank/internal/ocp/return_test.go delete mode 100644 gangplank/internal/ocp/sa_secrets.go delete mode 100644 gangplank/internal/ocp/source_extract.go delete mode 100644 gangplank/internal/ocp/ssh.go delete mode 100644 gangplank/internal/ocp/volumes.go delete mode 100644 gangplank/internal/ocp/worker.go delete mode 100644 gangplank/internal/spec/cli.go delete mode 100644 gangplank/internal/spec/clone.go delete mode 100644 gangplank/internal/spec/clouds.go delete mode 100644 gangplank/internal/spec/jobspec.go delete mode 100644 gangplank/internal/spec/jobspec_test.go delete mode 100644 gangplank/internal/spec/kola.go delete mode 100644 gangplank/internal/spec/override.go delete mode 100644 gangplank/internal/spec/override_test.go delete mode 100644 gangplank/internal/spec/render_test.go delete mode 100644 gangplank/internal/spec/stage_test.go delete mode 100644 gangplank/internal/spec/stages.go delete mode 100644 gangplank/internal/spec/tmpl.go diff --git a/gangplank/.gitignore b/gangplank/.gitignore deleted file mode 100644 index f5920f0d..00000000 --- a/gangplank/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -bin/* -srv/* -**/.minio.sys/* diff --git a/gangplank/.vscode/settings.json b/gangplank/.vscode/settings.json deleted file mode 100644 index 63b7bc33..00000000 --- a/gangplank/.vscode/settings.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "go buildtags": "containers_image_openpgp", - "go.toolsEnvVars": { - "CGO_ENABLED": "0" - } -} diff --git a/gangplank/Makefile b/gangplank/Makefile deleted file mode 100644 index ce5159ad..00000000 --- a/gangplank/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -my_dir := $(abspath $(shell dirname $(lastword $(MAKEFILE_LIST)))) -state = $(shell test -n "`git status -s`" && echo dirty || echo clean) -version = $(shell cd .. && git log -n 1 --date=short --pretty=format:%cs.%h~${state} -- gangplank) -cosa_dir = $(shell test -d /usr/lib/coreos-assembler && echo /usr/lib/coreos-assembler) -ldflags=-X main.version=${version} -X main.cosaDir=${cosa_dir} -buildtags=containers_image_openpgp -export PATH := $(my_dir)/bin:$(shell readlink -f ../tools/bin):$(PATH) - -PREFIX ?= /usr -DESTDIR ?= -ARCH:=$(shell uname -m) - -pkgs := $(shell go list -mod=vendor ./...) -.PHONY: build -build: - cd "${my_dir}/cmd/gangway" && go build -ldflags "${ldflags}" -tags ${buildtags},gangway -mod vendor -v -o ${my_dir}/bin/gangway - cd "${my_dir}/cmd/gangplank" && env CGO_ENABLED=0 \ - go build -ldflags "${ldflags}" -tags ${buildtags} -mod vendor -v -o ${my_dir}/bin/gangplank . - -.PHONY: docs -docs: - for i in ocp cosa spec; do cd ${my_dir}/$$i; pwd; gomarkdoc \ - -u -o ../../docs/gangplank/api-$$i.md ./...; done \ - -.PHONY: fmt -fmt: - gofmt -d -e -l $(shell find . -iname "*.go" -not -path "./vendor/*") - -.PHONY: staticanalysis -staticanalysis: - golangci-lint run -v --build-tags ${buildtags},gangway ./... - env CGO_ENABLED=0 golangci-lint run -v --build-tags ${buildtags},gangway ./... - -.PHONY: test -test: miniotag ?= ",!minio" -test: fmt - go test -mod=vendor -cover=1 -tags ${buildtags},gangway,!minio -v ${pkgs} && \ - env CGO_ENABLED=0 go test -cover=1 -mod=vendor -tags ${buildtags},!gangway${miniotag} -v -cover ${pkgs} - -.PHONY: test-full -test-full: - $(MAKE) test miniotag=",minio" - -.PHONY: clean -clean: - @go clean . - @rm -rf bin - -.PHONY: schema -schema: - $(MAKE) -C ../tools schema - -.PHONY: install -install: bin/gangplank - install -v -D -t $(DESTDIR)$(PREFIX)/bin bin/gangplank - install -v -D -t $(DESTDIR)$(PREFIX)/bin bin/gangway - -.PHONY: go-deps -go-deps: - go mod tidy - go mod vendor - go mod download - -my_uid = $(shell id -u) -.PHONY: devtest -devtest: build - mkdir -p srv - podman run --rm -i --tty \ - -a=stdin -a=stdout -a=stderr \ - --uidmap=$(my_uid):0:1 --uidmap=0:1:1000 --uidmap 1001:1001:64536 \ - --security-opt label=disable --privileged=true \ - --device /dev/fuse \ - --device /dev/kvm \ - --tmpfs /tmp \ - --volume=/var/tmp:/var/tmp \ - --volume=$(shell realpath .)/srv:/srv \ - --env="BUILD=`jq -cM "." ocp/build.json`" \ - --env="SOURCE_REPOSITORY=http://github.com/coreos/fedora-coreos-config" \ - --env="SOURCE_REF=testing-devel" \ - --env='COSA_CMDS=cosa fetch; cosa build;' \ - --volume=$(shell realpath .)/bin:/run/bin \ - --entrypoint='["/usr/bin/dumb-init", "/run/bin/gangplank"]' \ - quay.io/coreos-assembler/coreos-assembler:latest \ - builder - -.PHONY: dev-image -dev-image: build - $(MAKE) -C ../ocp build-dev diff --git a/gangplank/cmd/gangplank/commonflags.go b/gangplank/cmd/gangplank/commonflags.go deleted file mode 100644 index 25b8225f..00000000 --- a/gangplank/cmd/gangplank/commonflags.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "os/user" - - jobspec "github.com/coreos/gangplank/internal/spec" - flag "github.com/spf13/pflag" -) - -// Flags has the configuration flags. -var specCommonFlags = flag.NewFlagSet("", flag.ContinueOnError) - -// sshFlags are specific to minio and remote podman -var ( - sshFlags = flag.NewFlagSet("", flag.ContinueOnError) - - // minioSshRemoteHost is the SSH remote host to forward the local - // minio instance over. - minioSshRemoteHost string - - // minioSshRemoteUser is the name of the SSH user to use with minioSshRemoteHost - minioSshRemoteUser string - - // minioSshRemoteKey is the SSH key to use with minioSshRemoteHost - minioSshRemoteKey string - - // minioSshRemotePort is the SSH port to use with minioSshRemotePort - minioSshRemotePort int -) - -// cosaKolaTests are used to generate automatic Kola stages. -var cosaKolaTests []string - -func init() { - specCommonFlags.StringSliceVar(&generateCommands, "singleCmd", []string{}, "commands to run in stage") - specCommonFlags.StringSliceVar(&generateSingleRequires, "singleReq", []string{}, "artifacts to require") - specCommonFlags.StringVarP(&cosaSrvDir, "srvDir", "S", "", "directory for /srv; in pod mount this will be bind mounted") - specCommonFlags.StringSliceVar(&generateReturnFiles, "returnFiles", []string{}, "Extra files to upload to the minio server") - jobspec.AddKolaTestFlags(&cosaKolaTests, specCommonFlags) - - username := "" - user, err := user.Current() - if err != nil && user != nil { - username = user.Username - } - - sshFlags.StringVar(&minioSshRemoteHost, "forwardMinioSSH", containerHost(), "forward and use minio to ssh host") - sshFlags.StringVar(&minioSshRemoteUser, "sshUser", username, "name of SSH; used with forwardMinioSSH") - sshFlags.StringVar(&minioSshRemoteKey, "sshKey", "", "path to SSH key; used with forwardMinioSSH") -} diff --git a/gangplank/cmd/gangplank/generate.go b/gangplank/cmd/gangplank/generate.go deleted file mode 100644 index dd1aca78..00000000 --- a/gangplank/cmd/gangplank/generate.go +++ /dev/null @@ -1,168 +0,0 @@ -package main - -import ( - "fmt" - "os" - "time" - - jobspec "github.com/coreos/gangplank/internal/spec" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -var ( - // generateFileName is a file handle where the generate JobSpec - // will be written to with generateCommand - generateFileName string - - // generateSingleCommands is a list of command that will be run in the stage - generateCommands []string - - // generateReturnFiles defines a list of extra files to upload to minio server - generateReturnFiles []string - - // generateSingleStage indicates that all commands/artfiacts should be in the same stage - generateSingleStage bool - - // generateSingleRequires insdicates all the artifacts that can be required - generateSingleRequires []string - - // minioBucket defines the minio bucket to use - minioBucket string - - // minioPathPrefix desings the minio path to use relative to the bucket - minioPathPrefix string -) - -var ( - // cmdGenerate creates a jobspec and dumps it. - cmdGenerate = &cobra.Command{ - Use: "generate", - Short: "generate jobspec from CLI args", - Run: generateCLICommand, - } - - // generateSinglePod creates a single pod specification - cmdGenerateSingle = &cobra.Command{ - Use: "generateSinglePod", - Short: "generate a single pod job spec", - Run: func(c *cobra.Command, args []string) { - generateSingleStage = true - generateCLICommand(c, args) - }, - } -) - -func init() { - cmdRoot.PersistentFlags().StringSliceVarP(&automaticBuildStages, "build-artifact", "A", []string{}, - fmt.Sprintf("build artifact for any of: %v", jobspec.GetArtifactShortHandNames())) - - cmdRoot.PersistentFlags().StringVar(&minioBucket, "bucket", "builder", "name minio bucket to use") - cmdRoot.PersistentFlags().StringVar(&minioPathPrefix, "keyPathPrefix", "", "path prefix to use inside the bucket") - - // Add the jobspec flags to the CLI - spec.AddCliFlags(cmdGenerate.Flags()) - spec.AddCliFlags(cmdGenerateSingle.Flags()) - - // Define cmdGenerate flags - cmdRoot.AddCommand(cmdGenerate) - cmdGenerate.Flags().StringVar(&generateFileName, "yaml-out", "", "write YAML to file") - jobspec.AddKolaTestFlags(&cosaKolaTests, cmdGenerate.Flags()) - - // Define cmdGenerateSingle flags - cmdRoot.AddCommand(cmdGenerateSingle) - cmdGenerateSingle.Flags().StringVar(&generateFileName, "yaml-out", "", "write YAML to file") - cmdGenerateSingle.Flags().StringSliceVar(&generateCommands, "cmd", []string{}, "commands to run in stage") - cmdGenerateSingle.Flags().StringSliceVar(&generateSingleRequires, "req", []string{}, "artifacts to require") - cmdGenerateSingle.Flags().StringSliceVar(&generateReturnFiles, "returnFiles", []string{}, "Extra files to upload to the minio server") - jobspec.AddKolaTestFlags(&cosaKolaTests, cmdGenerateSingle.Flags()) -} - -// setCliSpec reads or generates a jobspec based on CLI arguments. -func setCliSpec() { - defer func() { - // Always add repos - if spec.Recipe.Repos == nil { - spec.AddRepos() - } - // Override CopyBuild from the CLI - spec.AddCopyBuild() - if minioSshRemoteHost != "" && minioCfgFile == "" { - spec.Minio.SSHForward = minioSshRemoteHost - spec.Minio.SSHUser = minioSshRemoteUser - spec.Minio.SSHKey = minioSshRemoteKey - spec.Minio.SSHPort = minioSshRemotePort - } - if minioCfgFile != "" { - spec.Minio.ConfigFile = minioCfgFile - } - if spec.Minio.KeyPrefix == "" { - spec.Minio.KeyPrefix = minioPathPrefix - } - if spec.Minio.Bucket == "" { - spec.Minio.Bucket = minioBucket - } - log.WithFields(log.Fields{"prefix": minioPathPrefix, "bucket": minioBucket}).Info("Remote paths defined") - }() - - if specFile != "" { - js, err := jobspec.JobSpecFromFile(specFile) - if err != nil { - log.WithError(err).Fatal("failed to read jobspec") - } - spec = js - - log.WithFields(log.Fields{ - "jobspec": specFile, - "ingored cli args": "-A|--artifact|--singleReq|--singleCmd", - }).Info("Using jobspec from file, some cli arguments will be ignored") - return - } - - log.Info("Generating jobspec from CLI arguments") - if len(generateCommands) != 0 || len(generateSingleRequires) != 0 { - log.Info("--cmd and --req forces single stage mode, only one stage will be run") - generateSingleStage = true - } - - log.Info("Generating stages") - if err := spec.GenerateStages(automaticBuildStages, cosaKolaTests, generateSingleStage); err != nil { - log.WithError(err).Fatal("failed to generate the jobpsec") - } - - if spec.Stages == nil { - spec.Stages = []jobspec.Stage{ - { - ID: "CLI Commands", - ExecutionOrder: 1, - }, - } - } - - spec.Stages[0].AddCommands(generateCommands) - spec.Stages[0].AddRequires(generateSingleRequires) - spec.Stages[0].AddReturnFiles(generateReturnFiles) -} - -// generateCLICommand is the full spec generator command -func generateCLICommand(*cobra.Command, []string) { - var out *os.File = os.Stdout - if generateFileName != "" { - f, err := os.OpenFile(generateFileName, os.O_CREATE|os.O_WRONLY, 0755) - if err != nil { - log.WithError(err).Fatalf("unable to open %s for writing", generateFileName) - } - defer f.Close() - out = f - } - setCliSpec() - defer out.Sync() //nolint - - now := time.Now().Format(time.RFC3339) - if _, err := out.Write([]byte("# Generated by Gangplank CLI\n# " + now + "\n")); err != nil { - log.WithError(err).Fatalf("Failed to write header to file") - } - if err := spec.WriteYAML(out); err != nil { - log.WithError(err).Fatal("Faield to write Gangplank YAML") - } -} diff --git a/gangplank/cmd/gangplank/main.go b/gangplank/cmd/gangplank/main.go deleted file mode 100644 index a4292f09..00000000 --- a/gangplank/cmd/gangplank/main.go +++ /dev/null @@ -1,195 +0,0 @@ -package main - -/* - Definition for the main Gangplank command. This defined the "human" - interfaces for `run` and `run-steps` -*/ - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "time" - - "github.com/coreos/coreos-assembler-schema/cosa" - jobspec "github.com/coreos/gangplank/internal/spec" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -const cosaContainerDir = "/usr/lib/coreos-assembler" - -var ( - ctx, cancel = context.WithCancel(context.Background()) - - version = "devel" - - // cosaDir is the installed location of COSA. This defaults to - // cosaContainerDir and is set via `-ldflags` at build time. - cosaDir string - - // spec is a job spec. - spec jobspec.JobSpec - specFile string - - // envVars are set for command execution - envVars []string - - cmdRoot = &cobra.Command{ - Use: "gangplank [command]", - Short: "COSA Gangplank", - Long: `OpenShift COSA Job Runner -Wrapper for COSA commands and templates`, - PersistentPreRun: preRun, - } - - cmdVersion = &cobra.Command{ - Use: "version", - Short: "Print the version number and exit.", - Run: func(cmd *cobra.Command, args []string) { - cmd.Printf("gangplank/%s version %s\n", - cmd.Root().Name(), version) - }, - } - - cmdSingle = &cobra.Command{ - Use: "run", - Short: "Run the commands and bail", - Args: cobra.MinimumNArgs(1), - Run: runSingle, - } - - cmdPodless = &cobra.Command{ - Use: "podless", - Short: "Run outisde of pod (via prow/ci)", - RunE: runSpecLocally, - } -) - -var ( - // cosaInit indicates that cosa init should be run - cosaInit bool - - // buildArch indicates the target architecture to build - buildArch = cosa.BuilderArch() -) - -func init() { - if cosaDir == "" { - path, err := os.Getwd() - if err != nil { - cosaDir = cosaContainerDir - } else { - cosaDir = filepath.Dir(path) - } - } - - envVars = os.Environ() - - log.SetOutput(os.Stdout) - log.SetLevel(log.DebugLevel) - newPath := fmt.Sprintf("%s:%s", cosaDir, os.Getenv("PATH")) - os.Setenv("PATH", newPath) - - // cmdRoot options - cmdRoot.PersistentFlags().StringVarP(&buildArch, "arch", "a", buildArch, "override the build arch") - cmdRoot.PersistentFlags().StringVarP(&specFile, "spec", "s", "", "location of the spec") - cmdRoot.AddCommand(cmdVersion) - cmdRoot.AddCommand(cmdSingle) - cmdRoot.Flags().StringVarP(&specFile, "spec", "s", "", "location of the spec") - spec.AddCliFlags(cmdRoot.PersistentFlags()) - - // cmdPodless options - cmdRoot.AddCommand(cmdPodless) - cmdPodless.Flags().AddFlagSet(specCommonFlags) - cmdPodless.Flags().BoolVar(&cosaInit, "init", false, "force initialize srv dir") -} - -func main() { - log.Infof("Gangplank: COSA OpenShift job runner, %s", version) - if err := cmdRoot.Execute(); err != nil { - log.Fatal(err) - } - os.Exit(0) -} - -// runSpecLocally executes a jobspec locally. -func runSpecLocally(c *cobra.Command, args []string) error { - myDir, _ := os.Getwd() - defer func() { - ctx.Done() - _ = os.Chdir(myDir) - }() - - if _, err := os.Stat("src"); err != nil || cosaInit { - log.WithField("dir", cosaSrvDir).Info("Initalizing Build Tree") - spec.Stages = append(spec.Stages, jobspec.Stage{ - ID: "Initialization", - ExecutionOrder: 0, - Commands: []string{ - "cosa init --force --branch {{ .JobSpec.Recipe.GitRef }} --force {{ .JobSpec.Recipe.GitURL}} ", - }, - }) - } - - setCliSpec() - rd := &jobspec.RenderData{ - JobSpec: &spec, - } - - if err := os.Chdir(cosaSrvDir); err != nil { - log.WithError(err).Fatal("failed to change to srv dir") - } - for _, stage := range spec.Stages { - if err := stage.Execute(ctx, rd, envVars); err != nil { - log.WithError(err).Fatal("failed to execute job") - } - } - - log.Infof("Execution complete") - return nil -} - -// runSingle renders args as templates and executes the command. -func runSingle(c *cobra.Command, args []string) { - rd := &jobspec.RenderData{ - JobSpec: &spec, - } - x, err := rd.ExecuteTemplateFromString(args...) - if err != nil { - log.Fatal(err) - } - cmd := exec.CommandContext(ctx, x[0], x[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - log.Fatal(err) - } - log.Infof("Done") -} - -// preRun processes the spec file. -func preRun(c *cobra.Command, args []string) { - // Set the build arch from the commandline - if buildArch != cosa.BuilderArch() { - cosa.SetArch(buildArch) - log.WithField("arch", cosa.BuilderArch()).Info("Using non-native arch") - } - - // Terminal "keep alive" helper. When following logs via the `oc` commands, - // cloud-deployed instances will send an EOF. To get around the EOF, the func sends a - // null character that is not printed to the screen or reflected in the logs. - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(20 * time.Second): - fmt.Print("\x00") - time.Sleep(1 * time.Second) - } - } - }() -} diff --git a/gangplank/cmd/gangplank/minio.go b/gangplank/cmd/gangplank/minio.go deleted file mode 100644 index 1bd89812..00000000 --- a/gangplank/cmd/gangplank/minio.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "os" - - "github.com/coreos/gangplank/internal/ocp" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -/* - In enviroments where something else handles the visiualization of stages (such as Jenkins) - using the inherent stages in Gangplank can produced jumbled garbage. - - Each Gangpplank worker requires a Minio Server running on the coordinating host/pod. - The `gangplank minio` command allows for starting a Minio instance and then saving - the configuration to allow for a shared `minio` instance. - - When using a shared instance, you must use `cosa build --delay-meta-merge`. - - Example start - nohup gangplank minio -m /tmp/minio.cfg -d /srv & - - Example worker accessing minio: - gangplank pod -m /tmp/minio.cfg --spec test.spec - - When running in minio mode, Gangplank will continue to run until it is sig{kill,term}ed. - -*/ - -var cmdMinio = &cobra.Command{ - Use: "minio", - Short: "Start running a minio server", - Run: runMinio, -} - -var ( - // minioCfgFile is an ocp.minioServerCfg file - minioCfgFile string - - // minioServeDir is the directory that Gangplank runs minio from - minioServeDir string -) - -func init() { - cmdRoot.AddCommand(cmdMinio) - cmdRoot.PersistentFlags().StringVarP(&minioCfgFile, "minioCfgFile", "m", "", "location of where to create of external minio config file") - cmdMinio.Flags().StringVarP(&minioServeDir, "minioServeDir", "d", "", "location to service minio from") - cmdMinio.Flags().AddFlagSet(sshFlags) -} - -func runMinio(c *cobra.Command, args []string) { - defer cancel() - defer ctx.Done() - - if minioCfgFile == "" { - log.Fatal("must define --minioCfgfile to run in Minio mode") - } - if minioServeDir == "" { - log.Fatal("must define the workdir to serve minio from") - } - if _, err := os.Stat(minioCfgFile); err == nil { - log.Fatalf("existing minio configuration exists, refusing to overwrite") - } - - var minioSSH *ocp.SSHForwardPort - if minioSshRemoteHost != "" { - minioSSH = &ocp.SSHForwardPort{ - Host: minioSshRemoteHost, - User: minioSshRemoteUser, - } - } - m, err := ocp.StartStandaloneMinioServer(ctx, minioServeDir, minioCfgFile, minioSSH) - if err != nil { - log.WithError(err).Fatalf("failed to start minio server") - } - defer m.Kill() - - log.Info("Waiting for kill signal for minio") - - m.Wait() -} diff --git a/gangplank/cmd/gangplank/pod.go b/gangplank/cmd/gangplank/pod.go deleted file mode 100644 index e487ace0..00000000 --- a/gangplank/cmd/gangplank/pod.go +++ /dev/null @@ -1,214 +0,0 @@ -package main - -import ( - "os" - "os/exec" - "os/signal" - "strconv" - "strings" - "syscall" - - "github.com/coreos/gangplank/internal/ocp" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -const ( - cosaDefaultImage = "quay.io/coreos-assembler/coreos-assembler:latest" - cosaWorkDirSelinuxLabel = "system_u:object_r:container_file_t:s0" - - // podmanRemoteEnvVar is the podman-remote envVar used to tell the podman - // remote API to use a remote host. - podmanRemoteEnvVar = "CONTAINER_HOST" - // podmanSshKeyEnvVar is the podman-remote envVar for the ssh key to use - podmanSshKeyEnvVar = "CONTAINER_SSHKEY" -) - -var ( - cmdPod = &cobra.Command{ - Use: "pod", - Short: "Execute COSA command in an OpenShift Cluster (default) or Podman", - Run: runPod, - } - - // cosaImage uses a different image - cosaImage string - - // serviceAccount is the service acount to use for pod creation - // and reading of the secrets. - serviceAccount string - - // Run CI pod via podman (out of cluster) - cosaViaPodman bool - - // Remote URI for podman - cosaPodmanRemote string - - // SSH Key to use for remote Podman calls - cosaPodmanRemoteSshKey string - - // cosaWorkDir is used for podman mode and is where the "builds" directory will live - cosaWorkDir string - - // cosaWorkDirContext when true, ensures that the selinux system_u:object_r:container_file_t:s0 - // is set for the working directory. - cosaWorkDirContext bool - - // cosaSrvDir is used as the scratch directory builds. - cosaSrvDir string - - // cosaNamespace when defined will launch the pod in another namespace - cosaNamespace string - - // automaticBuildStages is used to create automatic build stages - automaticBuildStages []string - - // remoteKubeConfig sets Gangplank in remote mode - remoteKubeConfig string -) - -func init() { - cmdRoot.AddCommand(cmdPod) - - spec.AddCliFlags(cmdPod.Flags()) - cmdPod.Flags().BoolVar(&cosaWorkDirContext, "setWorkDirCtx", false, "set workDir's selinux content") - cmdPod.Flags().BoolVarP(&cosaViaPodman, "podman", "", false, "use podman to execute task") - cmdPod.Flags().StringVar(&cosaPodmanRemote, "remote", os.Getenv(podmanRemoteEnvVar), "address of the remote podman to execute task") - cmdPod.Flags().StringVar(&cosaPodmanRemoteSshKey, "sshkey", os.Getenv(podmanSshKeyEnvVar), "address of the remote podman to execute task") - cmdPod.Flags().StringVarP(&cosaImage, "image", "i", "", "use an alternative image") - cmdPod.Flags().StringVarP(&cosaWorkDir, "workDir", "w", "", "podman mode - workdir to use") - cmdPod.Flags().StringVar(&serviceAccount, "serviceaccount", "", "service account to use") - - cmdPod.Flags().StringVarP(&remoteKubeConfig, "remoteKubeConfig", "R", "", "launch COSA in a remote cluster") - cmdPod.Flags().StringVarP(&cosaNamespace, "namespace", "N", "", "use a different namespace") - cmdPod.Flags().AddFlagSet(specCommonFlags) - cmdPod.Flags().AddFlagSet(sshFlags) -} - -// runPod is the Jenkins/CI interface into Gangplank. It "mocks" -// the OpenShift buildconfig API with just-enough information to be -// useful. -func runPod(c *cobra.Command, args []string) { - defer cancel() - - cluster := ocp.NewCluster(true) - - if cosaViaPodman { - if cosaPodmanRemote != "" { - if cosaPodmanRemoteSshKey != "" { - os.Setenv(podmanSshKeyEnvVar, cosaPodmanRemoteSshKey) - } - os.Setenv(podmanRemoteEnvVar, cosaPodmanRemote) - - if minioCfgFile == "" { - minioSshRemoteHost = containerHost() - if strings.Contains(minioSshRemoteHost, "@") { - parts := strings.Split(minioSshRemoteHost, "@") - if strings.Contains(parts[1], ":") { - hostparts := strings.Split(parts[1], ":") - port, err := strconv.Atoi(hostparts[1]) - if err != nil { - log.WithError(err).Fatalf("failed to define minio ssh port %s", hostparts[1]) - } - parts[1] = hostparts[0] - minioSshRemotePort = port - } - minioSshRemoteHost = parts[1] - minioSshRemoteUser = parts[0] - minioSshRemoteKey = cosaPodmanRemoteSshKey - } - log.WithFields(log.Fields{ - "remote user": minioSshRemoteUser, - "remote key": cosaPodmanRemoteSshKey, - "remote host": minioSshRemoteHost, - "remote port": minioSshRemotePort, - }).Info("Minio will be forwarded to remote host") - } - - log.WithFields(log.Fields{ - "ssh key": cosaPodmanRemoteSshKey, - "container host": cosaPodmanRemote, - }).Info("Podman container will be executed on a remote host") - } - cluster = ocp.NewCluster(false) - cluster.SetPodman(cosaSrvDir) - } - - if cosaViaPodman || remoteKubeConfig != "" { - if cosaWorkDir == "" { - cosaWorkDir, _ = os.Getwd() - } - if cosaImage == "" { - log.WithField("image", cosaDefaultImage).Info("Using default COSA image") - cosaImage = cosaDefaultImage - } - } - - if remoteKubeConfig != "" { - log.Info("Using a hop pod via a remote cluster") - cluster.SetRemoteCluster(remoteKubeConfig, cosaNamespace) - if cosaWorkDir == "" { - cosaWorkDir, _ = os.Getwd() - } - log.Infof("Logs will written to %s/logs", cosaWorkDir) - } - - clusterCtx := ocp.NewClusterContext(ctx, cluster) - setCliSpec() - - if cosaWorkDirContext { - for _, d := range []string{cosaWorkDir, cosaSrvDir} { - if d == "" { - continue - } - log.WithField("dir", d).Infof("Applying selinux %q content", cosaWorkDirSelinuxLabel) - args := []string{"chcon", "-R", cosaWorkDirSelinuxLabel, d} - cmd := exec.CommandContext(ctx, args[0], args[1:]...) - if err := cmd.Run(); err != nil { - log.WithError(err).Fatalf("failed set dir context on %s", d) - } - } - } - - if remoteKubeConfig != "" { - h := ocp.NewHopPod(clusterCtx, cosaImage, serviceAccount, cosaWorkDir, &spec) - term := make(chan bool) - - sig := make(chan os.Signal, 256) - signal.Notify(sig, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGHUP) - go func() { - <-sig - term <- true - }() - defer func() { - term <- true - }() - - err := h.WorkerRunner(term, nil) - if err != nil { - log.WithError(err).Fatal("failed remote exection of a pod") - } - } - - // Run in cluster or podman mode - pb, err := ocp.NewPodBuilder(clusterCtx, cosaImage, serviceAccount, cosaWorkDir, &spec) - if err != nil { - log.Fatalf("failed to define builder pod: %v", err) - } - - if err := pb.Exec(clusterCtx); err != nil { - log.Fatalf("failed to execute CI builder: %v", err) - } -} - -func containerHost() string { - containerHost, ok := os.LookupEnv(podmanRemoteEnvVar) - if !ok { - return "" - } - if !strings.HasPrefix(containerHost, "ssh://") { - return "" - } - parts := strings.Split(strings.TrimPrefix(containerHost, "ssh://"), "/") - return parts[0] -} diff --git a/gangplank/cmd/gangway/main.go b/gangplank/cmd/gangway/main.go deleted file mode 100644 index 90058d86..00000000 --- a/gangplank/cmd/gangway/main.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -/* - Gangway is the Gangplank worker. -*/ - -import ( - "context" - - "github.com/coreos/gangplank/internal/ocp" - log "github.com/sirupsen/logrus" -) - -func main() { - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - defer ctx.Done() - - cluster := ocp.NewCluster(true) - clusterCtx := ocp.NewClusterContext(ctx, cluster) - - b, err := ocp.NewBuilder(clusterCtx) - if err != nil { - log.Fatal("Failed to find the build environment.") - } - - if err := b.Exec(clusterCtx); err != nil { - log.WithFields(log.Fields{ - "err": err, - }).Fatal("Failed to prepare environment.") - } - -} diff --git a/gangplank/internal/ocp/bc.go b/gangplank/internal/ocp/bc.go deleted file mode 100644 index 0984cab5..00000000 --- a/gangplank/internal/ocp/bc.go +++ /dev/null @@ -1,894 +0,0 @@ -/* - Main interface into OCP Build targets. - - This supports running via: - - generic Pod with a Service Account - - an OpenShift buildConfig - -*/ - -package ocp - -import ( - "context" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "os/signal" - "path/filepath" - "reflect" - "sort" - "strings" - "sync" - "syscall" - "time" - - "github.com/coreos/coreos-assembler-schema/cosa" - "github.com/coreos/gangplank/internal/spec" - "github.com/minio/minio-go/v7" - buildapiv1 "github.com/openshift/api/build/v1" - log "github.com/sirupsen/logrus" -) - -// srvBucket is the name of the bucket to use for remote -// files being served up -var srvBucket = "source" - -// buildConfig is a builder. -var _ Builder = &buildConfig{} - -// stageDependencyTimeOut is the length of time to wait for a stage's dependencies. -var stageDependencyTimeOut = 1 * time.Hour - -func init() { - buildJSONCodec = buildCodecFactory.LegacyCodec(buildapiv1.SchemeGroupVersion) -} - -// buildConfig represent the input into a buildConfig. -type buildConfig struct { - JobSpecURL string `envVar:"COSA_JOBSPEC_URL"` - JobSpecRef string `envVar:"COSA_JOBSPEC_REF"` - JobSpecFile string `envVar:"COSA_JOBSPEC_FILE"` - CosaCmds string `envVar:"COSA_CMDS"` - - // Information about the parent pod - PodName string `envVar:"COSA_POD_NAME"` - PodIP string `envVar:"COSA_POD_IP"` - PodNameSpace string `envVar:"COSA_POD_NAMESPACE"` - - // HostIP is the kubernetes IP address of the running pod. - HostIP string - HostPod string - - // Internal copy of the JobSpec - JobSpec spec.JobSpec - - ClusterCtx ClusterContext -} - -// newBC accepts a context and returns a buildConfig -func newBC(ctx context.Context, c *Cluster) (*buildConfig, error) { - var v buildConfig - rv := reflect.TypeOf(v) - for i := 0; i < rv.NumField(); i++ { - tag := rv.Field(i).Tag.Get(ocpStructTag) - if tag == "" { - continue - } - ev, found := os.LookupEnv(tag) - if found { - reflect.ValueOf(&v).Elem().Field(i).SetString(ev) - } - } - - // Init the OpenShift Build API Client. - if err := ocpBuildClient(); err != nil { - log.WithError(err).Error("Failed to initalized the OpenShift Build API Client") - return nil, err - } - - // Add the ClusterContext to the BuildConfig - v.ClusterCtx = NewClusterContext(ctx, *c.toKubernetesCluster()) - ac, ns, kubeErr := GetClient(v.ClusterCtx) - if kubeErr != nil { - log.WithError(kubeErr).Info("Running without a cluster client") - } else if ac != nil { - v.HostPod = fmt.Sprintf("%s-%s-build", - apiBuild.Annotations[buildapiv1.BuildConfigAnnotation], - apiBuild.Annotations[buildapiv1.BuildNumberAnnotation], - ) - - log.Info("Querying for host IP") - var e error - v.HostIP, e = getPodIP(v.ClusterCtx, ac, ns, getHostname()) - if e != nil { - log.WithError(e).Info("failed to query for hostname") - } - - log.WithFields(log.Fields{ - "buildconfig/name": apiBuild.Annotations[buildapiv1.BuildConfigAnnotation], - "buildconfig/number": apiBuild.Annotations[buildapiv1.BuildNumberAnnotation], - "podname": v.HostPod, - "podIP": v.HostIP, - }).Info("found build.openshift.io/buildconfig identity") - } - - if _, err := os.Stat(cosaSrvDir); os.IsNotExist(err) { - return nil, fmt.Errorf("context dir %q does not exist", cosaSrvDir) - } - - if err := os.Chdir(cosaSrvDir); err != nil { - return nil, fmt.Errorf("failed to switch to context dir: %s: %v", cosaSrvDir, err) - } - - // Locate the jobspec from local input OR from a remote repo. - jsF := spec.DefaultJobSpecFile - if v.JobSpecFile != "" { - jsF = v.JobSpecFile - } - v.JobSpecFile = jsF - jsF = filepath.Join(cosaSrvDir, jsF) - js, err := spec.JobSpecFromFile(jsF) - if err != nil { - v.JobSpec = js - } else { - njs, err := spec.JobSpecFromRepo(v.JobSpecURL, v.JobSpecFile, filepath.Base(jsF)) - if err != nil { - v.JobSpec = njs - } - } - - // Set default bucket if not defined - if v.JobSpec.Minio.Bucket == "" && !v.JobSpec.Job.StrictMode { - v.JobSpec.Minio.Bucket = "builder" - } - - log.Info("Running Pod in buildconfig mode.") - return &v, nil -} - -// Exec executes the command using the closure for the commands -func (bc *buildConfig) Exec(ctx ClusterContext) (err error) { - curD, _ := os.Getwd() - defer func(c string) { _ = os.Chdir(c) }(curD) - - if err := os.Chdir(cosaSrvDir); err != nil { - return err - } - - // Define, but do not start minio. - m := newMinioServer(bc.JobSpec.Minio.ConfigFile) - m.dir = cosaSrvDir - if !m.ExternalServer { - if mf := getSshMinioForwarder(&bc.JobSpec); mf != nil { - m.overSSH = mf - m.Host = "127.0.0.1" - } - } - - // returnTo informs the workers where to send their bits - returnTo := &Return{ - Minio: m, - Bucket: bc.JobSpec.Minio.Bucket, - KeyPrefix: bc.JobSpec.Minio.KeyPrefix, - } - - // Prepare the remote files. - var remoteFiles []*RemoteFile - r, err := bc.ocpBinaryInput(m) - if err != nil { - return fmt.Errorf("failed to process binary input: %w", err) - } - remoteFiles = append(remoteFiles, r...) - defer func() { _ = os.RemoveAll(filepath.Join(cosaSrvDir, sourceSubPath)) }() - - // Discover the stages and render each command into a script. - r, err = bc.discoverStages(m) - if err != nil { - return fmt.Errorf("failed to discover stages: %w", err) - } - remoteFiles = append(remoteFiles, r...) - - if len(bc.JobSpec.Stages) == 0 { - log.Info(` -No work to do. Please define one of the following: - - 'COSA_CMDS' envVar with the commands to execute - - Jobspec stages in your JobSpec file - - Provide files ending in .cosa.sh - -File can be provided in the Git Tree or by the OpenShift -binary build interface.`) - return nil - } - - // Start minio after all the setup. Each directory is an implicit - // bucket and files, are implicit keys. - // - // Job Control: - // terminate channel: uses to tell workFunctions to ceases - // errorCh channel: workFunctions report errors through this channel - // when a error is recieved over the channel, a terminate is signaled - // sig channel: this watches for sigterm and interrupts, which will - // signal a terminate. (i.e. sigterm or crtl-c) - // - // The go-routine will run until it recieves a terminate itself. - // - errorCh := make(chan error) - terminate := make(chan bool) - if m.overSSH == nil { - if err := m.start(ctx); err != nil { - return fmt.Errorf("failed to start Minio: %w", err) - } - } else { - if err := m.startMinioAndForwardOverSSH(ctx, terminate, errorCh); err != nil { - return fmt.Errorf("failed to start Minio: %w", err) - } - } - defer m.Kill() - - // Set the cosa builds IO Backend to minio. - mc, err := m.client() - if err != nil { - return fmt.Errorf("failed to get minio client") - } - if err := m.ensureBucketExists(ctx, bc.JobSpec.Minio.Bucket); err != nil { - return fmt.Errorf("failed to ensure '%s' bucket exists: %v", bc.JobSpec.Minio.Bucket, err) - } - - // Set the cosa backend to minio. This allows for Gangplank to use - // either a local directory (via minio) or remote object store (AWS or minio) - // as the artifact store. - if err := cosa.SetIOBackendMinio(ctx, mc, bc.JobSpec.Minio.Bucket, bc.JobSpec.Minio.KeyPrefix); err != nil { - return err - } - - // Find the last build, if any. - lastBuild, _, err := cosa.ReadBuild("", "", cosa.BuilderArch()) - if err == nil { - keyPath := filepath.Join(lastBuild.BuildID, cosa.BuilderArch()) - l := log.WithFields(log.Fields{ - "build": lastBuild.BuildID, - }) - l.Info("found prior build") - remoteFiles = append( - remoteFiles, - getBuildMeta(lastBuild.BuildID, keyPath, m, l, &bc.JobSpec.Minio)..., - ) - - } else { - lastBuild = new(cosa.Build) - log.Infof("no prior build found for arch: %s", cosa.BuilderArch()) - } - - // Copy any other builds requested by the user. - if bc.JobSpec.CopyBuild != "" { - copyBuild, _, err := cosa.ReadBuild("", bc.JobSpec.CopyBuild, cosa.BuilderArch()) - if err != nil { - return fmt.Errorf("Failed to find build specified by CopyBuild: %v", bc.JobSpec.CopyBuild) - } - l := log.WithFields(log.Fields{ - "build": copyBuild.BuildID, - }) - l.Info("copying requested build") - keyPath := filepath.Join(copyBuild.BuildID, cosa.BuilderArch()) - remoteFiles = append( - remoteFiles, - getBuildMeta(copyBuild.BuildID, keyPath, m, l, &bc.JobSpec.Minio)..., - ) - } - - // Dump the jobspec - log.Infof("Using JobSpec definition:") - if err := bc.JobSpec.WriteYAML(log.New().Out); err != nil { - return err - } - - // Create a cancelable context from the core context. - podCtx, cancel := context.WithCancel(ctx) - defer cancel() - - type workFunction func(terminate termChan) error - workerFuncs := make(map[int][]workFunction) - - // Range over the stages and create workFunction, which is added to the - // workerFuncs. Each workFunction is executed as a go routine that begins - // work as soon as the `build_dependencies` are available. - for idx, ss := range bc.JobSpec.Stages { - - // copy the stage to prevent corruption - // using ss directly has proven to lead to memory corruptions (yikes!) - s, err := ss.DeepCopy() - if err != nil { - return err - } - - l := log.WithFields(log.Fields{ - "stage": s.ID, - "require_artifacts": s.RequireArtifacts, - }) - - cpod, err := NewCosaPodder(podCtx, apiBuild, idx) - if err != nil { - l.WithError(err).Error("Failed to create pod definition") - return err - } - - l.Info("Pod definition created") - - // ready spawns a go-routine that writes the return channel - // when the stage's dependencies have been meet. - ready := func(ws *workSpec, terminate <-chan bool) <-chan bool { - out := make(chan bool) - - foundNewBuild := false - buildID := lastBuild.BuildID - - // TODO: allow for selectable build id, instead of default - // to the latest build ID. - go func(out chan<- bool) { - check := func() bool { - - build, foundRemoteFiles, err := getStageFiles(buildID, l, m, lastBuild, &s, &bc.JobSpec.Minio) - if build != nil && buildID != build.BuildID && !foundNewBuild { - l.WithField("build ID", build.BuildID).Info("Using new buildID for lifetime of this build") - buildID = build.BuildID - } - if err == nil { - ws.RemoteFiles = append(remoteFiles, foundRemoteFiles...) - out <- true - return true - } - return false - } - - for { - if check() { - l.Debug("all dependencies for stage have been meet") - return - } - // Wait for the next check or terminate. - select { - case <-terminate: - return - case <-time.After(15 * time.Second): - return - } - } - }(out) - - return out - } - - // anonFunc performs the actual work.. - anonFunc := func(terminate termChan) error { - ws := &workSpec{ - APIBuild: apiBuild, - ExecuteStages: []string{s.ID}, - JobSpec: bc.JobSpec, - RemoteFiles: remoteFiles, - Return: returnTo, - } - - select { - case <-terminate: - return errors.New("terminate signal recieved, aborting stage") - case <-time.After(stageDependencyTimeOut): - return errors.New("required artifacts never appeared") - case ok := <-ready(ws, terminate): - if !ok { - return fmt.Errorf("%s failed to become ready", s.ID) - } - - l.Info("Worker dependences have been defined") - eVars, err := ws.getEnvVars() - if err != nil { - return err - } - - l.Info("Executing worker pod") - if err := cpod.WorkerRunner(terminate, eVars); err != nil { - return fmt.Errorf("%s failed: %w", s.ID, err) - } - } - return nil - } - - // If there is no default execution order, default to 2. The default - // is due the short-hand defaults in stage.go that asssigns certain short-hands - // to certain execution groups. - eOrder := s.ExecutionOrder - if eOrder == 0 { - eOrder = 2 - } - workerFuncs[eOrder] = append(workerFuncs[eOrder], anonFunc) - } - - // Sort the ordering of the workerFuncs - var order []int - for key := range workerFuncs { - order = append(order, key) - } - sort.Ints(order) - - // Watch the channels for signals to terminate - errored := false - go func() { - sig := make(chan os.Signal, 256) - signal.Notify(sig, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGHUP) - - term := func() { - defer func() { - recover() // nolint - }() - terminate <- true - } - - for { - select { - case err, ok := <-errorCh: - if err != nil { - if err != nil { - errored = true - log.WithError(err).Error("Stage sent error") - term() - } - } - if !ok { - return - } - case <-ctx.Done(): - log.Warning("Received cancellation") - term() - case s := <-sig: - log.Warningf("Received signal %s", s) - term() - case die, ok := <-terminate: - if !ok || die { - log.Debug("Watch go-routine finished") - return - } - } - // Let the channel settle - time.Sleep(1 * time.Second) - } - }() - - // For each execution group, launch all workers and wait for the group - // to complete. If a workerFunc fails, then bail as soon as possible. - for _, idx := range order { - l := log.WithField("execution group", idx) - wg := &sync.WaitGroup{} - for _, v := range workerFuncs[idx] { - wg.Add(1) - go func(v workFunction) { - defer func() { - wg.Done() - log.Debug("execution done") - }() - errorCh <- v(terminate) - }(v) - } - wg.Wait() - l.Debug("done with execution group") - } - - close(terminate) - - if errored { - return fmt.Errorf("process failed") - } - return nil -} - -func copyFile(src, dest string) error { - srcF, err := os.Open(src) - if err != nil { - return err - } - defer srcF.Close() - - destF, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644) - if err != nil { - return err - } - defer destF.Close() - - if _, err := io.Copy(destF, srcF); err != nil { - return err - } - return err -} - -// discoverStages supports the envVar and *.cosa.sh scripts as implied stages. -// The envVar stage will be run first, followed by the `*.cosa.sh` scripts. -func (bc *buildConfig) discoverStages(m *minioServer) ([]*RemoteFile, error) { - var remoteFiles []*RemoteFile - - if bc.JobSpec.Job.StrictMode { - log.Info("Job strict mode is set, skipping automated stage discovery.") - return nil, nil - } - log.Info("Strict mode is off: envVars and *.cosa.sh files are implied stages.") - - sPrefix := "/bin/bash -xeu -o pipefail %s" - // Add the envVar commands - if bc.CosaCmds != "" { - bc.JobSpec.Stages = append( - bc.JobSpec.Stages, - spec.Stage{ - Description: "envVar defined commands", - DirectExec: true, - Commands: []string{ - fmt.Sprintf(sPrefix, bc.CosaCmds), - }, - ID: "envVar", - }, - ) - } - - // Add discovered *.cosa.sh scripts into a single stage. - // *.cosa.sh scripts are all run on the same worker pod. - scripts := []string{} - foundScripts, _ := filepath.Glob("*.cosa.sh") - for _, s := range foundScripts { - dn := filepath.Base(s) - destPath := filepath.Join(cosaSrvDir, srvBucket, dn) - if err := copyFile(s, destPath); err != nil { - return remoteFiles, err - } - - // We _could_ embed the scripts directly into the jobspec's stage - // but the jospec is embedded as a envVar. To avoid runing into the - // 32K character limit and we have an object store running, we'll just use - // that. - remoteFiles = append( - remoteFiles, - &RemoteFile{ - Bucket: srvBucket, - Object: dn, - Minio: m, - }, - ) - - // Add the script to the command interface. - scripts = append( - scripts, - fmt.Sprintf(sPrefix, filepath.Join(cosaSrvDir, srvBucket, dn)), - ) - } - if len(scripts) > 0 { - bc.JobSpec.Stages = append( - bc.JobSpec.Stages, - spec.Stage{ - Description: "*.cosa.sh scripts", - DirectExec: true, - Commands: scripts, - ID: "cosa.sh", - }, - ) - } - return remoteFiles, nil -} - -// ocpBinaryInput decompresses the binary input. If the binary input is a tarball -// with an embedded JobSpec, its extracted, read and used. -func (bc *buildConfig) ocpBinaryInput(m *minioServer) ([]*RemoteFile, error) { - var remoteFiles []*RemoteFile - bin, err := recieveInputBinary() - if err != nil { - return nil, err - } - if bin == "" { - return nil, nil - } - - if strings.HasSuffix(bin, "source.bin") { - f, err := os.Open(bin) - if err != nil { - return nil, err - } - - if err := decompress(f, cosaSrvDir); err != nil { - return nil, err - } - dir, key := filepath.Split(bin) - bucket := filepath.Base(dir) - r := &RemoteFile{ - Bucket: bucket, - Object: key, - Minio: m, - Compressed: true, - } - remoteFiles = append(remoteFiles, r) - log.Info("Binary input will be served to remote workers") - } - - // Look for a jobspec in the binary payload. - jsFile := "" - candidateSpec := filepath.Join(cosaSrvDir, bc.JobSpecFile) - _, err = os.Stat(candidateSpec) - if err == nil { - log.Info("Found jobspec file in binary payload.") - jsFile = candidateSpec - } - - // Treat any yaml files as jobspec's. - if strings.HasSuffix(apiBuild.Spec.Source.Binary.AsFile, "yaml") { - jsFile = bin - } - - // Load the JobSpecFile - if jsFile != "" { - log.WithField("jobspec", bin).Info("treating source as a jobspec") - js, err := spec.JobSpecFromFile(jsFile) - if err != nil { - return nil, err - } - log.Info("Using OpenShift provided JobSpec") - bc.JobSpec = js - - if bc.JobSpec.Recipe.GitURL != "" { - log.Info("Jobpsec references a git repo -- ignoring buildconfig reference") - apiBuild.Spec.Source.Git = new(buildapiv1.GitBuildSource) - apiBuild.Spec.Source.Git.URI = bc.JobSpec.Recipe.GitURL - apiBuild.Spec.Source.Git.Ref = bc.JobSpec.Recipe.GitRef - } - } - return remoteFiles, nil -} - -// getBuildMeta searches a path for all build meta files and creates remoteFiles -// for them. The keyPathBase is the relative path for the object. -func getBuildMeta(jsonPath, keyPathBase string, m *minioServer, l *log.Entry, mcfg *spec.Minio) []*RemoteFile { - var metas []*RemoteFile - - mc, err := m.client() - if err != nil { - log.WithError(err).Warn("failed to get client") - return nil - } - - bucket, searchPath := getBucketObjectPath(mcfg, jsonPath) - - v := mc.ListObjects(context.Background(), bucket, - minio.ListObjectsOptions{ - Recursive: true, - Prefix: searchPath, - }, - ) - for { - info, ok := <-v - if !ok { - break - } - if strings.HasSuffix(info.Key, "/") { - continue - } - - n := filepath.Base(info.Key) - if !isKnownBuildMeta(n) { - continue - } - - metas = append( - metas, - &RemoteFile{ - Bucket: bucket, - Minio: m, - Object: info.Key, - ForcePath: filepath.Join("/srv/", "builds", getKeyLocalPath(mcfg, info.Key)), - }, - ) - l.WithFields(log.Fields{ - "bucket": bucket, - "key": info.Key, - }).Info("Included metadata") - } - - bucket, obj := getBucketObjectPath(mcfg, "builds.json") - if _, err := mc.StatObject(context.Background(), bucket, obj, minio.StatObjectOptions{}); err == nil { - metas = append( - metas, - &RemoteFile{ - Minio: m, - Bucket: bucket, - Object: obj, - ForcePath: "/srv/builds/builds.json", - }, - ) - } - - return metas -} - -// getStageFiles returns the newest build and RemoteFiles for the stage. -// Depending on the stages dependencies, it will ensure that all meta-data -// and artifacts are send. If the stage requires/requests the caches, it will be -// included in the RemoteFiles. -func getStageFiles(buildID string, - l *log.Entry, m *minioServer, lastBuild *cosa.Build, s *spec.Stage, mcfg *spec.Minio) (*cosa.Build, []*RemoteFile, error) { - var remoteFiles []*RemoteFile - var keyPathBase string - - errMissingArtifactDependency := errors.New("missing an artifact depenedency") - - // For _each_ stage, we need to check if a meta.json exists. - // mBuild - *cosa.Build representing meta.json - mBuild, _, err := cosa.ReadBuild("", buildID, "") - if err != nil { - l.Info("No build history found") - } - - // Handle {Require,Request}{Cache,CacheRepo} - includeCache := func(tarball string, required, requested bool) error { - if !required && !requested { - return nil - } - - bucket, obj := getBucketObjectPath(mcfg, "cache", tarball) - cacheFound := m.Exists(bucket, obj) - if !cacheFound { - if required { - l.WithField("cache", tarball).Debug("Does not exists yet") - return errMissingArtifactDependency - } - return nil - } - - remoteFiles = append( - remoteFiles, - &RemoteFile{ - Bucket: bucket, - Compressed: true, - ForceExtractPath: "/", // will extract to /srv/cache - Minio: m, - Object: obj, - }) - return nil - } - if err := includeCache(cacheTarballName, s.RequireCache, s.RequestCache); err != nil { - return nil, nil, errMissingArtifactDependency - } - if err := includeCache(cacheRepoTarballName, s.RequireCacheRepo, s.RequestCacheRepo); err != nil { - return nil, nil, errMissingArtifactDependency - } - - if mBuild != nil { - // If the buildID is not known AND the worker finds a build ID, - // then a new build has appeared. - if buildID == "" { - buildID = mBuild.BuildID - l = log.WithField("buildID", buildID) - log.WithField("buildID", mBuild.BuildID).Info("Found new build ID") - } - - // base of the keys to fetch from minio "/" - keyPathBase = filepath.Join(buildID, cosa.BuilderArch()) - - // Locate build meta data - if lastBuild.BuildID != mBuild.BuildID { - remoteFiles = append( - remoteFiles, - getBuildMeta(buildID, keyPathBase, m, l, mcfg)..., - ) - } - } - - // If no artfiacts are required we can skip checking for artifacts. - if mBuild == nil { - if len(s.RequireArtifacts) > 0 { - l.Debug("Waiting for build to appear") - return nil, nil, errMissingArtifactDependency - } - } - - // addArtifact is a helper function for adding artifacts - addArtifact := func(artifact string) error { - bArtifact, err := mBuild.GetArtifact(artifact) - if err != nil { - return errMissingArtifactDependency - } - - // get the Minio relative path for the object - // the full path needs to be broken in to // - bucket, key := getBucketObjectPath(mcfg, keyPathBase, filepath.Base(bArtifact.Path)) - - // Check if the remote server has this - if !m.Exists(bucket, key) { - return errMissingArtifactDependency - } - - r := &RemoteFile{ - Artifact: bArtifact, - Bucket: bucket, - Minio: m, - Object: key, - ForcePath: filepath.Join("/srv", "builds", getKeyLocalPath(mcfg, key)), - } - remoteFiles = append(remoteFiles, r) - return nil - } - - // Handle optional artifacts - for _, artifact := range s.RequestArtifacts { - if err = addArtifact(artifact); err != nil { - l.WithField("artifact", artifact).Debug("skipping optional artifact") - } - } - - // Handle the required artifacts - foundCount := 0 - for _, artifact := range s.RequireArtifacts { - if err := addArtifact(artifact); err != nil { - l.WithField("artifact", artifact).Warn("required artifact has not appeared yet") - return mBuild, nil, errMissingArtifactDependency - } - foundCount++ - } - - if len(s.RequireArtifacts) != foundCount { - return mBuild, nil, errMissingArtifactDependency - } - - // Create a single tarball of from all the arbitrary overrides, then place in the cache directory. - if len(s.Overrides) > 0 { - overrideToken, _ := randomString(10) - tmpD, err := ioutil.TempDir("", "override") - if err != nil { - return nil, nil, err - } - defer os.RemoveAll(tmpD) //nolint - - for _, override := range s.Overrides { - l.WithField("override", override.URI).Info("Processing Override") - if err := override.Fetch(l, tmpD, decompress); err != nil { - return nil, nil, fmt.Errorf("failed to write remote file: %v", err) - } - } - - bucket, key := getBucketObjectPath(mcfg, "cache", fmt.Sprintf("overrides-%s.tar.gz", overrideToken)) - if err := uploadPathAsTarBall( - context.Background(), bucket, key, ".", tmpD, false, - &Return{Minio: m}); err != nil { - return nil, nil, err - } - remoteFiles = append( - remoteFiles, - &RemoteFile{ - Bucket: mcfg.Bucket, - Compressed: true, - ForceExtractPath: "/srv", - Minio: m, - Object: key, - }, - ) - } - - for _, rf := range remoteFiles { - l.WithFields(log.Fields{ - "bucket": rf.Bucket, - "object": rf.Object, - }).Debug("will request") - } - - return mBuild, remoteFiles, nil - -} - -// getBucketObjectPath returns the bucket and the approriate path much like -// filepath.Join does, but for remote objects -func getBucketObjectPath(m *spec.Minio, parts ...string) (string, string) { - path := filepath.Join(parts...) - bucket := m.Bucket - if m.KeyPrefix != "" { - path = filepath.Join(m.KeyPrefix, path) - } - return bucket, path -} - -// getKeyLocalPath strips off the -func getKeyLocalPath(m *spec.Minio, key string) string { - return strings.TrimPrefix(key, m.KeyPrefix) -} diff --git a/gangplank/internal/ocp/bc_ci_test.go b/gangplank/internal/ocp/bc_ci_test.go deleted file mode 100644 index e5237510..00000000 --- a/gangplank/internal/ocp/bc_ci_test.go +++ /dev/null @@ -1,34 +0,0 @@ -// +ci -package ocp - -/* - Since our builds do CI testing in OpenShift for this - set of tests we have to force not being in the cluster. -*/ - -import ( - "context" - "testing" - - log "github.com/sirupsen/logrus" -) - -func init() { - log.Debug("CI mode enabled: forcing Kubernetes out-of-cluster errors") -} - -func TestNoEnv(t *testing.T) { - if _, err := newBC(context.Background(), &Cluster{inCluster: false}); err != ErrNoOCPBuildSpec { - t.Errorf("failed to raise error\n want: %v\n got: %v", ErrInvalidOCPMode, err) - } -} - -func TestNoOCP(t *testing.T) { - newO, err := newBC(context.Background(), &Cluster{inCluster: false}) - if newO != nil { - t.Errorf("should return nil") - } - if err == nil { - t.Errorf("expected error") - } -} diff --git a/gangplank/internal/ocp/bc_test.go b/gangplank/internal/ocp/bc_test.go deleted file mode 100644 index 5352790b..00000000 --- a/gangplank/internal/ocp/bc_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package ocp - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "testing" -) - -const testDataFile = "build.json" - -func TestOCPBuild(t *testing.T) { - tmpd, _ := ioutil.TempDir("", "test") - defer os.RemoveAll(tmpd) - cosaSrvDir = tmpd - - bData, err := ioutil.ReadFile(testDataFile) - if err != nil { - t.Errorf("failed to read %s: %v", testDataFile, err) - } - - env := map[string]string{ - "BUILD": string(bData), - "COSA_CMDS": "cosa init", - } - for k, v := range env { - os.Setenv(k, v) - } - defer func() { - for k := range env { - os.Unsetenv(k) - } - }() - - c := Cluster{inCluster: false} - newO, err := newBC(context.Background(), &c) - if err != nil { - t.Errorf("failed to read OCP envvars: %v", err) - } - if newO == nil { - t.Errorf("failed to get API build") - } else if newO.CosaCmds != "cosa init" { - t.Errorf("cosa commands not set") - } -} - -func TestGetPushTagless(t *testing.T) { - tests := []struct { - in string - registry string - path string - }{ - { - in: "registry.foo:500/bar/baz/bin:tagged", - registry: "registry.foo:500", - path: "bar/baz/bin", - }, - { - in: "registry.foo/bar/baz/bin:tagged", - registry: "registry.foo", - path: "bar/baz/bin", - }, - } - for idx, v := range tests { - t.Run(fmt.Sprintf("test-%d", idx), func(t *testing.T) { - reg, path := getPushTagless(v.in) - if reg != v.registry { - t.Errorf("registry:\n got: %s\n want: %s\n", reg, v.registry) - } - if path != v.path { - t.Errorf("registry:\n got: %s\n want: %s\n", path, v.path) - } - }) - - } -} diff --git a/gangplank/internal/ocp/build.json b/gangplank/internal/ocp/build.json deleted file mode 100644 index 4b865199..00000000 --- a/gangplank/internal/ocp/build.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "kind": "Build", - "apiVersion": "build.openshift.io/v1", - "metadata": { - "name": "r-master-19", - "namespace": "example", - "selfLink": "/apis/build.openshift.io/v1/namespaces/example/builds/cosa-runner-master-19", - "uid": "9fd308ea-0586-11eb-98b7-fa163e4ff028", - "resourceVersion": "165628275", - "creationTimestamp": "2020-10-03T14:42:13Z", - "labels": { - "app": "cosa", - "buildconfig": "cosa-runner-master", - "openshift.io/build-config.name": "cosa-runner-master", - "openshift.io/build.start-policy": "Parallel", - "template": "cosa-template" - }, - "annotations": { - "openshift.io/build-config.name": "cosa-runner-master", - "openshift.io/build.number": "19" - }, - "ownerReferences": [ - { - "apiVersion": "build.openshift.io/v1", - "kind": "BuildConfig", - "name": "cosa-runner-master", - "uid": "6e824268-04dd-11eb-98b7-fa163e4ff028", - "controller": true - } - ] - }, - "spec": { - "serviceAccount": "jenkins-kvm", - "source": { - "type": "Git", - "git": { - "uri": "https://github.com/coreos/fedora-coreos-config", - "ref": "testing-devel" - } - }, - "strategy": { - "type": "Custom", - "customStrategy": { - "from": { - "kind": "DockerImage", - "name": "docker-registry.default.svc:5000/example/coreos-assembler@sha256:4a16bf77decab2e65485ec9cb529063961bdf0e574d651b42d9ebaae60354e98" - }, - "pullSecret": { - "name": "jenkins-kvm-dockercfg-g77h5" - }, - "env": [ - { - "name": "OCP_CUSTOM_BUILDER", - "value": "1" - }, - { - "name": "OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE", - "value": "docker-registry.default.svc:5000/example/coreos-assembler@sha256:4a16bf77decab2e65485ec9cb529063961bdf0e574d651b42d9ebaae60354e98" - }, - { - "name": "COSA_CMDS", - "value": "cosa init" - } - ] - } - }, - "output": {}, - "resources": {}, - "postCommit": {}, - "nodeSelector": null, - "triggeredBy": [ - { - "message": "Manually triggered" - } - ] - }, - "status": { - "phase": "New", - "config": { - "kind": "BuildConfig", - "namespace": "example", - "name": "cosa-runner-master" - }, - "output": {} - } -} diff --git a/gangplank/internal/ocp/client.go b/gangplank/internal/ocp/client.go deleted file mode 100644 index 94bed310..00000000 --- a/gangplank/internal/ocp/client.go +++ /dev/null @@ -1,36 +0,0 @@ -package ocp - -import ( - "os" - - log "github.com/sirupsen/logrus" -) - -// Builder implements the Build -type Builder interface { - Exec(ctx ClusterContext) error -} - -// cosaSrvDir is where the build directory should be. When the build API -// defines a contextDir then it will be used. In most cases this should be /srv -var cosaSrvDir = defaultContextDir - -// NewBuilder returns a Builder. NewBuilder determines what -// "Builder" to return by first trying Worker and then an OpenShift builder. -func NewBuilder(ctx ClusterContext) (Builder, error) { - inCluster := true - if _, ok := os.LookupEnv(localPodEnvVar); ok { - log.Infof("EnvVar %s defined, using local pod mode", localPodEnvVar) - inCluster = false - } - - ws, err := newWorkSpec(ctx) - if err == nil { - return ws, nil - } - bc, err := newBC(ctx, &Cluster{inCluster: inCluster}) - if err == nil { - return bc, nil - } - return nil, ErrNoWorkFound -} diff --git a/gangplank/internal/ocp/conditions.go b/gangplank/internal/ocp/conditions.go deleted file mode 100644 index bdd34299..00000000 --- a/gangplank/internal/ocp/conditions.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ocp - -import ( - "fmt" - - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/watch" -) - -// ErrPodCompleted is returned by PodRunning or PodContainerRunning to indicate that -// the pod has already reached completed state. -var ErrPodCompleted = fmt.Errorf("pod ran to completion") - -// PodRunning returns true if the pod is running, false if the pod has not yet reached running state, -// returns ErrPodCompleted if the pod has run to completion, or an error in any other case. -func PodRunning(event watch.Event) (bool, error) { - switch event.Type { - case watch.Deleted: - return false, errors.NewNotFound(schema.GroupResource{Resource: "pods"}, "") - } - switch t := event.Object.(type) { - case *v1.Pod: - switch t.Status.Phase { - case v1.PodRunning: - return true, nil - case v1.PodFailed, v1.PodSucceeded: - return false, ErrPodCompleted - } - } - return false, nil -} - -// PodCompleted returns true if the pod has run to completion, false if the pod has not yet -// reached running state, or an error in any other case. -func PodCompleted(event watch.Event) (bool, error) { - switch event.Type { - case watch.Deleted: - return false, errors.NewNotFound(schema.GroupResource{Resource: "pods"}, "") - } - switch t := event.Object.(type) { - case *v1.Pod: - switch t.Status.Phase { - case v1.PodFailed, v1.PodSucceeded: - return true, nil - } - } - return false, nil -} - -// ServiceAccountHasSecrets returns true if the service account has at least one secret, -// false if it does not, or an error. -func ServiceAccountHasSecrets(event watch.Event) (bool, error) { - switch event.Type { - case watch.Deleted: - return false, errors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, "") - } - switch t := event.Object.(type) { - case *v1.ServiceAccount: - return len(t.Secrets) > 0, nil - } - return false, nil -} diff --git a/gangplank/internal/ocp/const.go b/gangplank/internal/ocp/const.go deleted file mode 100644 index 90ecf9e0..00000000 --- a/gangplank/internal/ocp/const.go +++ /dev/null @@ -1,28 +0,0 @@ -package ocp - -const ( - // ocpStructTag is the struct tag used to read in - // OCPBuilder from envvars - ocpStructTag = "envVar" - - // defaultContextdir is the default path to use for a build - defaultContextDir = "/srv" - - // secretLabelName is the label to search for secrets to automatically use - secretLabelName = "coreos-assembler.coreos.com/secret" - - // cosaSrvCache is the location of the cache files - cosaSrvCache = "/srv/cache" - - // cosaSrvTmpRepo is the location the repo files - cosaSrvTmpRepo = "/srv/tmp/repo" - - // cacheTarballName is the name of the file used when Stage.{Require,Return}Cache is true - cacheTarballName = "cache.tar.gz" - - // cacheRepoTarballName is the name of the file used when Stage.{Require,Return}RepoCache is true - cacheRepoTarballName = "repo.tar.gz" - - // cacheBucket is used for storing the cache - cacheBucket = "cache" -) diff --git a/gangplank/internal/ocp/cosa-pod-s390x.go b/gangplank/internal/ocp/cosa-pod-s390x.go deleted file mode 100644 index 5e9ecce0..00000000 --- a/gangplank/internal/ocp/cosa-pod-s390x.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build s390x - -package ocp - -import resource "k8s.io/apimachinery/pkg/api/resource" - -/* - s390x/Z is a bit greedy... -*/ - -func init() { - baseMem = *resource.NewQuantity(12*1024*1024*1024, resource.BinarySI) -} diff --git a/gangplank/internal/ocp/cosa-pod.go b/gangplank/internal/ocp/cosa-pod.go deleted file mode 100644 index 98952f64..00000000 --- a/gangplank/internal/ocp/cosa-pod.go +++ /dev/null @@ -1,624 +0,0 @@ -package ocp - -import ( - "bufio" - "context" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - buildapiv1 "github.com/openshift/api/build/v1" - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - resource "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes" -) - -const ( - kvmLabel = "devices.kubevirt.io/kvm" - localPodEnvVar = "COSA_FORCE_NO_CLUSTER" -) - -var ( - gangwayCmd = "/usr/bin/gangway" - - // volumes are the volumes used in all pods created - volumes = []v1.Volume{ - { - Name: "srv", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{ - Medium: "", - }, - }, - }, - { - Name: "pki-trust", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{ - Medium: "", - }, - }, - }, - { - Name: "pki-anchors", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{ - Medium: "", - }, - }, - }, - { - Name: "container-certs", - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{ - Medium: "", - }, - }, - }, - } - - // volumeMounts are the common mounts used in all pods - volumeMounts = []v1.VolumeMount{ - { - Name: "srv", - MountPath: "/srv", - }, - { - Name: "pki-trust", - MountPath: "/etc/pki/ca-trust/extracted", - }, - { - Name: "pki-anchors", - MountPath: "/etc/pki/ca-trust/anchors", - }, - { - Name: "container-certs", - MountPath: "/etc/containers/cert.d", - }, - } - - // Define basic envVars - ocpEnvVars = []v1.EnvVar{ - { - // SSL_CERT_FILE is understood by Golang code as a pointer to alternative - // directory for certificates. The contents is populated by the ocpInitCommand - Name: "SSL_CERT_FILE", - Value: "/etc/containers/cert.d/ca.crt", - }, - { - Name: "OSCONTAINER_CERT_DIR", - Value: "/etc/containers/cert.d", - }, - } - - // Define the Securite Contexts - ocpSecContext = &v1.SecurityContext{} - - // On OpenShift 3.x, we require privileges. - ocp3SecContext = &v1.SecurityContext{ - RunAsUser: ptrInt(0), - RunAsGroup: ptrInt(1000), - Privileged: ptrBool(true), - } - - // InitCommands to be run before work in pod is executed. - ocpInitCommand = []string{ - "mkdir -vp /etc/pki/ca-trust/extracted/{openssl,pem,java,edk2}", - - // Add any extra anchors which are defined in sa_secrets.go - "cp -av /etc/pki/ca-trust/source/anchors2/*{crt,pem} /etc/pki/ca-trust/anchors/ || :", - - // Always trust the cluster proivided certificates - "cp -av /run/secrets/kubernetes.io/serviceaccount/ca.crt /etc/pki/ca-trust/anchors/cluster-ca.crt || :", - "cp -av /run/secrets/kubernetes.io/serviceaccount/service-ca.crt /etc/pki/ca-trust/anchors/service-ca.crt || :", - - // Update the CA Certs - "update-ca-trust", - - // Explicitly add the cluster certs for podman/buildah/skopeo - "mkdir -vp /etc/containers/certs.d", - "cat /run/secrets/kubernetes.io/serviceaccount/*crt >> /etc/containers/certs.d/ca.crt || :", - "cat /etc/pki/ca-trust/extracted/pem/* >> /etc/containers/certs.d/ca.crt ||:", - } - - // On OpenShift 3.x, /dev/kvm is unlikely to world RW. So we have to give ourselves - // permission. Gangplank will run as root but `cosa` commands run as the builder - // user. Note: on 4.x, gangplank will run unprivileged. - ocp3InitCommand = append(ocpInitCommand, - "/usr/bin/chmod 0666 /dev/kvm || echo missing kvm", - "/usr/bin/stat /dev/kvm || :", - ) - - // Define the base requirements - // cpu are in mils, memory is in mib - baseCPU = *resource.NewQuantity(2, "") - baseMem = *resource.NewQuantity(4*1024*1024*1024, resource.BinarySI) - - ocp3Requirements = v1.ResourceList{ - v1.ResourceCPU: baseCPU, - v1.ResourceMemory: baseMem, - } - - ocpRequirements = v1.ResourceList{ - v1.ResourceCPU: baseCPU, - v1.ResourceMemory: baseMem, - kvmLabel: *resource.NewQuantity(1, ""), - } -) - -// podTimeOut is the lenght of time to wait for a pod to complete its work. -var podTimeOut = 90 * time.Minute - -// termChan is a channel used to singal a termination -type termChan <-chan bool - -// cosaPod is a COSA pod -type cosaPod struct { - apiBuild *buildapiv1.Build - clusterCtx ClusterContext - - ocpInitCommand []string - ocpRequirements v1.ResourceList - ocpSecContext *v1.SecurityContext - volumes []v1.Volume - volumeMounts []v1.VolumeMount - - index int -} - -func (cp *cosaPod) GetClusterCtx() ClusterContext { - return cp.clusterCtx -} - -// CosaPodder create COSA capable pods. -type CosaPodder interface { - WorkerRunner(term termChan, envVar []v1.EnvVar) error - GetClusterCtx() ClusterContext - getPodSpec([]v1.EnvVar) (*v1.Pod, error) -} - -// a cosaPod is a CosaPodder -var _ CosaPodder = &cosaPod{} - -// NewCosaPodder creates a CosaPodder -func NewCosaPodder( - ctx ClusterContext, - apiBuild *buildapiv1.Build, - index int) (CosaPodder, error) { - - cp := &cosaPod{ - apiBuild: apiBuild, - clusterCtx: ctx, - index: index, - - // Set defaults for OpenShift 4.x - ocpRequirements: ocpRequirements, - ocpSecContext: ocpSecContext, - ocpInitCommand: ocpInitCommand, - - volumes: volumes, - volumeMounts: volumeMounts, - } - - ac, _, err := GetClient(ctx) - if err != nil { - return nil, err - } - - // If the builder is in-cluster (either as a BuildConfig or an unbound pod), - // discover the version of OpenShift/Kubernetes. - if ac != nil { - vi, err := ac.DiscoveryClient.ServerVersion() - if err != nil { - return nil, fmt.Errorf("failed to query the kubernetes version: %w", err) - } - - minor, err := strconv.Atoi(strings.TrimRight(vi.Minor, "+")) - log.Infof("Kubernetes version of cluster is %s %s.%d", vi.String(), vi.Major, minor) - if err != nil { - return nil, fmt.Errorf("failed to detect OpenShift v4.x cluster version: %v", err) - } - // Hardcode the version for OpenShift 3.x. - if minor == 11 { - - log.Infof("Creating container with OpenShift v3.x defaults") - cp.ocpRequirements = ocp3Requirements - cp.ocpSecContext = ocp3SecContext - cp.ocpInitCommand = ocp3InitCommand - } - - if err := cp.addVolumesFromSecretLabels(); err != nil { - log.WithError(err).Errorf("failed to add secret volumes and mounts") - } - if err := cp.addVolumesFromConfigMapLabels(); err != nil { - log.WithError(err).Errorf("failed to add volumes from config maps") - } - } - - return cp, nil -} - -func ptrInt(i int64) *int64 { return &i } -func ptrBool(b bool) *bool { return &b } - -// getPodSpec returns a pod specification. -func (cp *cosaPod) getPodSpec(envVars []v1.EnvVar) (*v1.Pod, error) { - podName := fmt.Sprintf("%s-%s-worker-%d", - cp.apiBuild.Annotations[buildapiv1.BuildConfigAnnotation], - cp.apiBuild.Annotations[buildapiv1.BuildNumberAnnotation], - cp.index, - ) - log.Infof("Creating pod %s", podName) - - cosaBasePod := v1.Container{ - Name: podName, - Image: apiBuild.Spec.Strategy.CustomStrategy.From.Name, - Command: []string{"/usr/bin/dumb-init"}, - Args: []string{gangwayCmd}, - Env: append(ocpEnvVars, envVars...), - WorkingDir: "/srv", - VolumeMounts: cp.volumeMounts, - SecurityContext: cp.ocpSecContext, - ImagePullPolicy: v1.PullAlways, - Resources: v1.ResourceRequirements{ - Limits: cp.ocpRequirements, - Requests: cp.ocpRequirements, - }, - } - - cosaWork := []v1.Container{cosaBasePod} - cosaInit := []v1.Container{} - if len(cp.ocpInitCommand) > 0 { - log.Infof("InitContainer has been defined") - initCtr := cosaBasePod.DeepCopy() - initCtr.Name = "init" - initCtr.Args = []string{"/bin/bash", "-xc", fmt.Sprintf(`#!/bin/bash -export PATH=/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin:$PATH -%s -`, strings.Join(cp.ocpInitCommand, "\n"))} - - cosaInit = []v1.Container{*initCtr} - } - - pod := &v1.Pod{ - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - - // Cargo-cult the labels comming from the parent. - Labels: apiBuild.Labels, - }, - Spec: v1.PodSpec{ - ActiveDeadlineSeconds: ptrInt(1800), - AutomountServiceAccountToken: ptrBool(true), - Containers: cosaWork, - InitContainers: cosaInit, - RestartPolicy: v1.RestartPolicyNever, - ServiceAccountName: apiBuild.Spec.ServiceAccount, - TerminationGracePeriodSeconds: ptrInt(300), - Volumes: cp.volumes, - }, - } - - return pod, nil -} - -type podmanRunnerFunc func(termChan, CosaPodder, []v1.EnvVar) error - -// podmanFunc is set to unimplemented by default. -var podmanFunc podmanRunnerFunc = func(termChan, CosaPodder, []v1.EnvVar) error { - return errors.New("build was not compiled with podman supprt") -} - -// WorkerRunner runs a worker pod on either OpenShift/Kubernetes or -// in as a podman container. -func (cp *cosaPod) WorkerRunner(term termChan, envVars []v1.EnvVar) error { - cluster, err := GetCluster(cp.clusterCtx) - if err != nil { - return err - } - if cluster.inCluster { - return clusterRunner(term, cp, envVars) - } - return podmanFunc(term, cp, envVars) -} - -// clusterRunner creates an OpenShift/Kubernetes pod for the work to be done. -// The output of the pod is streamed and captured on the console. -func clusterRunner(term termChan, cp CosaPodder, envVars []v1.EnvVar) error { - ctx := cp.GetClusterCtx() - cs, ns, err := GetClient(ctx) - if err != nil { - return err - } - - pod, err := cp.getPodSpec(envVars) - if err != nil { - return err - } - l := log.WithField("podname", pod.Name) - - // start the pod - ac := cs.CoreV1() - createResp, err := ac.Pods(ns).Create(ctx, pod, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("failed to create pod %s: %w", pod.Name, err) - } - log.Infof("Pod created: %s", pod.Name) - - // Ensure that the pod is always deleted - defer func() { - termOpts := metav1.DeleteOptions{ - // the default grace period on OCP 3.x is 5min and OCP 4.x is 1min - // If the pod is in an error state it will appear to be hang. - GracePeriodSeconds: ptrInt(0), - } - if err := ac.Pods(ns).Delete(ctx, pod.Name, termOpts); err != nil { - l.WithError(err).Error("Failed delete on pod, yolo.") - } - }() - - watcher := func() <-chan error { - retCh := make(chan error) - go func() { - logStarted := false - watchOpts := metav1.ListOptions{ - Watch: true, - ResourceVersion: createResp.ResourceVersion, - FieldSelector: fields.Set{"metadata.name": pod.Name}.AsSelector().String(), - LabelSelector: labels.Everything().String(), - TimeoutSeconds: ptrInt(7200), // set a hard timeout to 2hrs - } - w, err := ac.Pods(ns).Watch(ctx, watchOpts) - if err != nil { - retCh <- err - return - } - - defer func() { - w.Stop() - }() - - for { - events, resultsOk := <-w.ResultChan() - if !resultsOk { - l.Error("failed waitching pod") - retCh <- fmt.Errorf("orphaned pod") - return - } - - resp, ok := events.Object.(*v1.Pod) - if !ok { - retCh <- fmt.Errorf("pod failed") - return - } - - status := resp.Status - l := log.WithFields(log.Fields{"phase": status.Phase}) - - // OCP 3 hack: PodRunning() does not return false - // with OCP if the conditions show completed. - for _, v := range resp.Status.ContainerStatuses { - if v.State.Terminated != nil && v.State.Terminated.ExitCode > 0 { - retCh <- fmt.Errorf("container %s exited with code %d", pod.Name, v.State.Terminated.ExitCode) - return - } - } - - reasons := []string{} - for _, v := range resp.Status.Conditions { - if v.Reason != "" { - reasons = append(reasons, v.Reason) - } - if v.Reason == "PodCompleted" { - retCh <- nil - return - } - } - // Check for running - running, err := PodRunning(events) - if err != nil { - if err == ErrPodCompleted { - retCh <- nil - return - } - l.WithError(err).Error("Pod was deleted") - retCh <- err - return - } - - if !logStarted && running { - l.Info("Starting logging") - if err := streamPodLogs(cs, ns, pod, term); err != nil { - log.WithError(err).Info("failure in code") - retCh <- err - return - } - logStarted = true - } - - // A pod can be running and completed, so do this _last_ - // in case the pod has completed - completed, err := PodCompleted(events) - if err != nil { - l.WithError(err).Error("Pod was deleted") - retCh <- err - return - } else if completed { - l.Info("Pod has completed") - retCh <- nil - return - } - - l.WithFields(log.Fields{ - "completed": completed, - "running": running, - "pod status": resp.Status.Phase, - "conditions": reasons, - }).Info("waiting...") - } - }() - return retCh - } - - // Block on either the watch function returning, timeout or cancellation. - select { - case err, ok := <-watcher(): - if !ok { - return nil - } - return err - case <-time.After(podTimeOut): - return fmt.Errorf("pod %s did not complete work in time", pod.Name) - case <-term: - return fmt.Errorf("pod %s was signalled to terminate by main process", pod.Name) - } -} - -// consoleLogWriter is an io.Writer that emits fancy logs to a screen. -type consoleLogWriter struct { - startTime time.Time - prefix string -} - -// consoleLogWriter is an io.Writer. -var _ io.Writer = &consoleLogWriter{} - -// newConosleLogWriter is a helper function for getting a new writer. -func newConsoleLogWriter(prefix string) *consoleLogWriter { - return &consoleLogWriter{ - prefix: prefix, - startTime: time.Now(), - } -} - -// Write implements io.Writer for Console Writer with -func (cw *consoleLogWriter) Write(b []byte) (int, error) { - since := time.Since(cw.startTime).Truncate(time.Millisecond) - prefix := []byte(fmt.Sprintf("%s [+%v]: ", cw.prefix, since)) - suffix := []byte("\n") - - _, _ = os.Stdout.Write(prefix) - n, err := os.Stdout.Write(b) - _, _ = os.Stdout.Write(suffix) - return n, err -} - -// writeToWriters writes in to outs until in or outs are closed. When run a -// go-routine, calls can terminate by closing "in". -func writeToWriters(l *log.Entry, in io.ReadCloser, outs ...io.Writer) <-chan error { - outCh := make(chan error) - go func() { - var err error - defer func() { - if err != nil { - if err.Error() == "http2: response body closed" { - outCh <- nil - return - } - l.WithError(err).Error("writeToWriters encountered an error") - outCh <- err - } - }() - - scanner := bufio.NewScanner(in) - outWriter := io.MultiWriter(outs...) - for scanner.Scan() { - _, err = outWriter.Write(scanner.Bytes()) - if err != nil { - l.WithError(err).Error("failed to write to logs") - return - } - } - err = scanner.Err() - if err != nil { - return - } - }() - return outCh -} - -// streamPodLogs steams the pod's logs to logging and to disk. Worker -// pods are responsible for their work, but not for their logs. -// To make streamPodLogs thread safe and non-blocking, it expects -// a pointer to a bool. If that pointer is nil or true, then we return. -func streamPodLogs(client *kubernetes.Clientset, namespace string, pod *v1.Pod, term termChan) error { - ctx := context.Background() - for _, pC := range append(pod.Spec.InitContainers, pod.Spec.Containers...) { - container := pC.Name - podLogOpts := v1.PodLogOptions{ - Follow: true, - SinceSeconds: ptrInt(300), - Container: container, - } - - req := client.CoreV1().Pods(namespace).GetLogs(pod.Name, &podLogOpts) - streamer, err := req.Stream(ctx) - if err != nil { - return err - } - - // Create the deafault file log - logD := filepath.Join(cosaSrvDir, "logs") - logN := filepath.Join(logD, fmt.Sprintf("%s-%s.log", pod.Name, container)) - if err := os.MkdirAll(logD, 0755); err != nil { - return fmt.Errorf("failed to create logs directory: %w", err) - } - logf, err := os.OpenFile(logN, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return fmt.Errorf("failed to create log for pod %s/%s container: %w", pod.Name, container, err) - } - - l := log.WithFields(log.Fields{ - "logfile": logf.Name, - "container": container, - "pod": pod.Name, - }) - - // Watch the logs until the termination is singaled OR the logs stream fails. - go func() { - // the defer will ensure that writeToWriters errors and terminates - defer func() { - lerr := logf.Close() - serr := streamer.Close() - if lerr != nil || serr != nil { - l.WithFields(log.Fields{ - "stream err": err, - "log err": lerr, - }).Info("failed closing logs, likely will have dangling go-routines") - } - l.Info("logging terminated") - }() - - for { - select { - case die, ok := <-term: - if die || !ok { - return - } - case err, ok := <-writeToWriters(l, streamer, logf, newConsoleLogWriter(container)): - if !ok { - return - } - if err != nil { - l.WithError(err).Warn("error recieved from writer") - return - } - } - } - }() - } - return nil -} diff --git a/gangplank/internal/ocp/cosa-podman.go b/gangplank/internal/ocp/cosa-podman.go deleted file mode 100644 index 44182b9f..00000000 --- a/gangplank/internal/ocp/cosa-podman.go +++ /dev/null @@ -1,259 +0,0 @@ -// +build !gangway - -package ocp - -import ( - "context" - "fmt" - "os" - "strings" - "time" - - "github.com/containers/podman/v3/pkg/bindings" - "github.com/containers/podman/v3/pkg/bindings/containers" - podImages "github.com/containers/podman/v3/pkg/bindings/images" - podVolumes "github.com/containers/podman/v3/pkg/bindings/volumes" - "github.com/containers/podman/v3/pkg/domain/entities" - "github.com/containers/podman/v3/pkg/specgen" - "github.com/containers/storage" - "github.com/containers/storage/pkg/idtools" - "github.com/opencontainers/runc/libcontainer/user" - cspec "github.com/opencontainers/runtime-spec/specs-go" - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" -) - -// podmanContainerHostEnvVar is used by both Gangplank and the podman API -// to decide if the execution of the pod should happen over SSH. -const podmanContainerHostEnvVar = "CONTAINER_HOST" - -func init() { - podmanFunc = podmanRunner -} - -// podmanRunner runs the work in a Podman container using workDir as `/srv` -// `podman kube play` does not work well due to permission mappings; there is -// no way to do id mappings. -func podmanRunner(term termChan, cp CosaPodder, envVars []v1.EnvVar) error { - ctx := cp.GetClusterCtx() - - // Populate pod envvars - envVars = append( - envVars, - v1.EnvVar{Name: localPodEnvVar, Value: "1"}, - v1.EnvVar{Name: "XDG_RUNTIME_DIR", Value: "/srv"}, - ) - mapEnvVars := map[string]string{} - for _, v := range envVars { - mapEnvVars[v.Name] = v.Value - } - - // Get our pod spec - podSpec, err := cp.getPodSpec(envVars) - if err != nil { - return err - } - l := log.WithFields(log.Fields{ - "method": "podman", - "image": podSpec.Spec.Containers[0].Image, - "podName": podSpec.Name, - }) - - // If a URI for the container API server has been specified - // by the user then let's honor that. Else construct one. - socket := os.Getenv(podmanContainerHostEnvVar) - if strings.HasPrefix(socket, "ssh://") { - l = l.WithField("podman socket", socket) - l.Info("Lauching remote pod") - } else { - // Once podman 3.2.0 is released use this instead: - // import "github.com/containers/podman/v3/pkg/util" - // socket = util.SocketPath() - sockDir := os.Getenv("XDG_RUNTIME_DIR") - socket = "unix:" + sockDir + "/podman/podman.sock" - } - - // Connect to Podman socket - connText, err := bindings.NewConnection(ctx, socket) - if err != nil { - return err - } - - // Get the StdIO from the cluster context. - clusterCtx, err := GetCluster(ctx) - if err != nil { - return err - } - stdIn, stdOut, stdErr := clusterCtx.GetStdIO() - if stdOut == nil { - stdOut = os.Stdout - } - if stdErr == nil { - stdErr = os.Stdout - } - if stdIn == nil { - stdIn = os.Stdin - } - - s := specgen.NewSpecGenerator(podSpec.Spec.Containers[0].Image, false) - s.CapAdd = podmanCaps - s.Name = podSpec.Name - s.ContainerNetworkConfig = specgen.ContainerNetworkConfig{ - NetNS: specgen.Namespace{ - NSMode: specgen.Host, - }, - } - - u, err := user.CurrentUser() - if err != nil { - return fmt.Errorf("unable to lookup the current user: %v", err) - } - - s.ContainerSecurityConfig = specgen.ContainerSecurityConfig{ - NoNewPrivileges: false, - Umask: "0022", - Privileged: true, - User: "builder", - IDMappings: &storage.IDMappingOptions{ - UIDMap: []idtools.IDMap{ - { - ContainerID: 0, - HostID: u.Uid, - Size: 1, - }, - { - ContainerID: 1000, - HostID: u.Uid, - Size: 200000, - }, - }, - }, - } - s.Env = mapEnvVars - s.Stdin = true - s.Terminal = true - s.Devices = []cspec.LinuxDevice{ - { - Path: "/dev/kvm", - Type: "char", - }, - { - Path: "/dev/fuse", - Type: "char", - }, - } - - var srvVol *entities.VolumeConfigResponse = nil - if clusterCtx.podmanSrvDir == "" { - // If running podman remotely or the srvDir is undefined, create and use an ephemeral - // volume. The volume will be removed via ender() - srvVol, err = podVolumes.Create(connText, entities.VolumeCreateOptions{Name: podSpec.Name}, nil) - if err != nil { - return err - } - s.Volumes = []*specgen.NamedVolume{ - { - Name: srvVol.Name, - Options: []string{}, - Dest: "/srv", - }, - } - l.WithField("ephemeral vol", srvVol.Name).Info("using ephemeral volule for /srv") - } else { - // Otherwise, create a mount from the host container for /srv. - l.WithField("bind mount", clusterCtx.podmanSrvDir).Info("using host directory for /srv") - s.Mounts = []cspec.Mount{ - { - Type: "bind", - Destination: "/srv", - Source: clusterCtx.podmanSrvDir, - }, - } - } - - s.WorkDir = "/srv" - s.Entrypoint = []string{"/usr/bin/dumb-init"} - s.Command = []string{gangwayCmd} - - if err := mustHaveImage(connText, s.Image); err != nil { - return fmt.Errorf("failed checking/pulling image: %v", err) - } - - // Validate and define the container spec - if err := s.Validate(); err != nil { - l.WithError(err).Error("Validation failed") - } - r, err := containers.CreateWithSpec(connText, s, nil) - if err != nil { - return fmt.Errorf("failed to create container: %w", err) - } - - // channels for logging - consoleOutChan := make(chan string, 1024) - - // Manually terminate the pod to ensure that we get all the logs first. - ender := func() { - time.Sleep(1 * time.Second) - _ = containers.Remove(connText, r.ID, new(containers.RemoveOptions).WithForce(true).WithVolumes(true)) - if srvVol != nil { - _ = podVolumes.Remove(connText, srvVol.Name, nil) - } - } - defer ender() - - if err := containers.Start(connText, r.ID, nil); err != nil { - l.WithError(err).Error("Start of pod failed") - return err - } - - go func() { - select { - case <-ctx.Done(): - ender() - case <-term: - ender() - } - }() - - // Watch the logs. - go func() { - for { - v, ok := <-consoleOutChan - if !ok { - return - } - os.Stdout.Write([]byte(v)) - os.Stdout.Write([]byte("\n")) - } - }() - - // Get the Logs. This will block until all logs are streamed. - if err := containers.Logs( - connText, r.ID, - &containers.LogOptions{ - Follow: ptrBool(true), - }, - consoleOutChan, consoleOutChan); err != nil { - return fmt.Errorf("Failed setting up console monitoring: %v", err) - } - - // Get the exit code of the container. - rc, err := containers.Wait(connText, r.ID, nil) - if rc != 0 { - return fmt.Errorf("work pod failed: return code %d", rc) - } - return err -} - -// mustHaveImage pulls the image if it is not found -func mustHaveImage(ctx context.Context, image string) error { - found, err := podImages.Exists(ctx, image, nil) - if err != nil { - return err - } - if found { - return nil - } - _, err = podImages.Pull(ctx, image, nil) - return err -} diff --git a/gangplank/internal/ocp/cosa_init.go b/gangplank/internal/ocp/cosa_init.go deleted file mode 100644 index 2304a1ac..00000000 --- a/gangplank/internal/ocp/cosa_init.go +++ /dev/null @@ -1,63 +0,0 @@ -package ocp - -import ( - "errors" - "os" - "os/exec" - "strings" - - "github.com/coreos/gangplank/internal/spec" - log "github.com/sirupsen/logrus" -) - -const ( - envVarSourceURI = "SOURCE_URI" - envVarSourceRef = "SOURCE_REF" -) - -// cosaInit does the initial COSA setup. To support both pod and buildConfig -// based builds, first check the API client, then check envVars. The use of envVars -// in this case is *safe*; `SOURCE_{URI,REF} == apiBuild.Spec.Source.Git.{URI,REF}`. That -// is, SOURCE_* envVars will always match the apiBuild.Spec.Source.Git.* values. -func cosaInit(js spec.JobSpec) error { - _ = os.Chdir(cosaSrvDir) - var gitURI, gitRef, gitCommit string - if js.Recipe.GitURL != "" { - gitURI = js.Recipe.GitURL - gitRef = js.Recipe.GitRef - gitCommit = js.Recipe.GitCommit - } else if apiBuild != nil && apiBuild.Spec.Source.Git != nil { - gitURI = apiBuild.Spec.Source.Git.URI - gitRef = apiBuild.Spec.Source.Git.Ref - } else { - gitURI, _ = os.LookupEnv(envVarSourceURI) - gitRef, _ = os.LookupEnv(envVarSourceRef) - } - if gitURI == "" { - log.Info("No Git Source, skipping checkout") - return ErrNoSourceInput - } - - args := []string{"cosa", "init"} - if gitRef != "" { - args = append(args, "--force", "--branch", gitRef) - } - if gitCommit != "" { - args = append(args, "--commit", gitCommit) - } - args = append(args, gitURI) - log.Infof("running '%v'", strings.Join(args, " ")) - cmd := exec.Command(args[0], args[1:]...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - out, _ := cmd.CombinedOutput() - log.WithFields(log.Fields{ - "cmd": args, - "error": err, - "out": string(out), - }).Error("Failed to checkout respository") - return errors.New("failed to run cosa init") - } - return nil -} diff --git a/gangplank/internal/ocp/errors.go b/gangplank/internal/ocp/errors.go deleted file mode 100644 index a462a329..00000000 --- a/gangplank/internal/ocp/errors.go +++ /dev/null @@ -1,30 +0,0 @@ -package ocp - -import "errors" - -var ( - // ErrNoSuchCloud is returned when the cloud is unknown - ErrNoSuchCloud = errors.New("unknown cloud credential type") - - // ErrNoOCPBuildSpec is raised when no OCP envvars are found - ErrNoOCPBuildSpec = errors.New("no OCP Build specification found") - - // ErrNotInCluster is used to singal that the host is not running in a - // Kubernetes cluster - ErrNotInCluster = errors.New("host is not in kubernetes cluster") - - // ErrInvalidOCPMode is used when there is no valid/supported mode the OCP - // package. Currently this is thrown when neither a build client or kubernetes API - // client can be initalized. - ErrInvalidOCPMode = errors.New("program is not running as a buildconfig or with valid kubernetes service account") - - // ErrNoSourceInput is used to signal no source found. - ErrNoSourceInput = errors.New("no source repo or binary payload defined") - - // ErrNotWorkPod is returned when the pod is not a work pod - ErrNotWorkPod = errors.New("not a work pod") - - // ErrNoWorkFound is returned when the build client is neither a - // workPod or BuildConfig. - ErrNoWorkFound = errors.New("neither a buildconfig or workspec found") -) diff --git a/gangplank/internal/ocp/filer.go b/gangplank/internal/ocp/filer.go deleted file mode 100644 index 9fef319e..00000000 --- a/gangplank/internal/ocp/filer.go +++ /dev/null @@ -1,686 +0,0 @@ -package ocp - -import ( - - // minio is needed for moving files around in OpenShift. - - "bufio" - "context" - "crypto/rand" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/minio/minio-go/v7/pkg/tags" - log "github.com/sirupsen/logrus" -) - -/* - Minio (https://github.com/minio/minio) is an S3-API Compatible - Object Store. When running in multi-pod mode, we start Minio - for pulling and pushing artifacts. Object Storage is a little better - than using PVC's. - - NOTE: This is intentionally private -- we do not want to expose this - functionality outside the ocp package. -*/ - -// minioServer describes a Minio S3 Object stoarge to start. -type minioServer struct { - AccessKey string `json:"accesskey"` - ExternalServer bool `json:"external_server"` //indicates that a server should not be started - Host string `json:"host"` - Port int `json:"port"` - Region string `json:"region"` - SecretKey string `json:"secretkey"` - Secure bool `json:"secure"` // indicates use of TLS - - // overSSH describes how to forward the Minio Port over SSH - // This option is only useful with envVar CONTAINER_HOST running - // in podman mode. - overSSH *SSHForwardPort - // sshStopCh is used to shutdown the SSH port forwarding. - sshStopCh chan bool - // sshErrCh is - sshErrCh chan error - - dir string - minioOptions minio.Options - cmd *exec.Cmd -} - -// StartStanaloneMinioServer starts a standalone minio server. -func StartStandaloneMinioServer(ctx context.Context, srvDir, cfgFile string, overSSH *SSHForwardPort) (*minioServer, error) { - m := newMinioServer("") - m.overSSH = overSSH - m.dir = srvDir - - // Start the minio server. If we're forwarding over SSH we'll call - // startMinioAndForwardOverSSH to start the minio server. because - // the port we use will be dynamically chosen based on the SSH - // connection. - if m.overSSH == nil { - if err := m.start(ctx); err != nil { - return nil, err - } - } else { - m.sshStopCh = make(chan bool, 1) - m.sshErrCh = make(chan error, 256) - if err := m.startMinioAndForwardOverSSH(ctx, m.sshStopCh, m.sshErrCh); err != nil { - return nil, err - } - } - - m.ExternalServer = true - if err := m.WriteToFile(cfgFile); err != nil { - return nil, fmt.Errorf("failed to write configuration for minio: %v", err) - } - - return m, nil -} - -// newMinioSever defines an ephemeral minioServer from a config or creates a new one. -// To prevent random pods/people accessing or relying on the server, we use entirely random keys. -func newMinioServer(cfgFile string) *minioServer { - if cfgFile != "" { - m, err := minioCfgFromFile(cfgFile) - if err != nil { - log.WithError(err).Fatalf("failed read minio cfg from %s", cfgFile) - } - log.Infof("Minio configuration defined from %s", cfgFile) - return &m - } - - // If Gangplank is running in cluster, then get the IP address. Using - // hostnames can be problematic. - host := getHostname() - ac, ns, err := k8sInClusterClient() - if err == nil && ac != nil { - var ctx ClusterContext = context.Background() - ip, err := getPodIP(ctx, ac, ns, host) - if err == nil { - host = ip - } - } - - log.Info("Defining a new minio server") - minioAccessKey, _ := randomString(12) - minioSecretKey, _ := randomString(12) - - m := &minioServer{ - AccessKey: minioAccessKey, - SecretKey: minioSecretKey, - Host: host, - dir: cosaSrvDir, - ExternalServer: false, - minioOptions: minio.Options{ - Creds: credentials.NewStaticV4(minioAccessKey, minioSecretKey, ""), - Secure: false, - Region: fmt.Sprintf("cosaHost-%s", getHostname()), - }, - } - - if m.overSSH != nil { - m.Host = "127.0.0.1" - } - - return m -} - -// GetClient returns a Minio Client -func (m *minioServer) client() (*minio.Client, error) { - region := m.Region - var secure bool - if m.ExternalServer { - if strings.Contains(m.Host, "s3.amazonaws.com") { - secure = true - if m.Region == "" { - region = "us-east-1" - } - } - } - return minio.New(fmt.Sprintf("%s:%d", m.Host, m.Port), - &minio.Options{ - Transport: &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 0, - DisableCompression: false, // force compression - }, - Creds: credentials.NewStaticV4(m.AccessKey, m.SecretKey, ""), - Secure: secure, - Region: region, - }, - ) -} - -// start executes the minio server and returns an error if not ready. -func (m *minioServer) start(ctx context.Context) error { - if m.ExternalServer { - return <-m.waitReadyChan(90 * time.Second) - } - - if err := retry(5, 1*time.Second, func() error { return m.exec(ctx) }); err != nil { - log.WithError(err).Warn("failed to start minio") - return err - } - - return nil -} - -// exec runs the minio command -func (m *minioServer) exec(ctx context.Context) error { - if m.Host == "" { - m.Host = getHostname() - } - - if m.Port == 0 { - m.Port = getPortOrNext(9000) - } - - l := log.WithFields(log.Fields{ - "hostname": m.Host, - "port": m.Port, - "access_key": m.AccessKey, - "secret_key": m.SecretKey, - "serv dir": m.dir, - }) - l.Infof("Starting Minio") - - mpath, err := exec.LookPath("minio") - if err != nil { - l.WithField("err", err).Error("minio binary not found") - return errors.New("failed to find minio") - } - - absPath, err := filepath.Abs(m.dir) - if err != nil { - return err - } - - cport := getPortOrNext(4747) - - addr := fmt.Sprintf(":%d", m.Port) - args := []string{ - mpath, "server", - "--quiet", "--anonymous", - "--console-address", fmt.Sprintf(":%d", cport), - "--address", addr, - absPath, - } - cmd := exec.CommandContext(ctx, args[0], args[1:]...) - cmd.SysProcAttr = &syscall.SysProcAttr{ - Foreground: false, // Background the process - Pdeathsig: syscall.SIGTERM, // Let minio finish before killing - Pgid: 0, // Use the pid of the minio as the pgroup id - Setpgid: true, // Set the pgroup - } - cmd.Dir = absPath - cmd.Env = append( - os.Environ(), - fmt.Sprintf("MINIO_ACCESS_KEY=%s", m.AccessKey), - fmt.Sprintf("MINIO_SECRET_KEY=%s", m.SecretKey), - ) - - outPipe, _ := cmd.StdoutPipe() - errPipe, sErr := cmd.StderrPipe() - if sErr != nil { - return fmt.Errorf("failed to start get output pipe: %v", sErr) - } - - if err := cmd.Start(); err != nil { - l.WithFields(log.Fields{ - "err": err, - }).Error("Failed to start minio") - return err - } - - minioMsgChan := make(chan string, 1) - go func() { - s := bufio.NewScanner(io.MultiReader(outPipe, errPipe)) - for s.Scan() { - minioMsgChan <- s.Text() - } - }() - - startChan := make(chan error, 1) - go func() { - for { - if cmd == nil || (cmd.ProcessState != nil && cmd.ProcessState.Exited()) { - startChan <- fmt.Errorf("minio started but exited") - return - - } - if cmd.ProcessState != nil && cmd.ProcessState.Exited() { - startChan <- fmt.Errorf("failed to start minio") - return - } - } - }() - - // Ensure the process gets terminated on shutdown - sigs := make(chan os.Signal, 64) - go func() { - signal.Notify(sigs, os.Interrupt, syscall.SIGTERM, syscall.SIGUSR1, syscall.SIGUSR2) - <-sigs - m.Kill() - }() - - m.cmd = cmd - for { - select { - case err := <-startChan: - if cmd != nil { - stdoutStderr, _ := cmd.CombinedOutput() - l.WithFields(log.Fields{ - "err": err, - "out": stdoutStderr, - }).Errorf("minio start failure") - } - return err - case msg := <-minioMsgChan: - fmt.Printf("MINIO: %s\n", msg) - case <-sigs: - return fmt.Errorf("minio startup was interrupted") - case err := <-m.waitReadyChan(90 * time.Second): - return err - } - } -} - -// Wait blocks until Minio is finished. -func (m *minioServer) Wait() { - if m.cmd != nil { - _ = m.cmd.Wait() - } -} - -// Kill terminates the minio server. -func (m *minioServer) Kill() { - if m.cmd == nil { - return - } - - // Kill any forward SSH connections. - if m.overSSH != nil && m.sshStopCh != nil { - m.sshStopCh <- true - } - - // Note the "-" before the processes PID. A negative pid to - // syscall.Kill kills the processes Pid group ensuring all forks/execs - // of minio are killed too. - _ = syscall.Kill(-m.cmd.Process.Pid, syscall.SIGTERM) - - // Wait for the command to end. - m.Wait() - - // Purge the minio files since they are used per-session. - if err := os.RemoveAll(filepath.Join(m.dir, ".minio.sys")); err != nil { - log.WithError(err).Error("failed to remove minio files") - } -} - -func randomString(n int) (string, error) { - const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - bits := make([]byte, n) - _, err := rand.Read(bits) - if err != nil { - return "", err - } - for i, b := range bits { - bits[i] = letters[b%byte(len(letters))] - } - return string(bits), nil -} - -func (m *minioServer) ensureBucketExists(ctx context.Context, bucket string) error { - mc, err := m.client() - if err != nil { - return err - } - - if be, err := mc.BucketExists(ctx, bucket); err != nil { - return err - } else if be { - return nil - } - - err = mc.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: m.Region}) - if err != nil { - return fmt.Errorf("failed call to create bucket: %w", err) - } - return nil -} - -// fetcher retrieves an object from a Minio server -func (m *minioServer) fetcher(ctx context.Context, bucket, object string, dest io.Writer) error { - return retry( - 3, - (3 * time.Second), - func() error { - if m.Host == "" { - return errors.New("host is undefined") - } - // Set the attributes - f, isFile := dest.(*os.File) - l := log.WithFields(log.Fields{ - "bucket": bucket, - "host": m.Host, - "object": object, - }) - - l.Info("Requesting remote object") - - mc, err := m.client() - if err != nil { - return err - } - - src, err := mc.GetObject(ctx, bucket, object, minio.GetObjectOptions{}) - if err != nil { - return err - } - defer src.Close() - n, err := io.Copy(dest, src) - if err != nil { - l.WithError(err).Error("failed to write the file") - return err - } - l.WithField("bytes", n).Info("Wrote file") - - if isFile { - info, err := src.Stat() - if err != nil { - return err - } - if err := os.Chtimes(f.Name(), info.LastModified, info.LastModified); err != nil { - return err - } - } - return nil - }) -} - -// putter uploads the contents of an io.Reader to a remote MinioServer -func (m *minioServer) putter(ctx context.Context, bucket, object, fpath string) error { - return retry( - 3, - (3 * time.Second), - func() error { - if err := m.ensureBucketExists(ctx, bucket); err != nil { - return fmt.Errorf("unable to validate %s bucket exists: %w", bucket, err) - } - fi, err := os.Stat(fpath) - if err != nil { - return err - } - l := log.WithFields(log.Fields{ - "bucket": bucket, - "from": fpath, - "func": "putter", - "object": object, - "size": fmt.Sprintf("%d", fi.Size()), - }) - - mC, err := m.client() - if err != nil { - return err - } - - i, err := mC.FPutObject(ctx, bucket, object, fpath, minio.PutObjectOptions{}) - if err != nil { - return fmt.Errorf("failed to upload to %s/%s: %w", bucket, object, err) - } - if err := m.stampFile(bucket, object); err != nil { - return fmt.Errorf("failed to stamp uploaded file %s/%s: %w", bucket, object, err) - } - stamp, _ := m.getStamp(bucket, object) - l.WithFields(log.Fields{ - fileStampName: stamp, - "etag": i.ETag, - "remote size": i.Size, - }).Info("Uploaded") - - return nil - }) -} - -// checkPort checks if a port is open -func checkPort(port int) error { - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) - if err != nil { - return err - } - defer ln.Close() //nolint - return nil -} - -// getNextPort iterates and finds the next available port -func getPortOrNext(port int) int { - for { - if err := checkPort(port); err == nil { - return port - } - port++ - } -} - -// minioCfgFromFile returns a minio configuration from a file -func minioCfgFromFile(f string) (mk minioServer, err error) { - in, err := os.Open(f) - if err != nil { - return mk, err - } - defer in.Close() - b := bufio.NewReader(in) - return minioCfgReader(b) -} - -// WriteJSON returns the jobspec -func (m *minioServer) WriteJSON(w io.Writer) error { - encode := json.NewEncoder(w) - encode.SetIndent("", " ") - return encode.Encode(*m) -} - -// minioKeysFromFile writes the minio keys to a file -func (m *minioServer) WriteToFile(f string) error { - out, err := os.OpenFile(f, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644) - if err != nil { - return err - } - defer out.Close() - return m.WriteJSON(out) -} - -// minioKeysReader takes an io.Reader and returns a minio cfg -func minioCfgReader(in io.Reader) (m minioServer, err error) { - d, err := ioutil.ReadAll(in) - if err != nil { - return m, err - } - - err = json.Unmarshal(d, &m) - if err != nil { - return m, err - } - return m, err -} - -// getHostname gets the current hostname -func getHostname() string { - data, _ := ioutil.ReadFile("/proc/sys/kernel/hostname") - return strings.TrimSpace(string(data)) -} - -// Exists check if bucket/object exists. -func (m *minioServer) Exists(bucket, object string) bool { - mc, err := m.client() - if err != nil { - return false - } - if _, err := mc.StatObject(context.Background(), bucket, object, minio.GetObjectOptions{}); err != nil { - return false - } - return true -} - -const fileStampName = "gangplank.coreos.com/cosa/stamp" - -// newFileStamp returns the Unix nanoseconds of the file as a string -// We use Unix nanoseconds for precision. -func newFileStamp() string { - return fmt.Sprintf("%d", time.Now().UTC().UnixNano()) -} - -// stampFile add the unique stamp -func (m *minioServer) stampFile(bucket, object string) error { - mc, err := m.client() - if err != nil { - return err - } - - tagMap := map[string]string{ - fileStampName: newFileStamp(), - } - - t, err := tags.NewTags(tagMap, true) - if err != nil { - return err - } - - return mc.PutObjectTagging(context.Background(), bucket, object, t, minio.PutObjectTaggingOptions{}) -} - -// isLocalNewer checks if the file is newer than the remote file, if any. If the file -// does not exist remotely, then it is considered newer. -func (m *minioServer) isLocalNewer(bucket, object string, path string) (bool, error) { - curStamp, err := m.getStamp(bucket, object) - if err != nil { - return true, err - } - modTime, err := getLocalFileStamp(path) - if err != nil { - return false, err - } - if modTime > curStamp { - return true, nil - } - return false, nil -} - -// getLocalFileStamp returns the local file mod time in UTC Unix epic nanoseconds. -func getLocalFileStamp(path string) (int64, error) { - f, err := os.Stat(path) - if err != nil { - return 0, err - } - modTime := f.ModTime().UTC().UnixNano() - return modTime, nil -} - -// getStamp returns the stamp. If the file does not exist remotely the stamp of -// zero is returned. If the file exists but has not been stamped, then UTC -// Unix epic in nanoseconds of the modification time is used (the stamps are lost -// when the minio instance is reaped). The obvious flaw is that this does require -// all hosts to have coordinate time; this should be the case for Kubernetes cluster -// and podman based builds will always use the same time source. -func (m *minioServer) getStamp(bucket, object string) (int64, error) { - mc, err := m.client() - if err != nil { - return 0, err - } - - if !m.Exists(bucket, object) { - return 0, nil - } - - tags, err := mc.GetObjectTagging(context.Background(), bucket, object, minio.GetObjectTaggingOptions{}) - if err != nil { - return 0, err - } - if tags == nil { - return 0, nil - } - - for k, v := range tags.ToMap() { - if k == fileStampName { - curStamp, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to convert stamp %s to int64", v) - } - return curStamp, nil - } - } - - // fallback to modtime - info, err := mc.StatObject(context.Background(), bucket, object, minio.GetObjectOptions{}) - if err == nil { - return info.LastModified.UTC().UnixNano(), nil - } - - return 0, err -} - -// retry tries the func upto n "tries" with a sleep between. -func retry(tries int, sleep time.Duration, fn func() error) error { - var err error - for i := 0; i <= tries; i++ { - newErr := fn() - if newErr == nil { - return nil - } - err = fmt.Errorf("error %d: %w", i, newErr) - if sleep < 0 { - time.Sleep(sleep) - } - } - return err -} - -// waitReadyChan returns a chan that emits true when the endpoint responds -func (m *minioServer) waitReadyChan(timeout time.Duration) <-chan error { - readyChan := make(chan error) - go func() { - startTime := time.Now() - for { - // set a timeout - if time.Since(startTime) > timeout { - readyChan <- errors.New("timeout waiting for minio to be ready") - return - } - - mc, err := m.client() - if err != nil { - readyChan <- err - } - - // Test if the remote bucket exists and that the error code does not - // match a magic string. - if _, err := mc.BucketExists(context.Background(), "testBucket"); err != nil { - if strings.Contains(err.Error(), "Server not initialized, please try again.") || - strings.Contains(err.Error(), "connection refused") { - time.Sleep(1 * time.Second) - continue - } - readyChan <- err - } - readyChan <- nil - - } - }() - return readyChan -} diff --git a/gangplank/internal/ocp/filer_test.go b/gangplank/internal/ocp/filer_test.go deleted file mode 100644 index 9c4ed3fa..00000000 --- a/gangplank/internal/ocp/filer_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// +minio -package ocp - -import ( - "context" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/containers/storage/pkg/ioutils" - "github.com/minio/minio-go/v7" - log "github.com/sirupsen/logrus" -) - -func TestFiler(t *testing.T) { - tmpd, err := ioutil.TempDir("", "cosa-test") - if err != nil { - t.Fatalf("Failed to create tempdir") - } - defer os.RemoveAll(tmpd) - - testBucket := "testbucket" - testFileContents := "this is a test" - - c, cancel := context.WithCancel(context.Background()) - defer cancel() - defer c.Done() - - m := newMinioServer("") - m.Host = "localhost" - m.dir = tmpd - if err := m.start(c); err != nil { - t.Fatalf("Failed to start minio: %v", err) - } - - mc, err := m.client() - if err != nil { - t.Errorf("Failed to create test minio client: %v", err) - } - defer m.Kill() - - if err := mc.MakeBucket(c, testBucket, minio.MakeBucketOptions{}); err != nil { - t.Errorf("Failed to create test bucket %s: %v", testBucket, err) - } - - r := strings.NewReader(testFileContents) - if _, err := mc.PutObject(c, testBucket, "test", r, -1, minio.PutObjectOptions{}); err != nil { - t.Errorf("Failed to place test file: %v", err) - } - - tfp := filepath.Join(tmpd, testBucket, "test") - f, err := ioutil.ReadFile(tfp) - if err != nil { - t.Errorf("Failed to find file: %v", err) - } - if string(f) != testFileContents { - t.Errorf("Test file should be %q, got %q", testFileContents, f) - } - - testBucket = "tb1" - if err = m.putter(c, testBucket, "test", tfp); err != nil { - t.Errorf("error: %v", err) - } - - log.Info("Done") -} - -func TestMultipleStandAlones(t *testing.T) { - tmpd, _ := ioutils.TempDir("", "") - srvOne := filepath.Join(tmpd, "one") - srvTwo := filepath.Join(tmpd, "two") - oneCfg := filepath.Join(srvOne, "test.cfg") - twoCfg := filepath.Join(srvTwo, "test.cfg") - - _ = os.MkdirAll(srvOne, 0755) - _ = os.MkdirAll(srvTwo, 0755) - - ctx := context.Background() - one, err := StartStandaloneMinioServer(ctx, srvOne, oneCfg, nil) - if err != nil { - t.Fatalf("%v: failed to start first minio server", err) - } - defer one.Kill() - - two, err := StartStandaloneMinioServer(ctx, srvTwo, twoCfg, nil) - if err != nil { - t.Fatalf("%v: failed to start first minio server", err) - } - defer two.Kill() - - // Fire up the users of the stand alone Minio's - // This tests using a minio from CFG - oneUser := newMinioServer(oneCfg) - if err := oneUser.start(ctx); err != nil { - t.Errorf("%v: failed to start first minio", err) - } - twoUser := newMinioServer(twoCfg) - if err := twoUser.start(ctx); err != nil { - t.Errorf("%v: failed to start second minior", err) - } - - // Connect and make sure that the servers serve different content - // This tests that: - // - host port selection is different - // - that a new minio server is not started - // - each server is using a different set of keys. - if err := oneUser.ensureBucketExists(ctx, "test1"); err != nil { - t.Errorf("%v: failed to create bucket on first minio", err) - } - if err := twoUser.ensureBucketExists(ctx, "test2"); err != nil { - t.Errorf("%v: failed to create bucket on first minio", err) - } - - oneClient, _ := oneUser.client() - twoClient, _ := twoUser.client() - - oneBuckets, oneErr := oneClient.ListBuckets(ctx) - twoBuckets, twoErr := twoClient.ListBuckets(ctx) - - if oneErr != nil || twoErr != nil { - t.Fatalf("failed to list buckets:\none: %v\ntwo: %v\n", oneErr, twoErr) - } - - for _, oneV := range oneBuckets { - for _, twoV := range twoBuckets { - if oneV.Name == twoV.Name && twoV.Name != "builds" { - t.Errorf("bucket two %q should not be in minio one", twoV.Name) - } - } - } - - for _, twoV := range twoBuckets { - for _, oneV := range oneBuckets { - if oneV.Name == twoV.Name && twoV.Name != "builds" { - t.Errorf("bucket two %q should not be in minio one", twoV.Name) - } - } - } - - // Try to access oneUser using twoUsers creds - oneUser.AccessKey = twoUser.AccessKey - oneUser.SecretKey = twoUser.SecretKey - if err := oneUser.ensureBucketExists(ctx, "bah"); err == nil { - t.Fatalf("using wrong credentials should fail") - } -} diff --git a/gangplank/internal/ocp/hop.go b/gangplank/internal/ocp/hop.go deleted file mode 100644 index 284d8e1a..00000000 --- a/gangplank/internal/ocp/hop.go +++ /dev/null @@ -1,134 +0,0 @@ -package ocp - -/* - Hop Pod's allow for Gangplank to create a pod in a remote cluster. - - The goal in a hop pod is two fold: - - CI environments like Prow (where Gangplank is run with insufficent perms) - - Developers who want to run Gangplank remotely -*/ - -import ( - "bytes" - "errors" - "fmt" - "time" - - "github.com/coreos/gangplank/internal/spec" - "github.com/opencontainers/runc/libcontainer/user" - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// hopPod describes a remote pod for running Gangplank in a -// cluster remotely. -type hopPod struct { - clusterCtx ClusterContext - js *spec.JobSpec - - image string - ns string - serviceAccount string -} - -// GetClusterCtx returns the cluster context of a hopPod -func (h *hopPod) GetClusterCtx() ClusterContext { - return h.clusterCtx -} - -// hopPod implements the CosaPodder interface. -var _ CosaPodder = &hopPod{} - -// NewHopPod returns a PodBuilder. -func NewHopPod(ctx ClusterContext, image, serviceAccount, workDir string, js *spec.JobSpec) CosaPodder { - cosaSrvDir = workDir - return &hopPod{ - clusterCtx: ctx, - image: image, - serviceAccount: serviceAccount, - js: js, - } -} - -// Exec Gangplank locally through a remote/hop pod that runs -// Gangplank in a cluster. -func (h *hopPod) WorkerRunner(term termChan, _ []v1.EnvVar) error { - if h.image == "" { - return errors.New("image must not be empty") - } - if h.serviceAccount == "" { - return errors.New("service account must not be empty") - } - return clusterRunner(term, h, nil) -} - -// getSpec createa a very generic pod that can run on any Cluster. -// The pod will mimic a build api pod. -func (h *hopPod) getPodSpec([]v1.EnvVar) (*v1.Pod, error) { - log.Debug("Generating hop podspec") - - u, _ := user.CurrentUser() - podName := fmt.Sprintf("%s-hop-%d", - u.Name, time.Now().UTC().Unix(), - ) - log.Infof("Creating pod %s", podName) - - buf := bytes.Buffer{} - if err := h.js.WriteYAML(&buf); err != nil { - return nil, err - } - - script := `#!/bin/bash -xe -find /run/secrets -cat > jobspec.yaml </<< data.key >> - - In the above example, it would: - - set the envVar "AWS_DEFAULT_REGION" to "us-east-1" - - write config to /srv/secrets/my-super-secret-AWS-keys/config - and set AWS_CONFIG_FILE to that location. - -*/ - -type varMap map[string]string - -type secretMap struct { - label string - envVarMap varMap - fileVarMap varMap -} - -// SecretMapper maps a secretMap -type SecretMapper interface { - Setup() error -} - -var ( - // create the secret mappings for the supported Clouds - secretMaps = []*secretMap{ - // Definition for Aliyun - { - label: "aliyun", - fileVarMap: varMap{ - "config.json": "ALIYUN_CONFIG_FILE", - }, - }, - // Definition for AWS - { - label: "aws", - envVarMap: varMap{ - "aws_access_key_id": "AWS_ACCESS_KEY_ID", - "aws_secret_access_key": "AWS_SECRET_ACCESS_KEY", - "aws_default_region": "AWS_DEFAULT_REGION", - "aws_ca_bundle": "AWS_CA_BUNDLE", - }, - fileVarMap: varMap{ - "config": "AWS_CONFIG_FILE", - }, - }, - // Definition for AWS-CN - // Must use AWS_CN_CONFIG_FILE for environment variable name otherwise it - // overwrites the plain aws secret - { - label: "aws-cn", - fileVarMap: varMap{ - "config": "AWS_CN_CONFIG_FILE", - }, - }, - // Definition for Azure - { - label: "azure", - fileVarMap: varMap{ - "azure.json": "AZURE_CONFIG", - "azure.pem": "AZURE_CERT_KEY", - "azureProfile.json": "AZURE_PROFILE", - }, - }, - // Definition for GCP - { - label: "gcp", - fileVarMap: varMap{ - // gce is the legacy name for GCP - "gce.json": "GCP_IMAGE_UPLOAD_CONFIG", - "gcp.json": "GCP_IMAGE_UPLOAD_CONFIG", - }, - }, - // Definition of Internal CA - { - label: "internal-ca", - fileVarMap: varMap{ - "ca.crt": "SSL_CERT_FILE", - }, - }, - // Push Secret - { - label: "push-secret", - fileVarMap: varMap{ - "docker.cfg": "PUSH_AUTH_JSON", - "docker.json": "PUSH_AUTH_JSON", - }, - }, - // Pull Secret - { - label: "pull-secret", - fileVarMap: varMap{ - "docker.cfg": "PULL_AUTH_JSON", - "docker.json": "PULL_AUTH_JSON", - }, - }, - // Koji Keytab - { - label: "keytab", - fileVarMap: varMap{ - "keytab": "KOJI_KEYTAB", - "principle": "KOJI_PRINCIPAL", - "config": "KOJI_CONFIG", - }, - }, - } -) - -// Get SecretMapping returns the secretMap and true if found. -func getSecretMapping(s string) (*secretMap, bool) { - for _, v := range secretMaps { - if v.label == s { - return v, true - } - } - return nil, false -} - -// writeSecretEnvVars creates envVars. -func (sm *secretMap) writeSecretEnvVars(d map[string][]byte, ret *[]string) error { - for k, v := range d { - envKey, ok := sm.envVarMap[k] - if !ok { - continue - } - log.Debugf("Set envVar %s from secret", envKey) - *ret = append(*ret, fmt.Sprintf("%s=%s", envKey, strings.TrimSuffix(string(v), "\n"))) - } - return nil -} - -// writeSecretFiles writes secrets to their location based on the map. -func (sm *secretMap) writeSecretFiles(toDir, name string, d map[string][]byte, ret *[]string) error { - sDir := filepath.Join(toDir, name) - if err := os.MkdirAll(sDir, 0755); err != nil { - return err - } - for k, v := range d { - eKey, ok := sm.fileVarMap[k] - if !ok { - continue - } - f := filepath.Join(sDir, k) - if err := ioutil.WriteFile(f, v, 0555); err != nil { - return err - } - *ret = append(*ret, fmt.Sprintf("%s=%s", eKey, f)) - } - return nil -} - -// kubernetesSecretSetup looks for matching secrets in the environment matching -// 'coreos-assembler.coreos.com/secret=k' and then maps the secret -// automatically in. "k" must be in the "known" secrets type to be mapped -// automatically. -func kubernetesSecretsSetup(ctx context.Context, ac *kubernetes.Clientset, ns, toDir string) ([]string, error) { - lo := metav1.ListOptions{ - LabelSelector: secretLabelName, - Limit: 100, - } - - var ret []string - - secrets, err := ac.CoreV1().Secrets(ns).List(ctx, lo) - if err != nil { - return ret, nil - } - log.Infof("Found %d secrets to consider", len(secrets.Items)) - - for _, secret := range secrets.Items { - sName := secret.GetObjectMeta().GetName() - labels := secret.GetObjectMeta().GetLabels() - for k, v := range labels { - if k != secretLabelName { - continue - } - m, ok := getSecretMapping(v) - if !ok { - log.Errorf("Unknown secret type for %s found at %s", v, sName) - continue - } - log.Infof("Known secret type for %s found, mapping automatically", v) - if err := m.writeSecretEnvVars(secret.Data, &ret); err != nil { - log.Errorf("Failed to set envVars for %s: %s", sName, err) - } - dirName := fmt.Sprintf(".%s", v) - if err := m.writeSecretFiles(toDir, dirName, secret.Data, &ret); err != nil { - log.Errorf("Failed to set files envVars for %s: %s", sName, err) - } - } - } - return ret, nil -} diff --git a/gangplank/internal/ocp/source_extract.go b/gangplank/internal/ocp/source_extract.go deleted file mode 100644 index 29883df7..00000000 --- a/gangplank/internal/ocp/source_extract.go +++ /dev/null @@ -1,56 +0,0 @@ -package ocp - -import ( - "io" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" -) - -const ( - // sourceBin stores binary input - sourceBin = "source.bin" - - // sourceSubPath is used when extracting binary inputs - sourceSubPath = "source" -) - -// extractInputBinary processes the provided input stream as directed by BinaryBuildSource -// into dir. OpenShift sends binary builds over stdin. To make our life easier, -// use the OpenShift API to process the input. Returns the name of the file -// written. -func recieveInputBinary() (string, error) { - srcd := filepath.Join(cosaSrvDir, sourceSubPath) - if err := os.MkdirAll(srcd, 0777); err != nil { - return "", err - } - _ = os.Chdir(srcd) - defer func() { _ = os.Chdir(cosaSrvDir) }() - - source := apiBuild.Spec.Source.Binary - if source == nil { - log.Debug("No binary payload found") - return "", nil - } - - // If stdin is a file, then write it out to the same name - // as send from the OCP binary - path := filepath.Join(srcd, sourceBin) - if len(source.AsFile) > 0 { - path = filepath.Join(srcd, source.AsFile) - } - log.Infof("Receiving source from STDIN as file %s", path) - - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) - if err != nil { - return "", err - } - defer f.Close() - n, err := io.Copy(f, os.Stdin) - if err != nil { - return "", err - } - log.Infof("Received %d bytes into %s", n, path) - return path, nil -} diff --git a/gangplank/internal/ocp/ssh.go b/gangplank/internal/ocp/ssh.go deleted file mode 100644 index 9fe77bf3..00000000 --- a/gangplank/internal/ocp/ssh.go +++ /dev/null @@ -1,230 +0,0 @@ -package ocp - -import ( - "context" - "fmt" - "io" - "net" - "os" - "strconv" - "strings" - "time" - - "github.com/containers/podman/v3/pkg/terminal" - "github.com/coreos/gangplank/internal/spec" - log "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" -) - -type SSHForwardPort struct { - Host string - User string - Key string - SSHPort int -} - -// getSshMinioForwarder returns an SSHForwardPort from the jobspec -// definition for forwarding a minio server, or nil if forwarding is -// not enabled. -func getSshMinioForwarder(j *spec.JobSpec) *SSHForwardPort { - if j.Minio.SSHForward == "" { - return nil - } - return &SSHForwardPort{ - Host: j.Minio.SSHForward, - User: j.Minio.SSHUser, - Key: j.Minio.SSHKey, - SSHPort: j.Minio.SSHPort, - } -} - -// sshClient is borrowed from libpod, and has been modified to return an sshClient. -func sshClient(user, host, port string, secure bool, identity string) (*ssh.Client, error) { - var signers []ssh.Signer // order Signers are appended to this list determines which key is presented to server - if len(identity) > 0 { - s, err := terminal.PublicKey(identity, []byte("")) - if err != nil { - return nil, fmt.Errorf("%w: failed to parse identity %q", err, identity) - } - signers = append(signers, s) - } - - if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { - c, err := net.Dial("unix", sock) - if err != nil { - return nil, err - } - - agentSigners, err := agent.NewClient(c).Signers() - if err != nil { - return nil, err - } - signers = append(signers, agentSigners...) - } - - var authMethods []ssh.AuthMethod - if len(signers) > 0 { - var dedup = make(map[string]ssh.Signer) - // Dedup signers based on fingerprint, ssh-agent keys override CONTAINER_SSHKEY - for _, s := range signers { - fp := ssh.FingerprintSHA256(s.PublicKey()) - _ = dedup[fp] - dedup[fp] = s - } - - var uniq []ssh.Signer - for _, s := range dedup { - uniq = append(uniq, s) - } - authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) { - return uniq, nil - })) - } - - if len(authMethods) == 0 { - callback := func() (string, error) { - pass, err := terminal.ReadPassword("Login password:") - return string(pass), err - } - authMethods = append(authMethods, ssh.PasswordCallback(callback)) - } - - callback := ssh.InsecureIgnoreHostKey() - if secure { - if port != "22" { - host = fmt.Sprintf("[%s]:%s", host, port) - } - key := terminal.HostKey(host) - if key != nil { - callback = ssh.FixedHostKey(key) - } - } - - return ssh.Dial("tcp", - net.JoinHostPort(host, port), - &ssh.ClientConfig{ - User: user, - Auth: authMethods, - HostKeyCallback: callback, - HostKeyAlgorithms: []string{ - ssh.KeyAlgoRSA, - ssh.KeyAlgoDSA, - ssh.KeyAlgoECDSA256, - ssh.KeyAlgoECDSA384, - ssh.KeyAlgoECDSA521, - ssh.KeyAlgoED25519, - }, - Timeout: 5 * time.Second, - }, - ) -} - -// startMinioAndForwardOverSSH starts minio and forwards the connection over SSH. -func (m *minioServer) startMinioAndForwardOverSSH(ctx context.Context, termCh termChan, errCh chan<- error) error { - sshPort := 22 - if m.overSSH.SSHPort != 0 { - sshPort = m.overSSH.SSHPort - } - sshport := strconv.Itoa(sshPort) - - l := log.WithFields(log.Fields{ - "remote host": m.overSSH.Host, - "remote user": m.overSSH.User, - "port": sshport, - }) - - l.Info("Forwarding local port over SSH to remote host") - - client, err := sshClient(m.overSSH.User, m.overSSH.Host, sshport, false, m.overSSH.Key) - if err != nil { - return err - } - - // Open the remote port over SSH, use empty port definition to have it - // dynamically chosen based on port availabilty on the remote. If - // we don't do this then multiple concurrent gangplank runs will fail - // because they'll try to use the same port. - var remoteConn net.Listener - var remoteSSHport int - // Loop until we've found a common port available locally and remote - for { - remoteConn, err = client.Listen("tcp4", "127.0.0.1:") - if err != nil { - err = fmt.Errorf("%w: failed to open remote port over ssh for proxy", err) - return err - } - remoteSSHport, err = strconv.Atoi(strings.Split(remoteConn.Addr().String(), ":")[1]) - if err != nil { - err = fmt.Errorf("%w: failed to parse remote ssh port from connection", err) - return err - } - log.Infof("The SSH forwarding chose port %d on the remote host", remoteSSHport) - - if getPortOrNext(remoteSSHport) == remoteSSHport { - break - } - - log.Infof("Local Port %d is not available, selecting another port", remoteSSHport) - remoteConn.Close() - } - // Update m.Port in the minioServer definition so the miniocfg - // that gets passed to the remote specifies the correct port for - // the local connection there. - log.Infof("Changing minio port for local and remote (forward) from %v to %v", - m.Port, remoteSSHport) - m.Port = remoteSSHport - - // Now that we know the port let's start the minio server. It's - // highly unlikely to have a port conflict here because we are - // running inside the cosa container where no other services are - // running/listening. - if err := m.start(ctx); err != nil { - return err - } - - // copyIO is a blind copier that copies between source and destination - copyIO := func(src, dest net.Conn) { - defer src.Close() //nolint - defer dest.Close() //nolint - _, _ = io.Copy(src, dest) - } - - // proxy is a helper function that connects the local port to the remoteClient - proxy := func(conn net.Conn) { - proxy, err := net.Dial("tcp4", fmt.Sprintf("127.0.0.1:%d", m.Port)) - if err != nil { - err = fmt.Errorf("%w: failed to open local port for proxy", err) - errCh <- err - return - } - go copyIO(conn, proxy) - go copyIO(proxy, conn) - } - - go func() { - // When the termination signal is recieved, the defers in copyio will be triggered, - // resulting in the go-routines exiting. - <-termCh - l.Info("Shutting down ssh forwarding") - errCh <- remoteConn.Close() - errCh <- client.Close() - }() - - go func() { - // Loop checking for connections or termination. - for { - // For each connection, create a proxy from the remote port to the local port - remoteClient, err := remoteConn.Accept() - if err != nil { - if err == io.EOF { - return - } - log.WithError(err).Warn("SSH Client error") - } - proxy(remoteClient) - } - }() - - return nil -} diff --git a/gangplank/internal/ocp/volumes.go b/gangplank/internal/ocp/volumes.go deleted file mode 100644 index c531b374..00000000 --- a/gangplank/internal/ocp/volumes.go +++ /dev/null @@ -1,260 +0,0 @@ -package ocp - -import ( - "fmt" - "path/filepath" - - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -/* - mountReferance describes secrets or configMaps that are mounted - as volumes. In general, these volumes contain data that is used by - systems level tooling use as Kerberos, CA certs, etc. - The label coreos-assembler.coreos.com/mount-ref is needed in this case -*/ - -// mountReferance is mapping of secrets or a configmap -type mountReferance struct { - volumes []v1.Volume - volumeMounts []v1.VolumeMount - requireData []string - addInitCommands []string -} - -// secretMountRefLabel is used for mounting of secrets -const mountRefLabel = "coreos-assembler.coreos.com/mount-ref" - -var ( - volMaps = map[string]mountReferance{ - // internal-ca should be a fully extracted pem file - "internal-ca": { - volumes: []v1.Volume{ - { - Name: "pki", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - DefaultMode: ptrInt32(444), - SecretName: "", - }, - }, - }, - }, - volumeMounts: []v1.VolumeMount{ - { - Name: "pki", - MountPath: "/etc/pki/ca-trust/source/anchors2/", - }, - }, - }, - - // Push/Pull secrets - "docker.json": { - volumes: []v1.Volume{ - { - Name: "docker-json", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - DefaultMode: ptrInt32(444), - SecretName: "", - }, - }, - }, - }, - volumeMounts: []v1.VolumeMount{ - { - Name: "docker-json", - MountPath: filepath.Join(cosaSrvDir, "secrets", "auths"), - }, - }, - }, - // Koji ConfigMap - "koji-ca": { - volumes: []v1.Volume{ - { - Name: "koji-ca", - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "", - }, - }, - }, - }, - }, - volumeMounts: []v1.VolumeMount{ - { - Name: "koji-ca", - MountPath: "/etc/pki/brew", - }, - }, - }, - - // Koji Configuration ConfigMap - "koji-config": { - volumes: []v1.Volume{ - { - Name: "koji-config", - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "", - }, - }, - }, - }, - }, - volumeMounts: []v1.VolumeMount{ - { - Name: "koji-config", - MountPath: "/etc/koji.conf.d", - }, - }, - }, - - // Kerberos Configuration ConfigMap: usually used by the brew code. - "krb5.conf": { - volumes: []v1.Volume{ - { - Name: "koji-kerberos", - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: "", - }, - }, - }, - }, - }, - volumeMounts: []v1.VolumeMount{ - { - Name: "koji-kerberos", - MountPath: "/etc/krb5.conf.d", - }, - }, - }, - } -) - -// ptrInt32 converts an int32 to a ptr of the int32 -func ptrInt32(i int32) *int32 { return &i } - -// byteField represents a configMap's data fields -type byteFields map[string][]byte - -// stringFields represent a secret's data fields -type stringFields map[string]string - -// toStringFields is used to convert from a byteFields to a stringFields -func toStringFields(bf byteFields) stringFields { - d := make(stringFields) - for k, v := range bf { - d[k] = string(v) - } - return d -} - -// addVolumesFromConfigMapLabels discovers configMaps with matching labels and if known, -// adds the defined volume mount from volMaps. -func (cp *cosaPod) addVolumesFromConfigMapLabels() error { - ac, ns, err := GetClient(cp.clusterCtx) - if err != nil { - return err - } - lo := metav1.ListOptions{ - LabelSelector: mountRefLabel, - Limit: 100, - } - - cfgMaps, err := ac.CoreV1().ConfigMaps(ns).List(cp.clusterCtx, lo) - if err != nil { - return err - } - log.Infof("Found %d configMaps to consider for mounting", len(cfgMaps.Items)) - - for _, cfgMap := range cfgMaps.Items { - if err := cp.addVolumeFromObjectLabel(cfgMap.GetObjectMeta(), cfgMap.Data); err != nil { - return err - } - log.WithField("secret", cfgMap.Name).Info("mounts defined for secret") - } - - return nil -} - -// addVolumesFromSecretLabels discovers secrets with matching labels and if known, -// adds the defined volume mount from volMaps. -func (cp *cosaPod) addVolumesFromSecretLabels() error { - ac, ns, err := GetClient(cp.clusterCtx) - if err != nil { - return err - } - lo := metav1.ListOptions{ - LabelSelector: mountRefLabel, - Limit: 100, - } - - secrets, err := ac.CoreV1().Secrets(ns).List(cp.clusterCtx, lo) - if err != nil { - return err - } - log.Infof("Found secret %d to consider for mounting", len(secrets.Items)) - - for _, secret := range secrets.Items { - if err := cp.addVolumeFromObjectLabel(secret.GetObjectMeta(), toStringFields(secret.Data)); err != nil { - return err - } - log.WithField("secret", secret.Name).Info("mounts defined for secret") - } - return nil -} - -// addVolumeFromObjectLabel is a helper that recieves an object and data and looks up -// the object's name from volMaps. If a mapping is found, then the object is added to -// cosaPod's definition. -func (cp *cosaPod) addVolumeFromObjectLabel(obj metav1.Object, fields stringFields) error { - oName := obj.GetName() - labels := obj.GetLabels() - for k, v := range labels { - if k != mountRefLabel { - continue - } - elem, ok := volMaps[v] - if !ok { - continue - } - - // Check for required elements to be in the secret - missing := make([]string, len(elem.requireData)) - for _, r := range elem.requireData { - _, found := fields[r] - if !found { - missing = append(missing, r) - } - } - if len(missing) > 0 { - return fmt.Errorf("object %s is missing required elements %v", oName, missing) - } - - // Set the object reference - for i := range elem.volumes { - log.WithField("volume", elem.volumes[i].Name).Info("Added mount definition") - if elem.volumes[i].VolumeSource.Secret != nil { - elem.volumes[i].VolumeSource.Secret.SecretName = oName - log.WithField("secret", oName).Info("Added secretRef volume mount") - } - if elem.volumes[i].VolumeSource.ConfigMap != nil { - elem.volumes[i].VolumeSource.ConfigMap.LocalObjectReference.Name = oName - log.WithField("configMaps", oName).Info("Added configmap volume mount") - } - } - - // Set the volumes in the defualt pod spec - cp.volumes = append(cp.volumes, elem.volumes...) - cp.volumeMounts = append(cp.volumeMounts, elem.volumeMounts...) - cp.ocpInitCommand = append(cp.ocpInitCommand, elem.addInitCommands...) - } - return nil -} diff --git a/gangplank/internal/ocp/worker.go b/gangplank/internal/ocp/worker.go deleted file mode 100644 index 46353c08..00000000 --- a/gangplank/internal/ocp/worker.go +++ /dev/null @@ -1,473 +0,0 @@ -package ocp - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/coreos/coreos-assembler-schema/cosa" - "github.com/coreos/gangplank/internal/spec" - buildapiv1 "github.com/openshift/api/build/v1" - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// workSpec is a Builder. -var _ Builder = &workSpec{} - -// workerBuild Dir is hard coded. Workers always read builds relative to their -// local paths and assume the build location is on /srv -var workerBuildDir string = filepath.Join("/srv", "builds") - -// workSpec define job for remote worker to do -// A workSpec is dispatched by a builder and is tightly coupled to -// to the dispatching pod. -type workSpec struct { - RemoteFiles []*RemoteFile `json:"remotefiles"` - JobSpec spec.JobSpec `json:"jobspec"` - ExecuteStages []string `json:"executeStages"` - APIBuild *buildapiv1.Build `json:"apiBuild"` - Return *Return `json:"return"` -} - -// CosaWorkPodEnvVarName is the envVar used to identify WorkSpec json. -const CosaWorkPodEnvVarName = "COSA_WORK_POD_JSON" - -// newWorkSpec returns a workspec from the environment -func newWorkSpec(ctx ClusterContext) (*workSpec, error) { - w, ok := os.LookupEnv(CosaWorkPodEnvVarName) - if !ok { - return nil, ErrNotWorkPod - } - r := strings.NewReader(w) - ws := workSpec{} - if err := ws.Unmarshal(r); err != nil { - return nil, err - } - if _, err := os.Stat(cosaSrvDir); os.IsNotExist(err) { - return nil, fmt.Errorf("context dir %q does not exist", cosaSrvDir) - } - - if err := os.Chdir(cosaSrvDir); err != nil { - return nil, fmt.Errorf("failed to switch to context dir: %s: %v", cosaSrvDir, err) - } - - log.Info("Running as a worker pod") - return &ws, nil -} - -// Unmarshal decodes an io.Reader to a workSpec. -func (ws *workSpec) Unmarshal(r io.Reader) error { - d, err := ioutil.ReadAll(r) - if err != nil { - return err - } - if err := json.Unmarshal(d, &ws); err != nil { - return err - } - return nil -} - -// Marshal returns the JSON of a WorkSpec. -func (ws *workSpec) Marshal() ([]byte, error) { - return json.Marshal(ws) -} - -// Exec executes the work spec tasks. -func (ws *workSpec) Exec(ctx ClusterContext) error { - apiBuild = ws.APIBuild - - envVars := os.Environ() - - // Check stdin for binary input. - inF, err := recieveInputBinary() - if err == nil && inF != "" { - log.WithField("file", inF).Info("Worker recieved binary input") - - f, err := os.Open(inF) - if err != nil { - return err - } - if err := decompress(f, cosaSrvDir); err != nil { - return err - } - } - - // Workers always will use /srv. The shell/Python code of COSA expects - // /srv to be on its own volume. - if err := os.Chdir(cosaSrvDir); err != nil { - return fmt.Errorf("unable to switch to %s: %w", cosaSrvDir, err) - } - - // Setup the incluster client - ac, pn, err := k8sInClusterClient() - if err == ErrNotInCluster { - log.Info("Worker is out-of-clstuer, no secrets will be available") - } else if err != nil { - return fmt.Errorf("failed create a kubernetes client: %w", err) - } - - // Only setup secrets for in-cluster use - if ac != nil { - ks, err := kubernetesSecretsSetup(ctx, ac, pn, cosaSrvDir) - if err != nil { - log.Errorf("Failed to setup Service Account Secrets: %v", err) - } - envVars = append(envVars, ks...) - } - - // Identify the buildConfig that launched this instance. - if apiBuild != nil { - bc := apiBuild.Annotations[buildapiv1.BuildConfigAnnotation] - bn := apiBuild.Annotations[buildapiv1.BuildNumberAnnotation] - log.Infof("Worker is part of buildconfig.openshift.io/%s-%s", bc, bn) - if err := cosaInit(ws.JobSpec); err != nil && err != ErrNoSourceInput { - return fmt.Errorf("failed to clone recipe: %w", err) - } - } else { - // Inform that Gangplank is running as an unbound worker. Unbound workers - // require something else to create and manage the pod, such as Jenkins. - log.Infof("Pod is running as a unbound worker") - } - - // Fetch the remote files and write them to the local path. - for _, f := range ws.RemoteFiles { - destf := filepath.Join(cosaSrvDir, f.Bucket, f.Object) - log.Infof("Fetching remote file %s/%s", f.Bucket, f.Object) - // Decompress the file if needed. - if f.Compressed { - if err := f.Extract(ctx, cosaSrvDir); err != nil { - return fmt.Errorf("failed to decompress from %s/%s: %w", f.Bucket, f.Object, err) - } - } - // Write the file. - if err := f.WriteToPath(ctx, destf); err != nil { - return fmt.Errorf("failed to write file from %s/%s: %w", f.Bucket, f.Object, err) - } - } - - // Populate `src/config/` - for _, r := range ws.JobSpec.Recipe.Repos { - path, err := r.Writer(filepath.Join("src", "config")) - if err != nil { - return fmt.Errorf("failed to write remote repo: %v", err) - } - log.WithFields(log.Fields{ - "path": path, - "url": r.URL, - }).Info("Wrote repo definition from url") - } - - // Ensure on shutdown that we record information that might be - // needed for prosperity. First, build artifacts are sent off the - // remote object storage. Then meta.json is written to /dev/console - // and /dev/termination-log. - defer func() { - if ws.Return != nil { - err := ws.Return.Run(ctx, ws) - log.WithError(err).Info("Processed Uploads") - } - - b, _, err := cosa.ReadBuild(workerBuildDir, "", cosa.BuilderArch()) - if err != nil && b != nil { - _ = b.WriteMeta(os.Stdout.Name(), false) - - err := b.WriteMeta("/dev/termination-log", false) - log.WithFields(log.Fields{ - "err": err, - "file": "/dev/termination-log", - "buildID": b.BuildID, - }).Info("wrote termination log") - } - }() - - // Expose the jobspec and meta.json (if its available) for templating. - mBuild, _, _ := cosa.ReadBuild(workerBuildDir, "", cosa.BuilderArch()) - if mBuild == nil { - mBuild = new(cosa.Build) - } - rd := &spec.RenderData{ - JobSpec: &ws.JobSpec, - Meta: mBuild, - } - - // Ensure a latest symlink to the build exists - if mBuild.BuildID != "" { - if err := func() error { - pwd, _ := os.Getwd() - defer os.Chdir(pwd) //nolint - - if err := os.Chdir(filepath.Join(cosaSrvDir, "builds")); err != nil { - return fmt.Errorf("unable to change to builds dir: %v", err) - } - - // create a relative symlink in the builds dir - latestLink := filepath.Join("latest") - latestTarget := filepath.Join(mBuild.BuildID) - if _, err := os.Lstat(latestLink); os.IsNotExist(err) { - if err := os.Symlink(latestTarget, latestLink); err != nil { - return fmt.Errorf("unable to create latest symlink from %s to %s", latestTarget, latestLink) - } - } - return nil - }(); err != nil { - return err - } - } - - // Range over the stages and perform the actual work. - for _, s := range ws.ExecuteStages { - stage, err := ws.JobSpec.GetStage(s) - l := log.WithFields(log.Fields{ - "stage id": s, - "build artifacts": stage.BuildArtifacts, - "required artifacts": stage.RequireArtifacts, - "optional artifacts": stage.RequestArtifacts, - "commands": stage.Commands, - "return files": stage.ReturnFiles, - }) - l.Info("Executing Stage") - - if err != nil { - l.WithError(err).Info("Error fetching stage") - return err - } - - if err := stage.Execute(ctx, rd, envVars); err != nil { - l.WithError(err).Error("failed stage execution") - return err - } - - next, _, _ := cosa.ReadBuild(workerBuildDir, "", cosa.BuilderArch()) - if next != nil && next.BuildArtifacts != nil && (mBuild.BuildArtifacts == nil || mBuild.BuildArtifacts.Ostree.Sha256 != next.BuildArtifacts.Ostree.Sha256) { - log.Debug("Stage produced a new OStree") - - // push for other defined registries - for _, v := range ws.JobSpec.PublishOscontainer.Registries { - if err := pushOstreeToRegistry(ctx, &v, next); err != nil { - log.WithError(err).Warningf("Push to registry %s failed", v.URL) - return err - } - } - - // push for custom build strategy - if err := uploadCustomBuildContainer(ctx, ws.JobSpec.PublishOscontainer.BuildStrategyTLSVerify, ws.APIBuild, next); err != nil { - log.WithError(err).Warning("Push to BuildSpec registry failed") - return err - } - } - - if stage.ReturnCache { - l.WithField("tarball", cacheTarballName).Infof("Sending %s back as a tarball", cosaSrvCache) - if err := uploadPathAsTarBall(ctx, cacheBucket, cacheTarballName, cosaSrvCache, "", true, ws.Return); err != nil { - return err - } - } - - if stage.ReturnCacheRepo { - l.WithField("tarball", cacheRepoTarballName).Infof("Sending %s back as a tarball", cosaSrvTmpRepo) - if err := uploadPathAsTarBall(ctx, cacheBucket, cacheRepoTarballName, cosaSrvTmpRepo, "", true, ws.Return); err != nil { - return err - } - } - if len(stage.ReturnFiles) != 0 { - l.WithField("files", stage.ReturnFiles).Infof("Sending requested files back to remote") - if err := uploadReturnFiles(ctx, cacheBucket, stage.ReturnFiles, ws.Return); err != nil { - return err - } - } - - } - log.Infof("Finished execution") - return nil -} - -// getEnvVars returns the envVars to be exposed to the worker pod. -// When `newWorkSpec` is called, the envVar will read the embedded string JSON -// and the worker will get its configuration. -func (ws *workSpec) getEnvVars() ([]v1.EnvVar, error) { - d, err := ws.Marshal() - if err != nil { - return nil, err - } - - evars := []v1.EnvVar{ - { - Name: CosaWorkPodEnvVarName, - Value: string(d), - }, - { - Name: "XDG_RUNTIME_DIR", - Value: cosaSrvDir, - }, - { - Name: "COSA_FORCE_ARCH", - Value: cosa.BuilderArch(), - }, - } - return evars, nil -} - -// pushOstreetoRegistry pushes the OStree to the defined registry location. -func pushOstreeToRegistry(ctx ClusterContext, push *spec.Registry, build *cosa.Build) error { - if push == nil { - return errors.New("unable to push to nil registry") - } - if build == nil { - return errors.New("unable to push to registry: cosa build is nil") - } - - // TODO: move this to a common validator - if push.URL == "" { - return errors.New("push registry URL is emtpy") - } - - cluster, _ := GetCluster(ctx) - - registry, registryPath := getPushTagless(push.URL) - pushPath := fmt.Sprintf("%s/%s", registry, registryPath) - - authPath := filepath.Join(cosaSrvDir, ".docker", "config.json") - authDir := filepath.Dir(authPath) - if err := os.MkdirAll(authDir, 0755); err != nil { - return fmt.Errorf("failed to directory path for push secret") - } - - switch v := strings.ToLower(string(push.SecretType)); v { - case spec.PushSecretTypeInline: - if err := ioutil.WriteFile(authPath, []byte(push.Secret), 0644); err != nil { - return fmt.Errorf("failed to write the inline secret to auth.json: %v", err) - } - case spec.PushSecretTypeCluster: - if !cluster.inCluster { - return errors.New("cluster secrets pushes are invalid out-of-cluster") - } - if err := writeDockerSecret(ctx, push.Secret, authPath); err != nil { - return fmt.Errorf("failed to locate the secret %s: %v", push.Secret, err) - } - case spec.PushSecretTypeToken: - if err := tokenRegistryLogin(ctx, push.TLSVerify, registry); err != nil { - return fmt.Errorf("failed to login into registry: %v", err) - } - // container XDG_RUNTIME_DIR is set to cosaSrvDir - authPath = filepath.Join(cosaSrvDir, "containers", "auth.json") - default: - return fmt.Errorf("secret type %s is unknown for push registries", push.SecretType) - } - - defer func() { - // Remove any logins that could interfere later with subsequent pushes. - _ = os.RemoveAll(filepath.Join(cosaSrvDir, "containers")) - _ = os.RemoveAll(filepath.Join(cosaSrvDir, ".docker")) - }() - - baseEnv := append( - os.Environ(), - "FORCE_UNPRIVILEGED=1", - fmt.Sprintf("REGISTRY_AUTH_FILE=%s", authPath), - // Tell the tools where to find the home directory - fmt.Sprintf("HOME=%s", cosaSrvDir), - ) - - tlsVerify := true - if push.TLSVerify != nil && !*push.TLSVerify { - tlsVerify = false - } - - l := log.WithFields( - log.Fields{ - "auth json": authPath, - "final push": push.URL, - "push path": pushPath, - "registry": registry, - "tls verification": tlsVerify, - "push definition": push, - }) - l.Info("Pushing to remote registry") - - // pushArgs invokes cosa upload code which creates a named tag - pushArgs := []string{ - "/usr/bin/coreos-assembler", "upload-oscontainer", - fmt.Sprintf("--name=%s", pushPath), - } - // copy the pushed image to the expected tag - copyArgs := []string{ - "skopeo", "copy", - fmt.Sprintf("docker://%s:%s", pushPath, build.BuildID), - fmt.Sprintf("docker://%s", push.URL), - } - - if !tlsVerify { - log.Warnf("TLS Verification has been disable for push to %s", push.URL) - copyArgs = append(copyArgs, "--src-tls-verify=false", "--dest-tls-verify=false") - baseEnv = append(baseEnv, "DISABLE_TLS_VERIFICATION=1") - } - - for _, args := range [][]string{pushArgs, copyArgs} { - l.WithField("cmd", args).Debug("Calling external tool ") - cmd := exec.CommandContext(ctx, args[0], args[1:]...) - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - cmd.Env = baseEnv - if err := cmd.Run(); err != nil { - return errors.New("upload to registry failed") - } - } - return nil -} - -// writeDockerSecret writes the .dockerCfg or .dockerconfig to the correct path. -// It accepts the cluster context, the name of the secret and the location to write to. -func writeDockerSecret(ctx ClusterContext, clusterSecretName, authPath string) error { - ac, ns, err := GetClient(ctx) - if err != nil { - return fmt.Errorf("unable to fetch cluster client: %v", err) - } - secret, err := ac.CoreV1().Secrets(ns).Get(ctx, clusterSecretName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to query for secret %s: %v", apiBuild.Spec.Output.PushSecret.Name, err) - } - if secret == nil { - return fmt.Errorf("secret is empty") - } - - var key string - switch secret.Type { - case v1.SecretTypeDockerConfigJson: - key = v1.DockerConfigJsonKey - case v1.SecretTypeDockercfg: - key = v1.DockerConfigKey - case v1.SecretTypeOpaque: - if _, ok := secret.Data["docker.json"]; ok { - key = "docker.json" - } else if _, ok := secret.Data["docker.cfg"]; ok { - key = "docker.cfg" - } - default: - return fmt.Errorf("writeDockerSecret is not supported for secret type %s", secret.Type) - } - - data, ok := secret.Data[key] - if !ok { - return fmt.Errorf("secret %s of type %s is malformed: missing %s", secret.Name, secret.Type, key) - } - - log.WithFields(log.Fields{ - "local path": authPath, - "type": string(secret.Type), - "secret key": key, - "name": secret.Name, - }).Info("Writing push secret") - - if err := ioutil.WriteFile(authPath, data, 0444); err != nil { - return fmt.Errorf("failed writing secret %s to %s", secret.Name, authPath) - } - return nil -} diff --git a/gangplank/internal/spec/cli.go b/gangplank/internal/spec/cli.go deleted file mode 100644 index 5c4f3951..00000000 --- a/gangplank/internal/spec/cli.go +++ /dev/null @@ -1,131 +0,0 @@ -package spec - -import ( - "os" - "strings" - - log "github.com/sirupsen/logrus" - "github.com/spf13/pflag" -) - -/* - cli.go supports creating inferred jobspecs. -*/ - -const ( - fedoraGitURL = "https://github.com/coreos/fedora-coreos-config" - fedoraGitRef = "testing-devel" - - rhcosGitURL = "https://github.com/openshift/os" - rhcosGitRef = "main" -) - -// Default to building Fedora -var ( - gitRef = fedoraGitRef - gitURL = fedoraGitURL - - // repos is a list a URLs that is added to the Repos. - repos []string - - // copy-build is an extra build to copy build metadata for - copyBuild string -) - -func init() { - r, ok := os.LookupEnv("COSA_YUM_REPOS") - if ok { - repos = strings.Split(r, ",") - } - - o, _ := os.LookupEnv("COSA_GANGPLANK_OS") - if strings.ToLower(o) == "rhcos" { - gitRef = rhcosGitRef - gitURL = rhcosGitURL - } - if strings.ToLower(o) == "fcos" { - gitRef = fedoraGitRef - gitURL = fedoraGitURL - } -} - -// strPtr is a helper for returning a string pointer -func strPtr(s string) *string { return &s } - -// AddCliFlags returns the pflag set for use in the CLI. -func (js *JobSpec) AddCliFlags(cmd *pflag.FlagSet) { - - // Define the job definition - cmd.StringVar(&js.Job.BuildName, "job-buildname", js.Job.BuildName, "job name to build") - cmd.StringVar(&js.Job.VersionSuffix, "job-suffix", js.Job.VersionSuffix, "job suffix") - cmd.BoolVar(&js.Job.IsProduction, "job-producution", js.Job.IsProduction, "job is a production job") - cmd.StringSliceVar(&repos, "repo", repos, "yum repos to include for base builds") - - // Default to building a fedora build - if js.Recipe.GitRef == "" { - js.Recipe.GitRef = gitRef - } - if js.Recipe.GitURL == "" { - js.Recipe.GitURL = gitURL - } - - // Define the recipe - cmd.StringVar(&js.Recipe.GitRef, "git-ref", js.Recipe.GitRef, "Git ref for recipe") - cmd.StringVar(&js.Recipe.GitURL, "git-url", js.Recipe.GitURL, "Git URL for recipe") - cmd.StringVar(&js.Recipe.GitCommit, "git-commit", "", "Optional Git commit to reset repo to for recipe") - - // Define any extra builds that we want to copy build metadata for - cmd.StringVar(©Build, "copy-build", "", "Optional: extra build to copy build metadata for") -} - -// AddRepos adds an repositories from the CLI -func (js *JobSpec) AddRepos() { - // Add in repositories - for _, r := range repos { - if r != "" { - js.Recipe.Repos = append( - js.Recipe.Repos, - &Repo{ - URL: &r, - }) - } - } -} - -// AddCopyBuild adds --copy-build from the CLI -func (js *JobSpec) AddCopyBuild() { - if copyBuild != "" { - log.Infof("Adding copy build meta for %s from the CLI", copyBuild) - js.CopyBuild = copyBuild - } -} - -// AddCommands adds commands to a stage -func (s *Stage) AddCommands(args []string) { - s.Commands = append(s.Commands, args...) -} - -// AddReturnFiles adds return files to a stage -func (s *Stage) AddReturnFiles(args []string) { - s.ReturnFiles = append(s.ReturnFiles, args...) -} - -// AddRequires adds in requires based on the arifacts that a stage requires -// inconsideration of what the stage builds -func (s *Stage) AddRequires(args []string) { - for _, req := range args { - add := true - for _, builds := range s.BuildArtifacts { - if isBaseArtifact(req) { - req = "base" - } - if builds == req { - add = false - break - } - } - if add { - s.RequireArtifacts = append(s.RequireArtifacts, req) - } - } -} diff --git a/gangplank/internal/spec/clone.go b/gangplank/internal/spec/clone.go deleted file mode 100644 index bb7850dc..00000000 --- a/gangplank/internal/spec/clone.go +++ /dev/null @@ -1,53 +0,0 @@ -package spec - -import ( - "io/ioutil" - "os" - "os/exec" - "path/filepath" - - log "github.com/sirupsen/logrus" -) - -// DefaultJobSpecFile is the default JobSpecFile name. -const DefaultJobSpecFile = "jobspec.yaml" - -// JobSpecFromRepo clones a git repo and returns the jobspec and error. -func JobSpecFromRepo(url, ref, specFile string) (JobSpec, error) { - // Fetch the remote jobspec - var j JobSpec - if url == "" { - log.Debug("jobpsec url is not defined, skipping") - return j, nil - } - - tmpd, err := ioutil.TempDir("", "arrmatey") - if err != nil { - return j, err - } - defer os.RemoveAll(tmpd) - - // Clone the JobSpec Repo - jsD := filepath.Join(tmpd, "jobspec") - args := []string{"clone"} - if ref != "" { - args = append(args, "--branch", ref) - } - args = append(args, url, jsD) - cmd := exec.Command("git", args...) - if err := cmd.Run(); err != nil { - return j, err - } - - jsF := specFile - if jsF == "" { - jsF = DefaultJobSpecFile - } - ns, err := JobSpecFromFile(filepath.Join(jsD, jsF)) - if err != nil { - return j, err - } - log.Infof("found jobspec for %q", ns.Job.BuildName) - - return ns, nil -} diff --git a/gangplank/internal/spec/clouds.go b/gangplank/internal/spec/clouds.go deleted file mode 100644 index 6980facd..00000000 --- a/gangplank/internal/spec/clouds.go +++ /dev/null @@ -1,210 +0,0 @@ -package spec - -import ( - "fmt" - "strings" -) - -type Cloud interface { - GetPublishCommand(string) (string, error) -} - -// Aliyun is nested under CloudsCfgs and describes where -// the Aliyun/Alibaba artifacts should be uploaded to. -type Aliyun struct { - Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty"` - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - Public bool `yaml:"public,omitempty" json:"public,omitempty"` - Regions []string `yaml:"regions,omitempty" json:"regions,omitempty"` -} - -// GetPublishCommand returns the cosa upload command for Aliyun -func (a *Aliyun) GetPublishCommand(buildID string) (string, error) { - if buildID == "" { - return "", fmt.Errorf("No build provided") - } - - if !a.Enabled { - return "", nil - } - - if a.Public { - return "", fmt.Errorf("Public is not supported on Aliyun") - } - - baseCmd := "coreos-assembler buildextend-aliyun" - args := []string{"--upload", - fmt.Sprintf("--build=%s", buildID), - fmt.Sprintf("--bucket=s3://%s", a.Bucket)} - - if len(a.Regions) > 0 { - args = append(args, fmt.Sprintf("--region=%s", a.Regions[0])) - } - - cmd := fmt.Sprintf("%s %s", baseCmd, strings.Join(args, " ")) - return cmd, nil -} - -// Aws describes the upload options for AWS images -// AmiPath: the bucket patch for pushing the AMI name -// Public: when true, mark as public -// Geo: the abbreviated AWS region, i.e aws-cn would be `cn` -// GrantUser: users to grant access to ami -// GrantUserSnapshot: users to grant access to snapshot -// Regions: name of AWS regions to push to. - -type Aws struct { - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - AmiPath string `yaml:"ami_path,omitempty" json:"ami_path,omitempty"` - Geo string `yaml:"geo,omitempty" json:"geo,omitempty"` - GrantUser []string `yaml:"grant_user,omitempty" json:"grant_user,omitempty"` - GrantUserSnapshot []string `yaml:"grant_user_snapshot,omitempty" json:"grant_user_snapshot,omitempty"` - Public bool `yaml:"public,omitempty" json:"public,omitempty"` - Regions []string `yaml:"regions,omitempty" json:"regions,omitempty"` -} - -// GetPublishCommand returns the cosa upload command for Aws -func (a *Aws) GetPublishCommand(buildID string) (string, error) { - if buildID == "" { - return "", fmt.Errorf("no build provided") - } - - if !a.Enabled { - return "", nil - } - - baseCmd := "coreos-assembler buildextend-aws" - args := []string{"--upload", - fmt.Sprintf("--build=%s", buildID), - fmt.Sprintf("--bucket=s3://%s", a.AmiPath)} - - if len(a.Regions) > 0 { - args = append(args, fmt.Sprintf("--region=%s", a.Regions[0])) - } - - if len(a.GrantUser) > 0 { - args = append(args, fmt.Sprintf("--grant-user %s", strings.Join(a.GrantUser, ","))) - } - - if len(a.GrantUserSnapshot) > 0 { - args = append(args, fmt.Sprintf("--grant-user-snapshot %s", strings.Join(a.GrantUserSnapshot, ","))) - } - - var env string - if a.Geo != "" { - env = fmt.Sprintf("AWS_CONFIG_FILE=$AWS_%s_CONFIG_FILE", - strings.ToUpper(a.Geo)) - } - - cmd := fmt.Sprintf("%s %s %s", env, baseCmd, strings.Join(args, " ")) - return cmd, nil -} - -// Azure describes upload options for Azure images. -// Enabled: upload if true -// ResourceGroup: the name of the Azure resource group -// StorageAccount: name of the storage account -// StorageContainer: name of the storage container -// StorageLocation: name of the Azure region, i.e. us-east-1 -type Azure struct { - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - ResourceGroup string `yaml:"resource_group,omitempty" json:"resource_group,omitempty"` - StorageAccount string `yaml:"storage_account,omitempty" json:"stoarge_account,omitempty"` - StorageContainer string `yaml:"storage_container,omitempty" json:"storage_container,omitempty"` - StorageLocation string `yaml:"storage_location,omitempty" json:"storage_location,omitempty"` - Force bool `yaml:"force,omitempty" json:"force,omitempty"` -} - -// GetPublishCommand returns the cosa upload command for Azure -func (a *Azure) GetPublishCommand(buildID string) (string, error) { - if buildID == "" { - return "", fmt.Errorf("no build provided") - } - - if !a.Enabled { - return "", nil - } - - baseCmd := "coreos-assembler buildextend-azure" - args := []string{"--upload", - fmt.Sprintf("--build %s", buildID), - "--auth $AZURE_CONFIG", - fmt.Sprintf("--container %s", a.StorageContainer), - "--profile $AZURE_PROFILE", - fmt.Sprintf("--resource-group %s", a.ResourceGroup), - fmt.Sprintf("--storage-account %s", a.StorageAccount)} - - if a.Force { - args = append(args, "--force") - } - - cmd := fmt.Sprintf("%s %s", baseCmd, strings.Join(args, " ")) - return cmd, nil -} - -// Gcp describes deploying to the GCP environment -// Bucket: name of GCP bucket to store image in -// Enabled: when true, publish to GCP -// Project: name of the GCP project to use -// CreateImage: Whether or not to create an image in GCP after upload -// Deprecated: If the image should be marked as deprecated -// Description: The description that should be attached to the image -// Enabled: toggle for uploading to GCP -// Family: GCP image family to attach image to -// License: The licenses that should be attached to the image -// LogLevel: log level--DEBUG, WARN, INFO -// Project: GCP project name -// Public: If the image should be given public ACLs -type Gcp struct { - Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty"` - CreateImage bool `yaml:"create_image" json:"create_image"` - Deprecated bool `yaml:"deprecated" json:"deprecated"` - Description string `yaml:"description" json:"description"` - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - Family string `yaml:"family" json:"family"` - License []string `yaml:"license" json:"license"` - LogLevel string `yaml:"log_level" json:"log_level"` - Project string `yaml:"project,omitempty" json:"project,omitempty"` - Public bool `yaml:"public,omitempty" json:"public,omitempty"` -} - -// GetPublishCommand returns the cosa upload command for GCP -func (g *Gcp) GetPublishCommand(buildID string) (string, error) { - if buildID == "" { - return "", fmt.Errorf("no build provided") - } - - if !g.Enabled { - return "", nil - } - - baseCmd := "coreos-assembler buildextend-gcp" - args := []string{"--upload", - fmt.Sprintf("--build %s", buildID), - fmt.Sprintf("--project %s", g.Project), - fmt.Sprintf("--bucket gs://%s", g.Bucket), - "--json $GCP_IMAGE_UPLOAD_CONFIG"} - - if g.Public { - args = append(args, "--public") - } - - if g.CreateImage { - args = append(args, "--create-image=true") - } - - if g.Family != "" { - args = append(args, fmt.Sprintf("--family %s", g.Family)) - } - - for _, f := range g.License { - args = append(args, fmt.Sprintf("--license %s", f)) - } - - if g.Description != "" { - args = append(args, fmt.Sprintf("--description %s", g.Description)) - } - - cmd := fmt.Sprintf("%s %s", baseCmd, strings.Join(args, " ")) - return cmd, nil -} diff --git a/gangplank/internal/spec/jobspec.go b/gangplank/internal/spec/jobspec.go deleted file mode 100644 index 453a9eb0..00000000 --- a/gangplank/internal/spec/jobspec.go +++ /dev/null @@ -1,327 +0,0 @@ -/* - The RHCOS JobSpec is a YAML file describing the various Jenkins Job - knobs for controlling Pipeline execution. The JobSpec pre-dates this - code, and has been in production since 2019. - - The JobSpec has considerably more options than reflected in this file. - - Only include options that are believed to be relavent to COSA -*/ - -package spec - -import ( - "bufio" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "reflect" - "strings" - - "gopkg.in/yaml.v2" -) - -// JobSpec is the root-level item for the JobSpec. -type JobSpec struct { - Archives Archives `yaml:"archives,omitempty" json:"archives,omitempty"` - CloudsCfgs CloudsCfgs `yaml:"clouds_cfgs,omitempty" json:"cloud_cofgs,omitempty"` - Job Job `yaml:"job,omitempty" json:"job,omitempty"` - Recipe Recipe `yaml:"recipe,omitempty" json:"recipe,omitempty"` - Spec Spec `yaml:"spec,omitempty" json:"spec,omitempty"` - - // Minio describes the configuration for corrdinating objects for builds - Minio Minio `yaml:"minio,omitempty" json:"minio,omitempty"` - - // PublishOscontainer is a list of push locations for the oscontainer - PublishOscontainer PublishOscontainer `yaml:"publish_oscontainer,omitempty" json:"publish_oscontainer,omitempty"` - - // Stages are specific stages to be run. Stages are - // only supported by Gangplank; they do not appear in the - // Groovy Jenkins Scripts. - Stages []Stage `yaml:"stages" json:"stages"` - - // DelayedMetaMerge ensures that 'cosa build' is called with - // --delayed-meta-merge - DelayedMetaMerge bool `yaml:"delay_meta_merge" json:"delay_meta_meta,omitempty"` - - // CopyBuild defines an extra build to copy the build metadata for - CopyBuild string `yaml:"copy-build,omitempty" json:"copy-build",omitempty"` -} - -// Artifacts describe the expect build outputs. -// All: name of the all the artifacts -// Primary: Non-cloud builds -// Clouds: Cloud publication stages. -type Artifacts struct { - All []string `yaml:"all,omitempty" json:"all,omitempty"` - Primary []string `yaml:"primary,omitempty" json:"primary,omitempty"` - Clouds []string `yaml:"clouds,omitempty" json:"clouds,omitempty"` -} - -// Archives describes the location of artifacts to push to -// Brew is a nested Brew struct -// S3: publish to S3. -type Archives struct { - Brew *Brew `yaml:"brew,omitempty" json:"brew,omitempty"` - S3 *S3 `yaml:"s3,omitempty" json:"s3,omitempty"` -} - -// Brew is the RHEL Koji instance for storing artifacts. -// Principle: the Kerberos user -// Profile: the profile to use, i.e. brew-testing -// Tag: the Brew tag to tag the build as. -type Brew struct { - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - Principle string `yaml:"principle,omitempty" json:"principle,omitempty"` - Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` - Tag string `yaml:"tag,omitempty" json:"tag,omitempty"` -} - -// CloudsCfgs (yes Clouds) is a nested struct of all -// supported cloudClonfigurations. -type CloudsCfgs struct { - Aliyun *Aliyun `yaml:"aliyun,omitempty" json:"aliyun,omitempty"` - Aws *Aws `yaml:"aws,omitempty" json:"aws,omitempty"` - AwsCn *Aws `yaml:"aws-cn,omitempty" json:"aws-cn,omitempty"` - Azure *Azure `yaml:"azure,omitempty" json:"azure,omitempty"` - Gcp *Gcp `yaml:"gcp,omitempty" json:"gcp,omitempty"` -} - -// getCloudsCfgs returns list of clouds that are defined in the jobspec. Since -// omitempty is used when unmarshaling some objects will not be available -func (c *CloudsCfgs) GetCloudCfg(cloud string) (Cloud, error) { - t := reflect.TypeOf(*c) - v := reflect.ValueOf(*c) - for i := 0; i < t.NumField(); i++ { - fieldName := strings.ToLower(t.Field(i).Name) - if strings.ReplaceAll(cloud, "-", "") == fieldName { - if ci, ok := v.Field(i).Interface().(Cloud); ok { - return ci, nil - } - return nil, fmt.Errorf("failed casting struct to Cloud interface for %q cloud", cloud) - } - } - return nil, fmt.Errorf("Could not find cloud config %s", cloud) -} - -// Job refers to the Jenkins options -// BuildName: i.e. rhcos-4.7 -// IsProduction: enforce KOLA tests -// StrictMode: only run explicitly defined stages -// VersionSuffix: name to append, ie. devel -type Job struct { - BuildName string `yaml:"build_name,omitempty" json:"build_name,omitempty"` - IsProduction bool `yaml:"is_production,omitempty" json:"is_production,omitempty"` - StrictMode bool `yaml:"strict,omitempty" json:"strict,omitempty"` - VersionSuffix string `yaml:"version_suffix,omitempty" json:"version_suffix,omitempty"` - // ForceArch forces a specific architecutre. - ForceArch string `yaml:"force_arch,omitempty" json:"force_arch,omitempty"` - // Unexported minio valued (run-time options) - MinioCfgFile string // not exported -} - -type Minio struct { - // Bucket is the bucket to put all the bits - Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty"` - // MinioKeyPrefix is the root path in the bucket to start looking for paths. - // The prefix is treated as a path prefix - KeyPrefix string `yaml:"key_prefix,omitempty" json:"key_prefix,omitempty"` - // Unexported minio valued (run-time options) - ConfigFile string `yaml:",omitempty" json:",omitempty"` - SSHForward string `yaml:",omitempty" json:",omitempty"` - SSHUser string `yaml:",omitempty" json:",omitempty"` - SSHKey string `yaml:",omitempty" json:",omitempty"` - SSHPort int `yaml:",omitempty" json:",omitempty"` -} - -// Recipe describes where to get the build recipe/config, i.e fedora-coreos-config -// GitRef: branch/ref to fetch from -// GitUrl: url of the repo -// GitCommit: a specific commit in the branch to build from -type Recipe struct { - GitRef string `yaml:"git_ref,omitempty" json:"git_ref,omitempty"` - GitURL string `yaml:"git_url,omitempty" json:"git_url,omitempty"` - GitCommit string `yaml:"git_commit,omitempty" json:"git_commit,omitempty"` - Repos []*Repo `yaml:"repos,omitempty" json:"repos,omitempty"` -} - -// Repo is a yum/dnf repositories to use as an installation source. -type Repo struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - - // URL indicates that the repo file is remote - URL *string `yaml:"url,omitempty" json:"url,omitempty"` - - // Inline indicates that the repo file is inline - Inline *string `yaml:"inline,omitempty" json:"inline,omitempty"` -} - -// httpGetFunc describes a func that returns an http.Response and error -type httpGetFunc func(string) (*http.Response, error) - -// httpGet defaults to http.Get -var httpGet httpGetFunc = http.Get - -// Writer places the remote repo file into path. If the repo has no name, -// then a SHA256 of the URL will be used. Returns path of the file and err. -func (r *Repo) Writer(path string) (string, error) { - if r.URL == nil && r.Inline == nil { - return "", errors.New("repo must be a URL or inline data") - } - rname := r.Name - var data string - if r.URL != nil { - data = *r.URL - } else { - data = *r.Inline - } - if rname == "" { - h := sha256.New() - if _, err := h.Write([]byte(data)); err != nil { - return "", fmt.Errorf("failed to calculate name: %v", err) - } - rname = fmt.Sprintf("%x", h.Sum(nil)) - } - - f := filepath.Join(path, fmt.Sprintf("%s.repo", rname)) - out, err := os.OpenFile(f, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) - if err != nil { - return f, fmt.Errorf("failed to open %s for writing: %v", f, err) - } - defer out.Close() - - closer := func() error { return nil } - var dataR io.Reader - if r.URL != nil && *r.URL != "" { - resp, err := httpGet(*r.URL) - if err != nil { - return f, err - } - - switch code := resp.StatusCode; { - case code == 204: - return f, fmt.Errorf("http response code 204: repo content is empty") - case code == 206: - return f, errors.New("http response code 206: repo content was truncated") - case code > 400: - return f, fmt.Errorf("server responded with %d", code) - } - - dataR = resp.Body - closer = resp.Body.Close - } else { - dataR = strings.NewReader(*r.Inline) - } - - defer closer() //nolint - - n, err := io.Copy(out, dataR) - if n == 0 { - return f, errors.New("No remote content fetched") - } - return f, err -} - -// S3 describes the location of the S3 Resource. -// Acl: is the s3 acl to use, usually 'private' or 'public' -// Bucket: name of the S3 bucket -// Path: the path inside the bucket -type S3 struct { - ACL string `yaml:"acl,omitempty" envVar:"S3_ACL" json:"acl,omitempty"` - Bucket string `yaml:"bucket,omitempty" envVar:"S3_BUCKET" json:"bucket,omitempty"` - Path string `yaml:"path,omitempty" envVar:"S3_PATH" json:"path,omitempty"` -} - -// Spec describes the RHCOS JobSpec. -// GitRef: branch/ref to fetch from -// GitUrl: url of the repo -type Spec struct { - GitRef string `yaml:"git_ref,omitempty" json:"git_ref,omitempty"` - GitURL string `yaml:"git_url,omitempty" json:"git_url,omitempty"` -} - -// PublishOscontainer describes where to push the OSContainer to. -type PublishOscontainer struct { - // BuildStrategyTLSVerify indicates whether to verify TLS certificates when pushing as part of a OCP Build Strategy. - // By default, TLS verification is turned on. - BuildStrategyTLSVerify *bool `yaml:"buildstrategy_tls_verify" json:"buildstrategy_tls_verify"` - - // Registries is a list of locations to push to. - Registries []Registry `yaml:"registries" json:"regristries"` -} - -// Registry describes the push locations. -type Registry struct { - // URL is the location that should be used to push the secret. - URL string `yaml:"url" json:"url"` - - // TLSVerify tells when to verify TLS. By default, its true - TLSVerify *bool `yaml:"tls_verify,omitempty" json:"tls_verify,omitempty"` - - // SecretType is name the secret to expect, should PushSecretType*s - SecretType PushSecretType `yaml:"secret_type,omitempty" json:"secret_type,omitempty"` - - // If the secret is inline, the string data, else, the cluster secret name - Secret string `yaml:"secret,omitempty" json:"secret,omitempty"` -} - -// PushSecretType describes the type of push secret. -type PushSecretType string - -// Supported push secret types. -const ( - // PushSecretTypeInline means that the secret string is a string literal - // of the docker auth.json. - PushSecretTypeInline = "inline" - // PushSecretTypeCluster indicates that the named secret in PushRegistry should be - // fetched via the service account from the cluster. - PushSecretTypeCluster = "cluster" - // PushSecretTypeToken indicates that the service account associated with the token - // has access to the push repository. - PushSecretTypeToken = "token" -) - -// JobSpecReader takes and io.Reader and returns a ptr to the JobSpec and err -func JobSpecReader(in io.Reader) (j JobSpec, err error) { - d, err := ioutil.ReadAll(in) - if err != nil { - return j, err - } - - err = yaml.Unmarshal(d, &j) - if err != nil { - return j, err - } - return j, err -} - -// JobSpecFromFile return a JobSpec read from a file -func JobSpecFromFile(f string) (j JobSpec, err error) { - in, err := os.Open(f) - if err != nil { - return j, err - } - defer in.Close() - b := bufio.NewReader(in) - return JobSpecReader(b) -} - -// WriteJSON returns the jobspec -func (js *JobSpec) WriteJSON(w io.Writer) error { - encode := json.NewEncoder(w) - encode.SetIndent("", " ") - return encode.Encode(js) -} - -// WriteYAML returns the jobspec in YAML -func (js *JobSpec) WriteYAML(w io.Writer) error { - encode := yaml.NewEncoder(w) - defer encode.Close() - return encode.Encode(js) -} diff --git a/gangplank/internal/spec/jobspec_test.go b/gangplank/internal/spec/jobspec_test.go deleted file mode 100644 index ef6e8a52..00000000 --- a/gangplank/internal/spec/jobspec_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package spec - -import ( - "bytes" - "crypto/sha256" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "testing" -) - -// setMockHttpGet sets the httpGet func to a single-use mocking func for returing -// an HTTP tests. -func setMockHttpGet(data []byte, status int, err error) { - httpGet = func(string) (*http.Response, error) { - defer func() { - httpGet = http.Get - }() - return &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader(data)), - StatusCode: status, - }, err - } -} - -func TestURL(t *testing.T) { - tmpd, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("unable to create tmpdir") - } - defer os.RemoveAll(tmpd) //nolint - - cases := []struct { - repo Repo - desc string - data []byte - statusCode int - wantErr bool - }{ - { - desc: "good repo", - data: []byte("good repo"), - repo: Repo{URL: strPtr("http://mirrors.kernel.org/fedora-buffet/archive/fedora/linux/releases/30/Everything/source/tree/media.repo")}, - statusCode: 200, - wantErr: false, - }, - { - desc: "named repo", - data: []byte("named repo"), - statusCode: 200, - repo: Repo{ - Name: "test", - URL: strPtr("http://mirrors.kernel.org/fedora-buffet/archive/fedora/linux/releases/30/Everything/source/tree/media.repo")}, - wantErr: false, - }, - { - desc: "bad repo", - data: nil, - statusCode: 404, - repo: Repo{URL: strPtr("http://mirrors.kernel.org/this/will/not/exist/no/really/it/shouldnt")}, - wantErr: true, - }, - { - desc: "inline repo", - repo: Repo{Inline: strPtr("meh this is a repo")}, - wantErr: false, - }, - { - desc: "named inline repo", - repo: Repo{ - Name: "named inline repo", - Inline: strPtr("meh this is a repo"), - }, - wantErr: false, - }, - } - - for idx, c := range cases { - t.Run(fmt.Sprintf("%s case %d", t.Name(), idx), func(t *testing.T) { - - setMockHttpGet(c.data, c.statusCode, nil) - - path, err := c.repo.Writer(tmpd) - if c.wantErr && err == nil { - t.Fatalf("%s: wanted error, got none", c.desc) - } - - wantPath := filepath.Join(tmpd, fmt.Sprintf("%s.repo", c.repo.Name)) - if c.repo.Name == "" { - h := sha256.New() - var data []byte - if c.repo.URL != nil { - data = []byte(*c.repo.URL) - } else { - data = []byte(*c.repo.Inline) - } - _, _ = h.Write(data) - wantPath = filepath.Join(tmpd, fmt.Sprintf("%x.repo", h.Sum(nil))) - } - if wantPath != path { - t.Fatalf("%s path test:\n wanted: %s\n got: %s", c.desc, wantPath, path) - } - - fi, err := os.Stat(path) - if err != nil { - t.Fatalf("%s: expected repo %s to be written", c.desc, path) - } - - if fi.Size() == 0 && !c.wantErr { - t.Fatalf("%s is not expected to be zero size", path) - } - }) - } -} diff --git a/gangplank/internal/spec/kola.go b/gangplank/internal/spec/kola.go deleted file mode 100644 index 75a6f34a..00000000 --- a/gangplank/internal/spec/kola.go +++ /dev/null @@ -1,73 +0,0 @@ -package spec - -import ( - "fmt" - "strings" - - "github.com/spf13/pflag" -) - -type kolaTests map[string]Stage - -// kolaTestDefinitions contain a map of the kola tests. -var kolaTestDefinitions = kolaTests{ - "basicBios": { - ID: "Kola Basic BIOS Test", - PostCommands: []string{"cosa kola run --qemu-nvme=true basic"}, - RequireArtifacts: []string{"qemu"}, - ExecutionOrder: 2, - }, - "basicQemu": { - ID: "Kola Basic Qemu", - PostCommands: []string{"cosa kola --basic-qemu-scenarios"}, - RequireArtifacts: []string{"qemu"}, - ExecutionOrder: 2, - }, - "basicUEFI": { - ID: "Basic UEFI Test", - PostCommands: []string{"cosa kola run --qemu-firmware=uefi basic"}, - RequireArtifacts: []string{"qemu"}, - ExecutionOrder: 2, - }, - "external": { - ID: "Enternal Kola Test", - PostCommands: []string{"cosa kola run 'ext.*'"}, - RequireArtifacts: []string{"qemu"}, - ExecutionOrder: 2, - }, - "long": { - ID: "Kola Long Tests", - PostCommands: []string{"cosa kola run --parallel 3"}, - ExecutionOrder: 2, - }, - "upgrade": { - ID: "Kola Upgrade Test", - PostCommands: []string{"kola run-upgrade --output-dir tmp/kola-upgrade"}, - ExecutionOrder: 2, - }, - - // Metal and live-ISO tests - "iso": { - ID: "Kola ISO Testing", - PostCommands: []string{"kola testiso -S"}, - ExecutionOrder: 4, - RequireArtifacts: []string{"live-iso"}, - }, - "metal4k": { - ID: "Kola ISO Testing 4K Disks", - PostCommands: []string{"kola testiso -S --qemu-native-4k --scenarios iso-install --output-dir tmp/kola-metal4k"}, - ExecutionOrder: 4, - RequireArtifacts: []string{"live-iso"}, - }, -} - -// AddKolaTestFlags adds a StringVar flag for populating supported supported test into a -// string slice. -func AddKolaTestFlags(targetVar *[]string, fs *pflag.FlagSet) { - tests := []string{} - for k := range kolaTestDefinitions { - tests = append(tests, k) - } - fs.StringSliceVar(targetVar, "kolaTest", - []string{}, fmt.Sprintf("Kola Tests to run [%s]", strings.Join(tests, ","))) -} diff --git a/gangplank/internal/spec/override.go b/gangplank/internal/spec/override.go deleted file mode 100644 index 4a699084..00000000 --- a/gangplank/internal/spec/override.go +++ /dev/null @@ -1,144 +0,0 @@ -package spec - -import ( - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - - log "github.com/sirupsen/logrus" -) - -// Override describes RPMs or Tarballs to include as an override in the OSTree compose. -type Override struct { - // URI is a string prefixed with "file://" or "http(s)://" and a path. - URI string `yaml:"uri,omitempty" json:"uri,omitempty"` - - // Rpm indicates that the file is RPM and should be placed in overrides/rpm - Rpm *bool `yaml:"rpm,omitempty" json:"rpm,omitempty"` - - // Tarball indicates that the file is a tarball and will be extracted to overrides. - Tarball *bool `yaml:"tarball,omitempty" json:"tarball,omitempty"` - - // Tarball type is an override Tarball type - TarballType *string `yaml:"tarball_type,omitempty" json:"tarball_type,omitempty"` -} - -const ( - TarballTypeAll = "all" - TarballTypeRpms = "rpms" - TarballTypeRpm = "rpm" - TarballTypeRootfs = "rootfs" - overrideBasePath = "overrides" -) - -// writePath gets the path that the file should be extract to -func (o *Override) writePath(basePath string) (string, error) { - obase := filepath.Join(basePath, overrideBasePath) - - if o.Rpm != nil && *o.Rpm { - return filepath.Join(obase, "rpm", filepath.Base(o.URI)), nil - } - - if o.Tarball == nil { - return "", fmt.Errorf("override must be either tarball or RPM") - } - - // assume that the tarball type is all - if o.TarballType == nil { - return obase, nil - } - - switch *o.TarballType { - case TarballTypeAll: - return obase, nil - case TarballTypeRpms, TarballTypeRpm: - return filepath.Join(obase, TarballTypeRpm), nil - case TarballTypeRootfs: - return filepath.Join(obase, TarballTypeRootfs), nil - default: - return "", fmt.Errorf("tarball type %s is unknown", *o.TarballType) - } -} - -// TarDecompressorFunc is a function that handles decompressing a file. -type TarDecompressorFunc func(io.ReadCloser, string) error - -// Fetch reads the source and writes it to disk. The decompressor function -// is likely lazy, but allows for testing. -func (o *Override) Fetch(l *log.Entry, path string, wf TarDecompressorFunc) error { - nl := l.WithFields(log.Fields{ - "uri": o.URI, - "path": path, - }) - - if o.URI == "" { - return errors.New("uri is undefined for override") - } - - parts := strings.Split(o.URI, "://") - if len(parts) == 1 { - return fmt.Errorf("path lack URI identifer: %s", o.URI) - } - - writePath, err := o.writePath(path) - if err != nil { - return err - } - - basePath := writePath - if o.Rpm != nil && *o.Rpm { - basePath = filepath.Dir(basePath) - } - - nl = nl.WithField("target path", writePath) - - nl.Warn("creating target dir") - if err := os.MkdirAll(basePath, 0755); err != nil { - nl.WithError(err).Error("failed to create target path") - return fmt.Errorf("unable to create target dir %s: %v", basePath, err) - } - - var in io.ReadCloser - switch parts[0] { - case "file": - f, err := os.Open(parts[1]) - if err != nil { - l.WithError(err).Error("failed to open file") - return err - } - in = f - case "https", "http": - resp, err := http.Get(o.URI) - if err != nil { - l.WithError(err).Error("failed to open remote address") - return err - } - if resp.StatusCode < 200 || resp.StatusCode > 305 { - return fmt.Errorf("unable to fetch resource status code %d: %s", resp.StatusCode, resp.Status) - } - in = resp.Body - } - defer in.Close() - - // If this is a tarball, run the decompressor func and bail. - if o.Tarball != nil && *o.Tarball { - nl.Info("extracting uri to path") - return wf(in, writePath) - } - - // Otherwise this an RPM -- treat it like a generic file - f, err := os.Create(writePath) - if err != nil { - return err - } - defer f.Close() - - nl.Info("writing uri to path") - _, err = io.Copy(f, in) - nl.Info("success") - return err -} diff --git a/gangplank/internal/spec/override_test.go b/gangplank/internal/spec/override_test.go deleted file mode 100644 index 8c98b8cb..00000000 --- a/gangplank/internal/spec/override_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package spec - -import ( - "fmt" - "testing" -) - -func TestOverridePathing(t *testing.T) { - trueVar := true - - testCases := []struct { - o *Override - desc string - want string - wantErr bool - }{ - { - o: &Override{ - URI: "incorrect://whocares", - }, - desc: "invalid no type", - wantErr: true, - }, - { - o: &Override{ - URI: "file://someplace.rpm", - Rpm: &trueVar, - }, - want: "test/overrides/rpm/someplace.rpm", - desc: "file rpm", - wantErr: false, - }, - { - o: &Override{ - URI: "https://someplace.rpm", - Rpm: &trueVar, - }, - want: "test/overrides/rpm/someplace.rpm", - desc: "net-based rpm", - wantErr: false, - }, - { - o: &Override{ - URI: "http://tarball-of-doom.tar", - Tarball: &trueVar, - TarballType: strPtr("all"), - }, - want: "test/overrides", - desc: "tarball of all", - wantErr: false, - }, - { - o: &Override{ - URI: "http://cuddly-demogorgrans.tar", - Tarball: &trueVar, - TarballType: strPtr("rootfs"), - }, - want: "test/overrides/rootfs", - desc: "tarball of rootfs", - wantErr: false, - }, - { - o: &Override{ - URI: "http://tormented-rpms.tar", - Tarball: &trueVar, - TarballType: strPtr("rpms"), - }, - want: "test/overrides/rpm", - desc: "tarball of rpms", - wantErr: false, - }, - } - - for idx, tc := range testCases { - t.Run(fmt.Sprintf("%d-%s", idx, tc.desc), func(t *testing.T) { - got, err := tc.o.writePath("test") - if err != nil && !tc.wantErr { - t.Fatalf("%s errored unexpectedly", tc.desc) - } - if err == nil && tc.wantErr { - t.Fatalf("%s exepcted error", tc.desc) - } - if tc.want != got { - t.Errorf("%s failed:\n want: %s\n got: %s\n", tc.desc, tc.want, got) - } - }) - } -} diff --git a/gangplank/internal/spec/render_test.go b/gangplank/internal/spec/render_test.go deleted file mode 100644 index 5cf27561..00000000 --- a/gangplank/internal/spec/render_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package spec - -import ( - "context" - "io/ioutil" - "os" - "testing" - - "gopkg.in/yaml.v2" -) - -// MockOSJobSpec is an example jobSpec for testing -var MockOSJobSpec = ` -clouds_cfgs: - aws: - ami_path: mock-ami/testing/amis - public: false - regions: - - us-east-1 - azure: - enabled: true - resource_group: os4-common - secret_name: mockOS-azure - storage_account: mock - storage_container: imagebucket - storage_location: eastus2 - gcp: - enabled: true - bucket: mockOS-devel/devel - platform_id: gce - project: openshift-mockOS-devel - secret_name: mockOS-gce-service-account - secret_payload: gce.json - aliyun: - enabled: false - bucket: mockOS-images - regions: - - us-west-1 -job: - build_name: "mockOS-99" - force_version: "99.99.999999999999" - version_suffix: "magical-unicorns" -recipe: - git_ref: "mock" - git_url: https://github.com/coreos/coreos-assembler -oscontainer: - push_url: registry.mock.example.org/mockOS-devel/machine-os-content -` - -func wantedGot(want, got interface{}, t *testing.T) { - if want != got { - t.Errorf("wanted: %v\n got: %v", want, got) - } -} - -func TestJobSpec(t *testing.T) { - rd := new(RenderData) - - if err := yaml.Unmarshal([]byte(MockOSJobSpec), &rd.JobSpec); err != nil { - t.Errorf("failed to read mock jobspec") - } - - wantedGot("mockOS-99", rd.JobSpec.Job.BuildName, t) - - // Test rendering from a string - s, err := rd.ExecuteTemplateFromString("good {{ .JobSpec.Job.BuildName }}") - wantedGot(nil, err, t) - wantedGot("good mockOS-99", s[0], t) - wantedGot(1, len(s), t) - - // Test rendering for a slice of strings - s, err = rd.ExecuteTemplateFromString("good", "{{ .JobSpec.Job.BuildName }}") - wantedGot(nil, err, t) - wantedGot("mockOS-99", s[1], t) - wantedGot(2, len(s), t) - - // Test a failure - _, err = rd.ExecuteTemplateFromString("this", "wont", "{{ .Work }}") - if err == nil { - t.Errorf("template should not render") - } - - // Test a script - f, err := ioutil.TempFile("", "meh") - defer os.Remove(f.Name()) - wantedGot(nil, err, t) - err = ioutil.WriteFile(f.Name(), []byte("echo {{ .JobSpec.Job.BuildName }}"), 0444) - wantedGot(nil, err, t) - - ctx := context.Background() - err = rd.RendererExecuter(ctx, []string{}, f.Name()) - wantedGot(nil, err, t) - -} diff --git a/gangplank/internal/spec/stage_test.go b/gangplank/internal/spec/stage_test.go deleted file mode 100644 index 4f47cbeb..00000000 --- a/gangplank/internal/spec/stage_test.go +++ /dev/null @@ -1,415 +0,0 @@ -package spec - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/coreos/coreos-assembler-schema/cosa" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" -) - -// MockStageYaml is used to test inline specification -var MockStageYaml = fmt.Sprintf(` -%s -stages: - - description: Test Stage - commands: - - echo {{ .JobSpec.Recipe.GitRef }} - - echo {{ .JobSpec.Job.BuildName }} - - description: Concurrent Stage Test - concurrent: true - prep_commands: - - touch prep - commands: - - touch cmds - - | - bash -c 'echo this is multiline; - echo test using inline yaml' - post_commands: - - test -f prep - - test -f cmds -`, MockOSJobSpec) - -func TestStages(t *testing.T) { - tmpd, _ := ioutil.TempDir("", "teststages") - defer os.RemoveAll(tmpd) - - rd := &RenderData{ - JobSpec: new(JobSpec), - Meta: new(cosa.Build), - } - rd.Meta.BuildID = "MockBuild" - rd.Meta.Architecture = "ARMv6" - - checkFunc := func() error { return nil } - tCases := []struct { - desc string - wantErr bool - stages []Stage - checkFunc func() error - }{ - { - desc: "Test Single Stage", - checkFunc: checkFunc, - wantErr: false, - stages: []Stage{ - { - Description: "Single should pass", - Commands: []string{"echo hello"}, - }, - }, - }, - { - desc: "Test Dual Stage", - checkFunc: checkFunc, - wantErr: false, - stages: []Stage{ - { - Description: "Dual single command should pass", - Commands: []string{"echo hello"}, - }, - { - Description: "Dual concurrent should pass", - Commands: []string{ - "echo {{ .JobSpec.Job.BuildName }}", - "echo {{ .JobSpec.Recipe.GitRef }}", - }, - ConcurrentExecution: true, - }, - }, - }, - { - desc: "Test Bad Template", - checkFunc: checkFunc, - wantErr: true, - stages: []Stage{ - { - Description: "Bad Template should fail", - Commands: []string{"echo {{ .This.Wont.Work }}"}, - }, - }, - }, - { - desc: "Test Bad Concurrent Template", - wantErr: true, - stages: []Stage{ - { - Description: "One command should fail", - ConcurrentExecution: true, - Commands: []string{ - "/bin/false", - "/bin/true", - "bob", - fmt.Sprintf("/bin/sleep 3; touch %s/check", tmpd), - }, - }, - }, - checkFunc: func() error { - if _, err := os.Open(filepath.Join(tmpd, "check")); err != nil { - return fmt.Errorf("check file is missing: %w", err) - } - return nil - }, - }, - { - desc: "Test Prep and Post", - wantErr: false, - stages: []Stage{ - { - Description: "Check command ordering", - ConcurrentExecution: true, - PrepCommands: []string{ - fmt.Sprintf("touch %s/prep", tmpd), - }, - Commands: []string{ - fmt.Sprintf("test -f %s/prep", tmpd), - fmt.Sprintf("touch %s/commands", tmpd), - }, - PostCommands: []string{ - fmt.Sprintf("test -f %s/commands", tmpd), - fmt.Sprintf("touch %s/post", tmpd), - }, - }, - }, - checkFunc: func() error { - for _, c := range []string{"prep", "commands", "post"} { - if _, err := os.Stat(filepath.Join(tmpd, c)); err != nil { - return fmt.Errorf("check file is missing: %w", err) - } - } - return nil - }, - }, - { - desc: "Test Templating", - wantErr: false, - stages: []Stage{ - { - Description: "Templating check", - Commands: []string{ - fmt.Sprintf("touch %s/{{.Meta.BuildID}}.{{.Meta.Architecture}}", tmpd), - }, - }, - }, - checkFunc: func() error { - _, err := os.Stat(filepath.Join(tmpd, fmt.Sprintf("%s.%s", rd.Meta.BuildID, rd.Meta.Architecture))) - return err - }, - }, - } - - testEnv := []string{ - "MOCK_ENV=1", - "TEST_VAR=2", - } - - js := JobSpec{} - if err := yaml.Unmarshal([]byte(MockOSJobSpec), &js); err != nil { - t.Errorf("failed to read mock jobspec") - } - ctx := context.Background() - - for _, c := range tCases { - t.Logf(" * %s ", c.desc) - for _, stage := range c.stages { - t.Logf(" - test name: %s", stage.Description) - err := stage.Execute(ctx, rd, testEnv) - if c.wantErr && err == nil { - t.Error(" SHOULD error, but did not") - } - if err != nil && !c.wantErr { - t.Errorf(" SHOULD NOT error, but did: %v", err) - } - if err = c.checkFunc(); err != nil { - t.Errorf(" %v", err) - } - } - } -} - -func TestStageYaml(t *testing.T) { - myD, _ := os.Getwd() - defer os.Chdir(myD) //nolint - tmpd, _ := ioutil.TempDir("", "stagetest") - _ = os.Chdir(tmpd) - defer os.RemoveAll(tmpd) - - r := strings.NewReader(MockStageYaml) - js, err := JobSpecReader(r) - if err != nil { - t.Fatalf("failed to get jobspec: %v", err) - } - - rd := &RenderData{ - JobSpec: &js, - Meta: nil, - } - - c, cancel := context.WithCancel(context.Background()) - defer c.Done() - defer cancel() - - for _, stage := range js.Stages { - t.Logf("* executing stage: %s", stage.Description) - if err := stage.Execute(c, rd, []string{}); err != nil { - t.Errorf("failed inline stage execution: %v", err) - } - } -} - -func TestIsArtifactValid(t *testing.T) { - testCases := []struct { - artifact string - want bool - }{ - { - artifact: "base", - want: true, - }, - { - artifact: "AWS", - want: true, - }, - { - artifact: "finalize", - want: true, - }, - { - artifact: "mandrake+root", - want: false, - }, - } - for idx, tc := range testCases { - t.Run(fmt.Sprintf("test-%d-%s", idx, tc.artifact), func(t *testing.T) { - got := isValidArtifactShortHand(tc.artifact) - if got != tc.want { - t.Errorf("artifact %s should return %v but got %v", tc.artifact, tc.want, got) - } - }) - } -} - -func TestBuildCommandOrders(t *testing.T) { - type testCase struct { - desc string - shorthands []string - stage *Stage - want *Stage - testFn func(t *testing.T) - } - - testCases := []testCase{} - testCases = append(testCases, - func() testCase { - // Test that base returns base - tc := testCase{ - desc: "base shorthand is understood", - shorthands: []string{"base"}, - stage: &Stage{ - BuildArtifacts: []string{"base"}, - }, - want: &Stage{ - BuildArtifacts: []string{"base"}, - }, - } - tc.testFn = func(t *testing.T) { - addAllShorthandsToStage(tc.stage, tc.shorthands...) - assert.Len(t, tc.stage.BuildArtifacts, 1) - for _, v := range tc.want.BuildArtifacts { - assert.Contains(t, tc.stage.BuildArtifacts, v) - } - } - return tc - }(), - - // Ensure that base implies qemu for aws - func() testCase { - tc := testCase{ - desc: "base should build before aws", - shorthands: []string{"aws"}, - stage: &Stage{ - BuildArtifacts: []string{"base"}, - }, - want: &Stage{ - BuildArtifacts: []string{"base", "aws"}, - }, - } - - tc.testFn = func(t *testing.T) { - addAllShorthandsToStage(tc.stage, tc.shorthands...) - assert.Equal(t, tc.want.BuildArtifacts, tc.stage.BuildArtifacts) - for idx, v := range tc.stage.BuildArtifacts { - if idx == 0 { - assert.Equal(t, "base", v, "base should be ordered before aws") - } - if idx == 1 { - assert.Equal(t, "aws", v, "aws should be ordered after base") - } - } - } - return tc - }(), - - // Base implies ostree and qemu - func() testCase { - tc := testCase{ - desc: "base implies ostree and qemu", - shorthands: []string{"ostree", "qemu"}, - stage: &Stage{ - BuildArtifacts: []string{"base"}, - }, - want: &Stage{ - BuildArtifacts: []string{"base"}, - }, - } - tc.testFn = func(t *testing.T) { - addAllShorthandsToStage(tc.stage, tc.shorthands...) - assert.Equal(t, tc.want.BuildArtifacts, tc.stage.BuildArtifacts) - } - return tc - }(), - /* - func() testCase { - tc := testCase{} - tc.testFn = func(t *testing.T) { - addAllShorthandsToStage(tc.stage, tc.shorthands...) - } - return tc - }(), - */ - ) - - for idx, tc := range testCases { - t.Run(fmt.Sprintf("test-%d-%v", idx, tc.desc), tc.testFn) - } -} - -func TestBuildCommands(t *testing.T) { - testCases := []struct { - desc string - artifact string - want []string - js *JobSpec - }{ - // Ensure that `cosa build X` commands are rendered properly. - { - desc: "base command should be 'cosa build'", - artifact: "base", - want: []string{fmt.Sprintf(defaultBaseCommand, "")}, - js: &JobSpec{DelayedMetaMerge: false}, - }, - { - desc: "ostree command should be 'cosa build ostree'", - artifact: "ostree", - want: []string{fmt.Sprintf(defaultBaseCommand, "ostree")}, - js: &JobSpec{DelayedMetaMerge: false}, - }, - { - desc: "qemu command should be 'cosa build qemu'", - artifact: "qemu", - want: []string{fmt.Sprintf(defaultBaseCommand, "qemu")}, - js: &JobSpec{DelayedMetaMerge: false}, - }, - - // Ensure that `cosa build --delay-merge X` commands are rendered properly. - { - desc: "base command should be 'cosa build --delay-merge'", - artifact: "base", - want: []string{fmt.Sprintf(defaultBaseDelayMergeCommand, "")}, - js: &JobSpec{DelayedMetaMerge: true}, - }, - { - desc: "ostree command should be 'cosa build --delay-merge ostree'", - artifact: "ostree", - want: []string{fmt.Sprintf(defaultBaseDelayMergeCommand, "ostree")}, - js: &JobSpec{DelayedMetaMerge: true}, - }, - { - desc: "qemu command should be 'cosa build --delay-merge qemu'", - artifact: "qemu", - want: []string{fmt.Sprintf(defaultBaseDelayMergeCommand, "qemu")}, - js: &JobSpec{DelayedMetaMerge: true}, - }, - - // Check finalize - { - desc: "finalize command should match", - artifact: "finalize", - want: []string{defaultFinalizeCommand}, - js: &JobSpec{DelayedMetaMerge: true}, - }, - } - - for idx, tc := range testCases { - t.Run(fmt.Sprintf("test-%d-%s", idx, tc.desc), func(t *testing.T) { - cmd, _ := cosaBuildCmd(tc.artifact, tc.js) - assert.EqualValues(t, tc.want, cmd) - }) - } -} diff --git a/gangplank/internal/spec/stages.go b/gangplank/internal/spec/stages.go deleted file mode 100644 index f7459247..00000000 --- a/gangplank/internal/spec/stages.go +++ /dev/null @@ -1,649 +0,0 @@ -package spec - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/coreos/coreos-assembler-schema/cosa" - - log "github.com/sirupsen/logrus" -) - -// GetStage returns the stage with the matching ID -func (j *JobSpec) GetStage(id string) (*Stage, error) { - for _, stage := range j.Stages { - if stage.ID == id { - return &stage, nil - } - } - return nil, fmt.Errorf("no such stage with ID %q", id) -} - -// Stage is a single stage. -type Stage struct { - ID string `yaml:"id,omitempty" json:"id,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` - ConcurrentExecution bool `yaml:"concurrent,omitempty" json:"concurrent,omitempty"` - - // DirectExec signals that the command should not be written - // to a file. Rather the command should directly executed. - DirectExec bool `yaml:"direct_exec,omitempty" json:"direct_exec,omitempty"` - - // NotBlocking means that the stage does not block another stage - // from starting execution (i.e. concurrent stage). - NotBlocking bool `yaml:"not_blocking,omitempty" json:"not_blocking,omitempty"` - - // RequireArtifacts is a name of the required artifacts. If the - // required artifact is missing (per the meta.json), the stage - // will not be executed. RequireArticts _implies_ sending builds/builds.json - // and builds//meta.json. - RequireArtifacts []string `yaml:"require_artifacts,flow,omitempty" json:"require_artifacts,omitempty"` - - // RequestArtifacts are files that are provided if they are there. Examples include - // 'caches' for `/srv/cache` and `/srv/tmp/repo` tarballs or `ostree` which are really useful - // for base builds. - RequestArtifacts []string `yaml:"request_artifacts,flow,omitempty" json:"request_artifacts,omitempty"` - - // BuildArtifacts produces "known" artifacts. The special "base" - // will produce an OSTree and QCOWs. - BuildArtifacts []string `yaml:"build_artifacts,flow,omitempty" json:"build_artifacts,omitempty"` - - // Commands are arbitrary commands run after an Artifact builds. - // Instead of running `cosa buildextend-?` as a command, its preferrable - // use the bare name in BuildArtifact. - Commands []string `yaml:"commands,flow,omitempty" json:"commands,omitempty"` - - // PublishArtifacts will upload defined BuildArtifacts to the cloud providers - PublishArtifacts []string `yaml:"publish_artifacts,omitempty" json:"publish_artifacts,omitempty"` - - // PrepCommands are run before Artifact builds, while - // PostCommands are run after. Prep and Post Commands are run serially. - PrepCommands []string `yaml:"prep_commands,flow,omitempty" json:"prep_commands,omitempty"` - PostCommands []string `yaml:"post_commands,flow,omitempty" json:"post_commands,omitempty"` - - // PostAlways ensures that the PostCommands are always run. - PostAlways bool `yaml:"post_always,omitempty" json:"post_always,omitempty"` - - // ExecutionOrder is a number value that defines the order of stages. If two stages - // share the same execution order number, then they are allowed to run concurrently to each other. - ExecutionOrder int `yaml:"execution_order,omitempty" json:"execution_order,omitempty"` - - // ReturnCache returns a tarball of `/srv/cache`, while RequireCahce ensures the tarball - // is fetched unpacked into `/srv/cahce`. RequestCache is a non-blocking, optional versopn - // of RequireCache. - ReturnCache bool `yaml:"return_cache,omitempty" json:"return_cache,omitempty"` - RequireCache bool `yaml:"require_cache,omitempty" json:"require_cache_repo,omitempty"` - RequestCache bool `yaml:"request_cache,omitempty" json:"reqest_cache_repo,omitempty"` - - // ReturnCacheRepo returns a tarball of `/srv/repo`, while RequireCacheRepo ensures the - // tarball is fetched and unpacked into `/srv/repo`. RequestCacheRepo is a non-blocking, optional - // version of RequireCacheRepo - ReturnCacheRepo bool `yaml:"return_cache_repo,omitempty" json:"return_cache_repo,omitempty"` - RequireCacheRepo bool `yaml:"require_cache_repo,omitempty" json:"require_cache_repo_repo,omitempty"` - RequestCacheRepo bool `yaml:"request_cache_repo,omitempty" json:"request_cache_repo_repo,omitempty"` - - // ReturnFiles returns a list of files that were requested to be returned. - ReturnFiles []string `yaml:"return_files,omitempty" json:"return_files,omitempty"` - - // KolaTests are shorthands for testing. - KolaTests []string `yaml:"kola_tests,omitempty" json:"kola_tests,omitempty"` - - // Overrides is a list of Overrides to apply to the OS tree - Overrides []Override `yaml:"overrides,omitempty" json:"overrides,omitempty"` -} - -// These are the only hard-coded commands that Gangplank understand. -const ( - // defaultBaseCommand is the basic build command - defaultBaseCommand = "cosa fetch; cosa build %s;" - // defaultBaseDelayMergeCommand is used for distributed build using - // parallel workers pods. - defaultBaseDelayMergeCommand = "cosa fetch; cosa build %s --delay-meta-merge;" - - // defaultFinalizeComamnd ensures that the meta.json is merged. - defaultFinalizeCommand = "cosa meta --finalize;" -) - -// cosaBuildCmds checks if b is a buildable artifact type and then -// returns it. -func cosaBuildCmd(b string, js *JobSpec) ([]string, error) { - log.WithField("command", b).Info("checking shorthand") - switch v := strings.ToLower(b); v { - case "base", "ostree", "qemu": - if v == "base" { - v = "" - } - if js.DelayedMetaMerge { - return []string{fmt.Sprintf(defaultBaseDelayMergeCommand, v)}, nil - } - return []string{fmt.Sprintf(defaultBaseCommand, v)}, nil - case "finalize": - return []string{defaultFinalizeCommand}, nil - case "live": - return []string{fmt.Sprintf("cosa buildextend-%s", b)}, nil - } - - if cosa.CanArtifact(b) { - return []string{fmt.Sprintf("cosa buildextend-%s", b)}, nil - } - return nil, fmt.Errorf("%s is not a known buildable artifact", b) -} - -// getCommands renders the automatic artifacts and publication commands -func (s *Stage) getCommands(rd *RenderData) ([]string, error) { - if len(s.BuildArtifacts) > 0 { - log.WithField("mapping artifacts", s.BuildArtifacts).Infof("Mapping artifacts") - } - numBuildArtifacts := len(s.BuildArtifacts) - totalCmds := len(s.Commands) + numBuildArtifacts - - ret := make([]string, totalCmds) - for i, ba := range s.BuildArtifacts { - log.WithField("artifact", ba).Info("mapping artifact to command") - cmds, err := cosaBuildCmd(ba, rd.JobSpec) - if err != nil { - log.WithError(err).Errorf("failed to map build artifacts: %v", ba) - return nil, err - } - ret[i] = strings.Join(cmds, "\n") - } - for i, c := range s.Commands { - ret[(numBuildArtifacts + i)] = c - } - return ret, nil -} - -// getPostCommands generates the post commands from a synthatis of pre-defined -// post commands, kola tests and the cloud publication steps. -func (s *Stage) getPostCommands(rd *RenderData) ([]string, error) { - ret := s.PostCommands - - log.WithField("mapping tests", s.KolaTests).Infof("Resolving test definitions") - for _, kolaTest := range s.KolaTests { - tk, ok := kolaTestDefinitions[kolaTest] - if !ok { - return nil, fmt.Errorf("test %q is an unknown short hand", kolaTest) - } - ret = append(ret, tk.PostCommands...) - } - - pc, err := s.getPublishCommands(rd) - if err != nil { - return nil, err - } - - ret = append(ret, pc...) - return ret, nil -} - -// getPublishCommands returns the cloud publication commands. -func (s *Stage) getPublishCommands(rd *RenderData) ([]string, error) { - var publishCommands []string - c := rd.JobSpec.CloudsCfgs - for _, cloud := range s.PublishArtifacts { - if !cosa.CanArtifact(cloud) { - return nil, fmt.Errorf("Invalid cloud artifact: %v", cloud) - } - - config, err := c.GetCloudCfg(cloud) - if err != nil { - return nil, err - } - - pc, err := config.GetPublishCommand(rd.Meta.BuildID) - if err != nil { - return nil, err - } - publishCommands = append(publishCommands, pc) - } - - return publishCommands, nil -} - -// Execute runs the commands of a stage. -func (s *Stage) Execute(ctx context.Context, rd *RenderData, envVars []string) error { - if ctx == nil { - return errors.New("context must not be nil") - } - - if rd == nil { - return errors.New("render data must not be nil") - } - - log.Infof("Stage: %v", s) - - cmds, err := s.getCommands(rd) - if err != nil { - log.WithError(err).Error("failed to get stage commands") - return err - } - - postCommands, err := s.getPostCommands(rd) - if err != nil { - log.WithError(err).Error("failed to get post commands") - return err - } - - if len(s.PrepCommands) == 0 && len(cmds) == 0 && len(postCommands) == 0 { - return errors.New("no commands to execute") - } - log.WithField("cmd", cmds).Info("stage commands readied") - - tmpd, err := ioutil.TempDir("", "stages") - if err != nil { - return err - } - defer os.RemoveAll(tmpd) - - // Render the pre and post scripts. - prepScript := filepath.Join(tmpd, "prep.sh") - if err := ioutil.WriteFile(prepScript, []byte(strings.Join(s.PrepCommands, "\n")), 0755); err != nil { - return err - } - if err := rd.RendererExecuter(ctx, envVars, prepScript); err != nil { - return fmt.Errorf("Failed execution of the prep stage: %w", err) - } - - postScript := filepath.Join(tmpd, "post.sh") - if err := ioutil.WriteFile(postScript, []byte(strings.Join(postCommands, "\n")), 0755); err != nil { - return err - } - if s.PostAlways { - log.Info("PostCommand will be executed regardless of command success") - defer func() { - _ = rd.RendererExecuter(ctx, envVars, postScript) - }() - } - - // Write out each command to their own file. To enable concurrent execution. - scripts := make(map[int]string) - for i, c := range cmds { - outf := filepath.Join(tmpd, fmt.Sprintf("script-%d.sh", i)) - if err := ioutil.WriteFile(outf, []byte(c), 0755); err != nil { - return nil - } - scripts[i] = outf - log.Infof("%s: %s", outf, c) - } - - // Execute the main command stage. - if !s.ConcurrentExecution { - // Non-concurrent commands are run serially. Any failure will immediately - // break the run. - log.Infof("Executing %d stage commands serially", len(scripts)) - // Don't use `range scripts` here because the map is unordered - // and we want to execute the commands in order. We know the map - // was populated in order with index[i] so just use the length - // here and count from 0 to len(scripts). - for i := 0; i < len(scripts); i++ { - if err := rd.RendererExecuter(ctx, envVars, scripts[i]); err != nil { - return err - } - } - } else { - // Concurrent commands are run in parallel until all complete OR - // one fails. - log.Infof("Executing %d stage commands concurrently", len(scripts)) - wg := &sync.WaitGroup{} - errors := make(chan error, len(scripts)) - for _, s := range scripts { - wg.Add(1) - go func(s string, w *sync.WaitGroup, ctx context.Context) { - defer w.Done() - log.Infof("STARTING command: %s", s) - e := rd.RendererExecuter(ctx, envVars, s) - errors <- e - if err != nil { - log.Infof("ERROR %s", s) - return - } - log.Infof("SUCCESS %s", s) - }(s, wg, ctx) - // hack: ensure that scripts are started serially - // but may run concurrently - time.Sleep(50 * time.Millisecond) - } - - // Wait for the concurrent commands to run, and check - // all errors to make sure non are swallowed. - wg.Wait() - var e error = nil - for x := 0; x <= len(errors); x++ { - err, ok := <-errors - if !ok { - break - } - if err != nil { - log.Errorf("error recieved: %v", err) - e = err - } - } - if e != nil { - return e - } - } - - // If PostAlways, then the postScript is executed in defer call above. - if !s.PostAlways { - return rd.RendererExecuter(ctx, envVars, postScript) - } - - return nil -} - -var ( - // pseudoStages are special setup and tear down phases. - pseudoStages = []string{"base", "finalize", "live"} - // buildableArtifacts are known artifacts types from the schema. - buildableArtifacts = append(pseudoStages, cosa.GetCommandBuildableArtifacts()...) - - // baseArtifacts are default built by the "base" short-hand - baseArtifacts = []string{"ostree", "qemu"} -) - -// isBaseArtifact is a check function for determining if an artifact -// is built by the base stage. -func isBaseArtifact(artifact string) bool { - for _, k := range baseArtifacts { - if k == artifact { - return true - } - } - return false -} - -// GetArtifactShortHandNames returns shorthands for buildable stages -func GetArtifactShortHandNames() []string { - return buildableArtifacts -} - -// addShorthandToStage adds in a build shorthand into the stage and -// ensures that required dependencies are correclty ordered -// Ordering assumptions: -// 1. Base builds -// 2. Basic Kola Tests -// 3. Metal and Live ISO images -// 4. Metal and Live ISO testings -// 5. Cloud stages -func addShorthandToStage(artifact string, stage *Stage) { - - quickStage := func(noun string) *Stage { - switch noun { - case "base": - return &Stage{ - BuildArtifacts: []string{"base"}, - ExecutionOrder: 1, - RequestArtifacts: []string{"ostree"}, - RequestCache: true, - RequestCacheRepo: true, - } - case "extensions": - return &Stage{ - BuildArtifacts: []string{"extensions"}, - ExecutionOrder: 2, - RequireArtifacts: []string{"ostree"}, - RequireCache: true, - RequireCacheRepo: true, - } - case "finalize": - return &Stage{ - BuildArtifacts: []string{"finalize"}, - ExecutionOrder: 999, - } - case "live": - return &Stage{ - ExecutionOrder: 2, - BuildArtifacts: []string{"live"}, - RequireArtifacts: []string{"ostree", "metal", "metal4k"}, - } - case "metal": - return &Stage{ - ExecutionOrder: 3, - BuildArtifacts: []string{"metal"}, - RequireArtifacts: []string{"ostree"}, - } - case "metal4k": - return &Stage{ - ExecutionOrder: 3, - BuildArtifacts: []string{"metal4k"}, - RequireArtifacts: []string{"ostree"}, - } - case "oscontainer": - return &Stage{ - BuildArtifacts: []string{"oscontainer"}, - ExecutionOrder: 2, - RequireArtifacts: []string{"ostree"}, - RequireCache: true, - RequireCacheRepo: true, - } - default: - // check if the short hand is a test stage - testStage, ok := kolaTestDefinitions[noun] - if ok { - return &testStage - } - // otherwise its likely a cloud stage - if !cosa.CanArtifact(artifact) { - break - } - return &Stage{ - ExecutionOrder: 5, - BuildArtifacts: []string{artifact}, - RequireArtifacts: []string{"qemu"}, - } - } - log.WithField("artifact", noun).Fatalf("unknown artifact type") - return nil - } - - working := quickStage(artifact) - - // remove is helper for removing the first matching item from a slice - remove := func(slice []string, key string) ([]string, bool) { - for x := 0; x < len(slice); x++ { - if slice[x] == key { - return append(slice[:x], slice[x+1:]...), true - } - } - return slice, false - } - - unique := func(strSlice []string) []string { - keys := make(map[string]bool) - list := []string{} - for _, entry := range strSlice { - if _, value := keys[entry]; !value { - keys[entry] = true - list = append(list, entry) - } - } - return list - } - - // if the stage returns cache/repo cache then it provides the requires - if working.RequireCache && !stage.ReturnCache { - stage.RequireCache = true - stage.RequestCache = false - } - if working.RequireCacheRepo && !stage.ReturnCacheRepo { - stage.RequireCacheRepo = true - stage.RequestCacheRepo = false - } - - // Handle the return/requires for cache and repo cache - if working.ReturnCache { - stage.ReturnCache = working.ReturnCache - } - if working.ReturnCacheRepo { - stage.ReturnCacheRepo = working.ReturnCacheRepo - } - - // Only set RequestCache[Repo] we don't require them. - if working.RequestCache && (!stage.RequireCache || !working.RequireCache) { - stage.RequestCache = true - } - if working.RequestCacheRepo && (!stage.RequireCacheRepo || !working.RequireCacheRepo) { - stage.RequestCacheRepo = true - } - - // if the stage returns cache/repo cache then it provides the requires - if working.RequireCache && !stage.ReturnCache { - stage.RequireCache = true - } - if working.RequireCacheRepo && !stage.ReturnCacheRepo { - stage.RequireCacheRepo = true - } - - // Add the commands if defined - stage.Commands = append(stage.Commands, working.Commands...) - stage.PrepCommands = append(stage.PrepCommands, working.PrepCommands...) - stage.PostCommands = append(stage.PostCommands, working.PostCommands...) - - stage.RequestArtifacts = append(stage.RequestArtifacts, working.RequestArtifacts...) - stage.BuildArtifacts = append(stage.BuildArtifacts, working.BuildArtifacts...) - stage.RequireArtifacts = append(stage.RequireArtifacts, working.RequireArtifacts...) - - // Assume the lowest stage execution order - if working.ExecutionOrder < stage.ExecutionOrder || stage.ExecutionOrder == 0 { - stage.ExecutionOrder = working.ExecutionOrder - } - - randID := time.Now().UTC().UnixNano() // Ensure a random ID - stage.ID = fmt.Sprintf("ExecOrder %d Stage %d", stage.ExecutionOrder, randID) - stage.Description = fmt.Sprintf("Stage %d execution %s", - stage.ExecutionOrder, strings.Join(append(stage.BuildArtifacts, stage.KolaTests...), ",")) - - // Get the order that artifacts should be built - artifactOrder := make(map[int][]string) - for _, v := range stage.BuildArtifacts { - if v == "caches" { - stage.RequireCache = true - stage.RequireCacheRepo = true - } else { - fakeStage := quickStage(v) - artifactOrder[fakeStage.ExecutionOrder] = append(artifactOrder[fakeStage.ExecutionOrder], v) - } - } - - newOrder := []string{} - for _, v := range artifactOrder { - newOrder = append(newOrder, v...) - } - stage.BuildArtifacts = unique(newOrder) - - // Base implies building ostree and qemu - buildArtifacts, buildsBase := remove(unique(newOrder), "base") - if buildsBase { - buildArtifacts, _ = remove(buildArtifacts, "ostree") - buildArtifacts, _ = remove(buildArtifacts, "qemu") - stage.BuildArtifacts = append([]string{"base"}, buildArtifacts...) - } - - // If the synthetic stages requires/request optional artifact, but also builds it - // then we need to remove it from the the requires. - realRequires := stage.RequireArtifacts - realOptional := stage.RequestArtifacts - - for _, ba := range stage.BuildArtifacts { - for _, ra := range stage.RequireArtifacts { - if ra == ba { - realRequires, _ = remove(realRequires, ra) - } - } - for _, oa := range stage.RequestArtifacts { - if oa == ba { - realOptional, _ = remove(realOptional, oa) - } - } - } - - // base is short hand of ostree and qemu. Its handled specially - // since we have to consider that "qemu" - var foundBase bool - realRequires, foundBase = remove(realRequires, "base") - if foundBase || buildsBase { - for _, v := range baseArtifacts { - realRequires, _ = remove(realRequires, v) - realOptional, _ = remove(realOptional, v) - } - } - stage.RequireArtifacts = unique(realRequires) - stage.RequestArtifacts = unique(realOptional) -} - -// isValidArtifactShortHand checks if the shortand is valid -func isValidArtifactShortHand(a string) bool { - valid := false - for _, v := range strings.Split(strings.ToLower(a), "+") { - if cosa.CanArtifact(v) { - valid = true - } - for _, ps := range pseudoStages { - if v == ps { - valid = true - break - } - } - } - return valid -} - -// GenerateStages creates stages. -func (j *JobSpec) GenerateStages(fromNames, testNames []string, singleStage bool) error { - j.DelayedMetaMerge = true - j.Job.StrictMode = true - - for _, k := range fromNames { - if !isValidArtifactShortHand(k) { - return fmt.Errorf("artifact %s is an invalid artifact", k) - } - } - for _, k := range testNames { - if _, ok := kolaTestDefinitions[k]; !ok { - return fmt.Errorf("kola test %s is an invalid kola name", k) - } - - } - - if singleStage && len(fromNames) > 0 { - newList := []string{strings.Join(append(fromNames, testNames...), "+")} - fromNames = newList - } - - for _, k := range append(fromNames, testNames...) { - var s Stage - for _, k := range strings.Split(k, "+") { - addShorthandToStage(k, &s) - } - j.Stages = append(j.Stages, s) - } - - return nil -} - -// DeepCopy does a lazy deep copy by rendering the stage to JSON -// and then returning a new Stage defined by the JSON -func (s *Stage) DeepCopy() (Stage, error) { - ns := Stage{} - out, err := json.Marshal(s) - if err != nil { - return ns, err - } - err = json.Unmarshal(out, &ns) - return ns, err -} - -// addAllShortandsToStage adds all the shorthands -func addAllShorthandsToStage(stage *Stage, shorthands ...string) { - for _, short := range shorthands { - addShorthandToStage(short, stage) - } -} diff --git a/gangplank/internal/spec/tmpl.go b/gangplank/internal/spec/tmpl.go deleted file mode 100644 index a486c522..00000000 --- a/gangplank/internal/spec/tmpl.go +++ /dev/null @@ -1,107 +0,0 @@ -package spec - -import ( - "bytes" - "context" - "fmt" - "html/template" - "io" - "io/ioutil" - "os" - "os/exec" - "strings" - - "github.com/coreos/coreos-assembler-schema/cosa" - log "github.com/sirupsen/logrus" -) - -// RenderData is used to render commands -type RenderData struct { - JobSpec *JobSpec - Meta *cosa.Build -} - -// executeTemplate applies the template to r. -func (rd *RenderData) executeTemplate(r io.Reader) ([]byte, error) { - var in bytes.Buffer - if _, err := in.ReadFrom(r); err != nil { - return nil, err - } - - var out bytes.Buffer - tmpl, err := template.New("args").Parse(in.String()) - if err != nil { - return nil, err - } - - err = tmpl.Execute(&out, rd) - if err != nil { - return nil, err - } - - return out.Bytes(), nil -} - -// ExecuteTemplateFromString returns strings. -func (rd *RenderData) ExecuteTemplateFromString(s ...string) ([]string, error) { - var ret []string - for _, x := range s { - r := strings.NewReader(x) - b, err := rd.executeTemplate(r) - if err != nil { - return nil, fmt.Errorf("failed to render strings: %v", err) - } - ret = append(ret, string(b)) - } - return ret, nil -} - -// ExecuteTemplateToWriter renders an io.Reader to an io.Writer. -func (rd *RenderData) ExecuteTemplateToWriter(in io.Reader, out io.Writer) error { - d, err := rd.executeTemplate(in) - if err != nil { - return err - } - if _, err := out.Write(d); err != nil { - return err - } - return nil -} - -// RendererExecuter renders a script with templates and then executes it -func (rd *RenderData) RendererExecuter(ctx context.Context, env []string, scripts ...string) error { - rendered := make(map[string]*os.File) - for _, script := range scripts { - in, err := os.Open(script) - if err != nil { - return err - } - t, err := ioutil.TempFile("", "rendered") - if err != nil { - return err - } - defer os.Remove(t.Name()) - rendered[script] = t - - if err := rd.ExecuteTemplateToWriter(in, t); err != nil { - return err - } - } - - for i, v := range rendered { - cArgs := []string{"-xeu", "-o", "pipefail", v.Name()} - log.WithFields(log.Fields{ - "cmd": "/bin/bash", - "rendered": v.Name(), - }).Info("Executing rendered script") - cmd := exec.CommandContext(ctx, "/bin/bash", cArgs...) - cmd.Env = env - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("Script exited with return code %v", err) - } - log.WithFields(log.Fields{"script": i}).Info("Script complete") - } - return nil -} -- Gitee