diff --git a/mantle/cmd/kola/console.go b/mantle/cmd/kola/console.go index b3fff738e4e5e56851fdb4dcde700a5c53b997dc..37c409fcacaa557476c3295500d0b545f579ce57 100644 --- a/mantle/cmd/kola/console.go +++ b/mantle/cmd/kola/console.go @@ -74,8 +74,9 @@ func runCheckConsole(cmd *cobra.Command, args []string) error { errorcount++ continue } - for _, badness := range kola.CheckConsole(console, nil) { - fmt.Printf("%v: %v\n", sourceName, badness) + _, badlines := kola.CheckConsole(console, nil) + for _, badline := range badlines { + fmt.Printf("%v: %v\n", sourceName, badline) errorcount++ } } diff --git a/mantle/cmd/kola/devshell.go b/mantle/cmd/kola/devshell.go index 5b7d453271742c9f14802177ef0276eb065e37b5..3966203a2721c3553c89b67351033e40c054ef7f 100644 --- a/mantle/cmd/kola/devshell.go +++ b/mantle/cmd/kola/devshell.go @@ -50,7 +50,10 @@ func stripControlCharacters(s string) string { }, s) } -func displayStatusMsg(status, msg string, termMaxWidth int) { +func displayStatusMsg(ontty bool, status, msg string, termMaxWidth int) { + if !ontty { + return + } s := strings.TrimSpace(msg) if s == "" { return @@ -63,8 +66,11 @@ func displayStatusMsg(status, msg string, termMaxWidth int) { } func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *conf.Conf, sshCommand string) error { - if !term.IsTerminal(0) { - return fmt.Errorf("stdin is not a tty") + ontty := term.IsTerminal(0) + if sshCommand == "" { + if !ontty { + return fmt.Errorf("stdin is not a tty") + } } termMaxWidth, _, err := term.GetSize(0) if err != nil { @@ -108,9 +114,19 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co if err != nil { return err } - serialLog, err = os.CreateTemp(tmpd, "cosa-run-serial") - if err != nil { - return err + + // Save console logs + var serialLog *os.File + if builder.ConsoleFile != "" { + serialLog, err = os.Create(builder.ConsoleFile) + if err != nil { + return err + } + } else { + serialLog, err = os.CreateTemp(tmpd, "cosa-run-serial") + if err != nil { + return err + } } builder.InheritConsole = false @@ -170,6 +186,7 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co // Start the SSH client sc := newSshClient(ip, agent.Socket, sshCommand) + sc.ontty = ontty go sc.controlStartStop() ready := false @@ -187,7 +204,7 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co // a status message on the console. case serialMsg := <-serialChan: if !ready { - displayStatusMsg(statusMsg, serialMsg, termMaxWidth) + displayStatusMsg(ontty, statusMsg, serialMsg, termMaxWidth) } lastMsg = serialMsg // monitor the err channel @@ -201,7 +218,7 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co // monitor the instance state case <-qemuWaitChan: - displayStatusMsg("DONE", "QEMU instance terminated", termMaxWidth) + displayStatusMsg(ontty, "DONE", "QEMU instance terminated", termMaxWidth) return nil // monitor the machine state events from console/serial logs @@ -232,17 +249,17 @@ func runDevShellSSH(ctx context.Context, builder *platform.QemuBuilder, conf *co statusMsg = "QEMU guest is booting" } } - displayStatusMsg(fmt.Sprintf("EVENT | %s", statusMsg), lastMsg, termMaxWidth) + displayStatusMsg(ontty, fmt.Sprintf("EVENT | %s", statusMsg), lastMsg, termMaxWidth) // monitor the SSH connection case err := <-sc.errChan: if err == nil { sc.controlChan <- sshNotReady - displayStatusMsg("SESSION", "Clean exit from SSH, terminating instance", termMaxWidth) + displayStatusMsg(ontty, "SESSION", "Clean exit from SSH, terminating instance", termMaxWidth) return nil } else if sshCommand != "" { sc.controlChan <- sshNotReady - displayStatusMsg("SESSION", "SSH command exited, terminating instance", termMaxWidth) + displayStatusMsg(ontty, "SESSION", "SSH command exited, terminating instance", termMaxWidth) return err } if ready { @@ -455,6 +472,7 @@ type sshClient struct { port string agent string cmd string + ontty bool controlChan chan sshControlMessage errChan chan error sshCmd *exec.Cmd @@ -511,8 +529,10 @@ func (sc *sshClient) start() { if sc.cmd != "" { sshArgs = append(sshArgs, "--", sc.cmd) } - fmt.Printf("\033[2K\r") // clear serial console line - fmt.Printf("[SESSION] Starting SSH\r") // and stage a status msg which will be erased + if sc.ontty { + fmt.Printf("\033[2K\r") // clear serial console line + fmt.Printf("[SESSION] Starting SSH\r") // and stage a status msg which will be erased + } sshCmd := exec.Command(sshArgs[0], sshArgs[1:]...) sshCmd.Stdin = os.Stdin sshCmd.Stdout = os.Stdout @@ -531,7 +551,7 @@ func (sc *sshClient) start() { for scanner.Scan() { msg := scanner.Text() if strings.Contains(msg, "Connection to 127.0.0.1 closed") { - displayStatusMsg("SSH", "connection closed", 0) + displayStatusMsg(sc.ontty, "SSH", "connection closed", 0) } } }() diff --git a/mantle/harness/harness.go b/mantle/harness/harness.go index 24f798496b77b2d5dcc16d3961874e1640d08ecc..bb96fc8bdfc552ea3d1926b6890bd96dadf62fc0 100644 --- a/mantle/harness/harness.go +++ b/mantle/harness/harness.go @@ -76,6 +76,7 @@ type H struct { isParallel bool nonExclusiveTestsStarted bool + warningOnFailure bool timeout time.Duration // Duration for which the test will be allowed to run execTimer *time.Timer // Used to interrupt the test after timeout @@ -165,6 +166,9 @@ func (h *H) Verbose() bool { func (c *H) status() testresult.TestResult { if c.Failed() { + if c.warningOnFailure { + return testresult.Warn + } return testresult.Fail } else if c.Skipped() { return testresult.Skip @@ -274,6 +278,10 @@ func (c *H) NonExclusiveTestStarted() { c.nonExclusiveTestsStarted = true } +func (c *H) WarningOnFailure() { + c.warningOnFailure = true +} + // Fail marks the function as having failed but continues execution. func (c *H) Fail() { if c.parent != nil { diff --git a/mantle/harness/harness_test.go b/mantle/harness/harness_test.go index 7b08ca01073fd8142724f6707aaa6debdc273cf9..17f39c0279b2b712c2e581687c11cd6cfa9fe685 100644 --- a/mantle/harness/harness_test.go +++ b/mantle/harness/harness_test.go @@ -76,6 +76,9 @@ func TestContextCancel(t *testing.T) { } func TestSubTests(t *testing.T) { + // When TERM is set, we add colors to highlight tests results: '--- FAIL' will be in red '--- \033[31mFAIL\033[0m' + // Let's unset it here for simplicity + os.Unsetenv("TERM") realTest := t testCases := []struct { desc string diff --git a/mantle/harness/testresult/status.go b/mantle/harness/testresult/status.go index 3817aed8fe6ead3ca0c8be01c98c111840bb8c9d..53daec575bd7e129b69e490eb2f7d04e28149c67 100644 --- a/mantle/harness/testresult/status.go +++ b/mantle/harness/testresult/status.go @@ -16,6 +16,7 @@ package testresult const ( Fail TestResult = "FAIL" + Warn TestResult = "WARN" Skip TestResult = "SKIP" Pass TestResult = "PASS" ) diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index 6b1cb3becb54e9383e122fa8d357703357695fa3..ad3dd8758ca38e12f8b4e584f3ed0122a9a2c227 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -114,8 +114,9 @@ var ( // ForceRunPlatformIndependent will cause tests that claim platform-independence to run ForceRunPlatformIndependent bool - DenylistedTests []string // tests which are on the denylist - Tags []string // tags to be ran + DenylistedTests []string // tests which are on the denylist + WarnOnErrorTests []string // denylisted tests we are going to run and warn in case of error + Tags []string // tags to be ran extTestNum = 1 // Assigns a unique number to each non-exclusive external test testResults protectedTestResults @@ -126,11 +127,13 @@ var ( consoleChecks = []struct { desc string match *regexp.Regexp + warnOnly bool skipFlag *register.Flag }{ { desc: "emergency shell", match: regexp.MustCompile("Press Enter for emergency shell|Starting Emergency Shell|You are in emergency mode"), + warnOnly: false, skipFlag: &[]register.Flag{register.NoEmergencyShellCheck}[0], }, { @@ -334,6 +337,7 @@ type DenyListObj struct { Platforms []string `yaml:"platforms"` SnoozeDate string `yaml:"snooze"` OsVersion []string `yaml:"osversion"` + Warn bool `yaml:"warn"` } type ManifestData struct { @@ -347,7 +351,7 @@ type InitConfigData struct { ConfigVariant string `json:"coreos-assembler.config-variant"` } -func parseDenyListYaml(pltfrm string) error { +func ParseDenyListYaml(pltfrm string) error { var objs []DenyListObj // Parse kola-denylist into structs @@ -409,19 +413,19 @@ func parseDenyListYaml(pltfrm string) error { // Accumulate patterns filtering by set policies plog.Debug("Processing denial patterns from yaml...") for _, obj := range objs { - if len(obj.Arches) > 0 && !hasString(arch, obj.Arches) { + if len(obj.Arches) > 0 && !HasString(arch, obj.Arches) { continue } - if len(obj.Platforms) > 0 && !hasString(pltfrm, obj.Platforms) { + if len(obj.Platforms) > 0 && !HasString(pltfrm, obj.Platforms) { continue } - if len(stream) > 0 && len(obj.Streams) > 0 && !hasString(stream, obj.Streams) { + if len(stream) > 0 && len(obj.Streams) > 0 && !HasString(stream, obj.Streams) { continue } - if len(osversion) > 0 && len(obj.OsVersion) > 0 && !hasString(osversion, obj.OsVersion) { + if len(osversion) > 0 && len(obj.OsVersion) > 0 && !HasString(osversion, obj.OsVersion) { continue } @@ -1432,6 +1436,9 @@ func makeNonExclusiveTest(bucket int, tests []*register.Test, flight platform.Fl // Collect the journal logs after execution is finished defer collectLogsExternalTest(h, t, newTC) } + if IsWarningOnFailure(t.Name) { + newTC.H.WarningOnFailure() + } t.Run(newTC) } @@ -1490,15 +1497,24 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig plog.Debugf("Skipping base checks for %s", t.Name) return } - for id, output := range c.ConsoleOutput() { - for _, badness := range CheckConsole([]byte(output), t) { - h.Errorf("Found %s on machine %s console", badness, id) + handleConsoleChecks := func(logtype, id, output string) { + warnOnly, badlines := CheckConsole([]byte(output), t) + if SkipConsoleWarnings { + warnOnly = true + } + for _, badline := range badlines { + if warnOnly { + plog.Warningf("Found %s on machine %s %s", badline, id, logtype) + } else { + h.Errorf("Found %s on machine %s %s", badline, id, logtype) + } } } + for id, output := range c.ConsoleOutput() { + handleConsoleChecks("console", id, output) + } for id, output := range c.JournalOutput() { - for _, badness := range CheckConsole([]byte(output), t) { - h.Errorf("Found %s on machine %s journal", badness, id) - } + handleConsoleChecks("journal", id, output) } }()