From 6aaeeb32d4ec8af5ad57892fd2bae0afe7b7df1e Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Fri, 4 Oct 2019 14:47:38 -0700 Subject: [PATCH] Refactor a bit more for testing, update tests --- README.md | 2 +- alert.go | 7 ++--- main.go | 74 ++++++++++++++++++++++++++++--------------------- monitor.go | 36 +++++++++++++++--------- monitor_test.go | 25 +++++++++++++++++ util.go | 13 +++++++++ util_test.go | 32 +++++++++++++++++++++ 7 files changed, 139 insertions(+), 50 deletions(-) create mode 100644 util_test.go diff --git a/README.md b/README.md index 4000c15..8ab301e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Initial target is meant to be roughly compatible requiring only minor changes to ## Differences from Python version -There are a few key differences between the Python version and the v0.x Go version. +There are a few key differences between the Python version and the v0.x Go version. First, configuration keys cannot have multiple types in Go, so a different key must be used when specifying a Shell command as a string rather than a list of args. Instead of `command`, you must use `command_shell`. Eg: diff --git a/alert.go b/alert.go index 2b78ac6..e6f4938 100644 --- a/alert.go +++ b/alert.go @@ -38,18 +38,15 @@ func (alert Alert) IsValid() bool { // BuildTemplates compiles command templates for the Alert func (alert *Alert) BuildTemplates() error { + log.Printf("DEBUG: Building template for alert %s", alert.Name) if alert.commandTemplate == nil && alert.Command != nil { - // build template - log.Println("Building template for command...") alert.commandTemplate = []*template.Template{} for i, cmdPart := range alert.Command { alert.commandTemplate = append(alert.commandTemplate, template.Must( template.New(alert.Name+string(i)).Parse(cmdPart), )) } - log.Printf("Template built: %v", alert.commandTemplate) } else if alert.commandShellTemplate == nil && alert.CommandShell != "" { - log.Println("Building template for shell command...") alert.commandShellTemplate = template.Must( template.New(alert.Name).Parse(alert.CommandShell), ) @@ -96,7 +93,7 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) { var output []byte output, err = cmd.CombinedOutput() output_str = string(output) - log.Printf("Check %s\n---\n%s\n---", alert.Name, output_str) + // log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str) return output_str, err } diff --git a/main.go b/main.go index 3004a74..a811de7 100644 --- a/main.go +++ b/main.go @@ -5,44 +5,56 @@ import ( "time" ) +func checkMonitors(config *Config) { + for _, monitor := range config.Monitors { + if monitor.ShouldCheck() { + _, alertNotice := monitor.Check() + + // Should probably consider refactoring everything below here + if alertNotice != nil { + log.Printf("DEBUG: Recieved an alert notice: %v", alertNotice) + alertNames := monitor.GetAlertNames(alertNotice.IsUp) + if alertNames == nil { + // TODO: Should this be a panic? Should this be validated against? Probably + log.Printf("WARNING: Found alert, but no alert mechanisms exist: %v", alertNotice) + } + for _, alertName := range alertNames { + if alert, ok := config.Alerts[alertName]; ok { + output, err := alert.Send(*alertNotice) + if err != nil { + log.Printf( + "ERROR: Alert '%s' failed. result=%v: output=%s", + alert.Name, + err, + output, + ) + // TODO: Maybe return this error instead of panicking here + log.Fatalf( + "ERROR: Unsuccessfully triggered alert '%s'. "+ + "Crashing to avoid false negatives: %v", + alert.Name, + err, + ) + } + } else { + // TODO: Maybe panic here. Also, probably validate up front + log.Printf("ERROR: Alert with name '%s' not found", alertName) + } + } + } + } + } +} + func main() { config, err := LoadConfig("config.yml") if err != nil { log.Fatalf("Error loading config: %v", err) } + // Start main loop for { - for _, monitor := range config.Monitors { - if monitor.ShouldCheck() { - _, alertNotice := monitor.Check() - - // Should probably consider refactoring everything below here - if alertNotice != nil { - //log.Printf("Recieved an alert notice: %v", alertNotice) - var alerts []string - if alertNotice.IsUp { - alerts = monitor.AlertUp - log.Printf("Alert up: %v", monitor.AlertUp) - } else { - alerts = monitor.AlertDown - log.Printf("Alert down: %v", monitor.AlertDown) - } - if alerts == nil { - log.Printf("WARNING: Found alert, but no alert mechanism: %v", alertNotice) - } - for _, alertName := range alerts { - if alert, ok := config.Alerts[alertName]; ok { - _, err := alert.Send(*alertNotice) - if err != nil { - panic(err) - } - } else { - log.Printf("WARNING: Could not find alert for %s", alertName) - } - } - } - } - } + checkMonitors(&config) sleepTime := time.Duration(config.CheckInterval) * time.Second time.Sleep(sleepTime) diff --git a/monitor.go b/monitor.go index 514926c..0577653 100644 --- a/monitor.go +++ b/monitor.go @@ -56,29 +56,27 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) { } output, err := cmd.CombinedOutput() - //log.Printf("Check %s\n---\n%s\n---", monitor.Name, string(output)) - - isSuccess := (err == nil) - if err != nil { - log.Printf("ERROR: %v", err) - } - monitor.lastCheck = time.Now() monitor.lastOutput = string(output) var alertNotice *AlertNotice + isSuccess := (err == nil) if isSuccess { alertNotice = monitor.success() } else { alertNotice = monitor.failure() } + // log.Printf("DEBUG: Command output: %s", monitor.lastOutput) + if err != nil { + log.Printf("DEBUG: Command result: %v", err) + } + log.Printf( - "Check result for %s: %v, %v at %v", + "INFO: %s success=%t, alert=%t", monitor.Name, isSuccess, - alertNotice, - monitor.lastCheck, + alertNotice != nil, ) return isSuccess, alertNotice @@ -89,7 +87,6 @@ func (monitor Monitor) isUp() bool { } func (monitor *Monitor) success() (notice *AlertNotice) { - log.Printf("Great success!") if !monitor.isUp() { // Alert that we have recovered notice = monitor.createAlertNotice(true) @@ -102,20 +99,23 @@ func (monitor *Monitor) success() (notice *AlertNotice) { } func (monitor *Monitor) failure() (notice *AlertNotice) { - log.Printf("Devastating failure. :(") monitor.failureCount++ // If we haven't hit the minimum failures, we can exit if monitor.failureCount < monitor.getAlertAfter() { log.Printf( - "Have not hit minimum failures. failures: %v alert after: %v", + "DEBUG: %s failed but did not hit minimum failures. "+ + "Count: %v alert after: %v", + monitor.Name, monitor.failureCount, monitor.getAlertAfter(), ) return } + // Take number of failures after minimum failureCount := (monitor.failureCount - monitor.getAlertAfter()) + // Use alert cadence to determine if we should alert if monitor.AlertEvery > 0 { // Handle integer number of failures before alerting if failureCount%monitor.AlertEvery == 0 { @@ -133,6 +133,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) { } } + // If we're going to alert, increment count if notice != nil { monitor.alertCount++ } @@ -150,6 +151,15 @@ func (monitor Monitor) getAlertAfter() int16 { } } +// GetAlertNames gives a list of alert names for a given monitor status +func (monitor Monitor) GetAlertNames(up bool) []string { + if up { + return monitor.AlertUp + } else { + return monitor.AlertDown + } +} + func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice { // TODO: Maybe add something about recovery status here return &AlertNotice{ diff --git a/monitor_test.go b/monitor_test.go index 9747f33..a645e11 100644 --- a/monitor_test.go +++ b/monitor_test.go @@ -85,6 +85,31 @@ func TestMonitorIsUp(t *testing.T) { } } +// TestMonitorGetAlertNames tests that proper alert names are returned +func TestMonitorGetAlertNames(t *testing.T) { + cases := []struct { + monitor Monitor + up bool + expected []string + name string + }{ + {Monitor{}, true, nil, "Empty up"}, + {Monitor{}, false, nil, "Empty down"}, + {Monitor{AlertUp: []string{"alert"}}, true, []string{"alert"}, "Return up"}, + {Monitor{AlertDown: []string{"alert"}}, false, []string{"alert"}, "Return down"}, + } + + for _, c := range cases { + log.Printf("Testing case %s", c.name) + actual := c.monitor.GetAlertNames(c.up) + if !EqualSliceString(actual, c.expected) { + t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual) + log.Printf("Case failed: %s", c.name) + } + log.Println("-----") + } +} + // TestMonitorSuccess tests the Monitor.success() func TestMonitorSuccess(t *testing.T) { cases := []struct { diff --git a/util.go b/util.go index b54c42f..26c9c48 100644 --- a/util.go +++ b/util.go @@ -21,3 +21,16 @@ func ShellCommand(command string) *exec.Cmd { //log.Printf("Shell command: %v", shellCommand) return exec.Command(shellCommand[0], shellCommand[1:]...) } + +// EqualSliceString checks if two string slices are equivalent +func EqualSliceString(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i, val := range a { + if val != b[i] { + return false + } + } + return true +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 0000000..df4cb47 --- /dev/null +++ b/util_test.go @@ -0,0 +1,32 @@ +package main + +import "testing" + +func TestUtilEqualSliceString(t *testing.T) { + cases := []struct { + a, b []string + expected bool + }{ + // Equal cases + {nil, nil, true}, + {nil, []string{}, true}, + {[]string{}, nil, true}, + {[]string{"a"}, []string{"a"}, true}, + // Inequal cases + {nil, []string{"b"}, false}, + {[]string{"a"}, nil, false}, + {[]string{"a"}, []string{"b"}, false}, + {[]string{"a"}, []string{"a", "b"}, false}, + {[]string{"a", "b"}, []string{"b"}, false}, + } + + for _, c := range cases { + actual := EqualSliceString(c.a, c.b) + if actual != c.expected { + t.Errorf( + "EqualSliceString(%v, %v), expected=%v actual=%v", + c.a, c.b, c.expected, actual, + ) + } + } +}