Refactor a bit more for testing, update tests

README.md

@@ -8,7 +8,7 @@ Initial target is meant to be roughly compatible requiring only minor changes to
 
 ## Differences from Python version
 
-There are a few key differences between the Python version and the v0.x Go version. 
+There are a few key differences between the Python version and the v0.x Go version.
 
 First, configuration keys cannot have multiple types in Go, so a different key must be used when specifying a Shell command as a string rather than a list of args. Instead of `command`, you must use `command_shell`. Eg:
 

alert.go

@@ -38,18 +38,15 @@ func (alert Alert) IsValid() bool {
 
 // BuildTemplates compiles command templates for the Alert
 func (alert *Alert) BuildTemplates() error {
+	log.Printf("DEBUG: Building template for alert %s", alert.Name)
 	if alert.commandTemplate == nil && alert.Command != nil {
-		// build template
-		log.Println("Building template for command...")
 		alert.commandTemplate = []*template.Template{}
 		for i, cmdPart := range alert.Command {
 			alert.commandTemplate = append(alert.commandTemplate, template.Must(
 				template.New(alert.Name+string(i)).Parse(cmdPart),
 			))
 		}
-		log.Printf("Template built: %v", alert.commandTemplate)
 	} else if alert.commandShellTemplate == nil && alert.CommandShell != "" {
-		log.Println("Building template for shell command...")
 		alert.commandShellTemplate = template.Must(
 			template.New(alert.Name).Parse(alert.CommandShell),
 		)
@@ -96,7 +93,7 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
 	var output []byte
 	output, err = cmd.CombinedOutput()
 	output_str = string(output)
-	log.Printf("Check %s\n---\n%s\n---", alert.Name, output_str)
+	// log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
 
 	return output_str, err
 }

main.go

@@ -5,44 +5,56 @@ import (
 	"time"
 )
 
+func checkMonitors(config *Config) {
+	for _, monitor := range config.Monitors {
+		if monitor.ShouldCheck() {
+			_, alertNotice := monitor.Check()
+
+			// Should probably consider refactoring everything below here
+			if alertNotice != nil {
+				log.Printf("DEBUG: Recieved an alert notice: %v", alertNotice)
+				alertNames := monitor.GetAlertNames(alertNotice.IsUp)
+				if alertNames == nil {
+					// TODO: Should this be a panic? Should this be validated against? Probably
+					log.Printf("WARNING: Found alert, but no alert mechanisms exist: %v", alertNotice)
+				}
+				for _, alertName := range alertNames {
+					if alert, ok := config.Alerts[alertName]; ok {
+						output, err := alert.Send(*alertNotice)
+						if err != nil {
+							log.Printf(
+								"ERROR: Alert '%s' failed. result=%v: output=%s",
+								alert.Name,
+								err,
+								output,
+							)
+							// TODO: Maybe return this error instead of panicking here
+							log.Fatalf(
+								"ERROR: Unsuccessfully triggered alert '%s'. "+
+									"Crashing to avoid false negatives: %v",
+								alert.Name,
+								err,
+							)
+						}
+					} else {
+						// TODO: Maybe panic here. Also, probably validate up front
+						log.Printf("ERROR: Alert with name '%s' not found", alertName)
+					}
+				}
+			}
+		}
+	}
+}
+
 func main() {
 	config, err := LoadConfig("config.yml")
 	if err != nil {
 		log.Fatalf("Error loading config: %v", err)
 	}
 
+	// Start main loop
 	for {
-		for _, monitor := range config.Monitors {
+		checkMonitors(&config)
-			if monitor.ShouldCheck() {
-				_, alertNotice := monitor.Check()
-
-				// Should probably consider refactoring everything below here
-				if alertNotice != nil {
-					//log.Printf("Recieved an alert notice: %v", alertNotice)
-					var alerts []string
-					if alertNotice.IsUp {
-						alerts = monitor.AlertUp
-						log.Printf("Alert up: %v", monitor.AlertUp)
-					} else {
-						alerts = monitor.AlertDown
-						log.Printf("Alert down: %v", monitor.AlertDown)
-					}
-					if alerts == nil {
-						log.Printf("WARNING: Found alert, but no alert mechanism: %v", alertNotice)
-					}
-					for _, alertName := range alerts {
-						if alert, ok := config.Alerts[alertName]; ok {
-							_, err := alert.Send(*alertNotice)
-							if err != nil {
-								panic(err)
-							}
-						} else {
-							log.Printf("WARNING: Could not find alert for %s", alertName)
-						}
-					}
-				}
-			}
-		}
 
 		sleepTime := time.Duration(config.CheckInterval) * time.Second
 		time.Sleep(sleepTime)

monitor.go

@@ -56,29 +56,27 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 	}
 
 	output, err := cmd.CombinedOutput()
-	//log.Printf("Check %s\n---\n%s\n---", monitor.Name, string(output))
-
-	isSuccess := (err == nil)
-	if err != nil {
-		log.Printf("ERROR: %v", err)
-	}
-
 	monitor.lastCheck = time.Now()
 	monitor.lastOutput = string(output)
 
 	var alertNotice *AlertNotice
+	isSuccess := (err == nil)
 	if isSuccess {
 		alertNotice = monitor.success()
 	} else {
 		alertNotice = monitor.failure()
 	}
 
+	// log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
+	if err != nil {
+		log.Printf("DEBUG: Command result: %v", err)
+	}
+
 	log.Printf(
-		"Check result for %s: %v, %v at %v",
+		"INFO: %s success=%t, alert=%t",
 		monitor.Name,
 		isSuccess,
-		alertNotice,
+		alertNotice != nil,
-		monitor.lastCheck,
 	)
 
 	return isSuccess, alertNotice
@@ -89,7 +87,6 @@ func (monitor Monitor) isUp() bool {
 }
 
 func (monitor *Monitor) success() (notice *AlertNotice) {
-	log.Printf("Great success!")
 	if !monitor.isUp() {
 		// Alert that we have recovered
 		notice = monitor.createAlertNotice(true)
@@ -102,20 +99,23 @@ func (monitor *Monitor) success() (notice *AlertNotice) {
 }
 
 func (monitor *Monitor) failure() (notice *AlertNotice) {
-	log.Printf("Devastating failure. :(")
 	monitor.failureCount++
 	// If we haven't hit the minimum failures, we can exit
 	if monitor.failureCount < monitor.getAlertAfter() {
 		log.Printf(
-			"Have not hit minimum failures. failures: %v alert after: %v",
+			"DEBUG: %s failed but did not hit minimum failures. "+
+				"Count: %v alert after: %v",
+			monitor.Name,
 			monitor.failureCount,
 			monitor.getAlertAfter(),
 		)
 		return
 	}
 
+	// Take number of failures after minimum
 	failureCount := (monitor.failureCount - monitor.getAlertAfter())
 
+	// Use alert cadence to determine if we should alert
 	if monitor.AlertEvery > 0 {
 		// Handle integer number of failures before alerting
 		if failureCount%monitor.AlertEvery == 0 {
@@ -133,6 +133,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 		}
 	}
 
+	// If we're going to alert, increment count
 	if notice != nil {
 		monitor.alertCount++
 	}
@@ -150,6 +151,15 @@ func (monitor Monitor) getAlertAfter() int16 {
 	}
 }
 
+// GetAlertNames gives a list of alert names for a given monitor status
+func (monitor Monitor) GetAlertNames(up bool) []string {
+	if up {
+		return monitor.AlertUp
+	} else {
+		return monitor.AlertDown
+	}
+}
+
 func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
 	// TODO: Maybe add something about recovery status here
 	return &AlertNotice{

monitor_test.go

@@ -85,6 +85,31 @@ func TestMonitorIsUp(t *testing.T) {
 	}
 }
 
+// TestMonitorGetAlertNames tests that proper alert names are returned
+func TestMonitorGetAlertNames(t *testing.T) {
+	cases := []struct {
+		monitor  Monitor
+		up       bool
+		expected []string
+		name     string
+	}{
+		{Monitor{}, true, nil, "Empty up"},
+		{Monitor{}, false, nil, "Empty down"},
+		{Monitor{AlertUp: []string{"alert"}}, true, []string{"alert"}, "Return up"},
+		{Monitor{AlertDown: []string{"alert"}}, false, []string{"alert"}, "Return down"},
+	}
+
+	for _, c := range cases {
+		log.Printf("Testing case %s", c.name)
+		actual := c.monitor.GetAlertNames(c.up)
+		if !EqualSliceString(actual, c.expected) {
+			t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
+			log.Printf("Case failed: %s", c.name)
+		}
+		log.Println("-----")
+	}
+}
+
 // TestMonitorSuccess tests the Monitor.success()
 func TestMonitorSuccess(t *testing.T) {
 	cases := []struct {

util.go

@@ -21,3 +21,16 @@ func ShellCommand(command string) *exec.Cmd {
 	//log.Printf("Shell command: %v", shellCommand)
 	return exec.Command(shellCommand[0], shellCommand[1:]...)
 }
+
+// EqualSliceString checks if two string slices are equivalent
+func EqualSliceString(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, val := range a {
+		if val != b[i] {
+			return false
+		}
+	}
+	return true
+}

util_test.go

@@ -0,0 +1,32 @@
+package main
+
+import "testing"
+
+func TestUtilEqualSliceString(t *testing.T) {
+	cases := []struct {
+		a, b     []string
+		expected bool
+	}{
+		// Equal cases
+		{nil, nil, true},
+		{nil, []string{}, true},
+		{[]string{}, nil, true},
+		{[]string{"a"}, []string{"a"}, true},
+		// Inequal cases
+		{nil, []string{"b"}, false},
+		{[]string{"a"}, nil, false},
+		{[]string{"a"}, []string{"b"}, false},
+		{[]string{"a"}, []string{"a", "b"}, false},
+		{[]string{"a", "b"}, []string{"b"}, false},
+	}
+
+	for _, c := range cases {
+		actual := EqualSliceString(c.a, c.b)
+		if actual != c.expected {
+			t.Errorf(
+				"EqualSliceString(%v, %v), expected=%v actual=%v",
+				c.a, c.b, c.expected, actual,
+			)
+		}
+	}
+}