Refactor a bit more for testing, update tests

alert.go

@@ -38,18 +38,15 @@ func (alert Alert) IsValid() bool {
 
 // BuildTemplates compiles command templates for the Alert
 func (alert *Alert) BuildTemplates() error {
+	log.Printf("DEBUG: Building template for alert %s", alert.Name)
 	if alert.commandTemplate == nil && alert.Command != nil {
-		// build template
-		log.Println("Building template for command...")
 		alert.commandTemplate = []*template.Template{}
 		for i, cmdPart := range alert.Command {
 			alert.commandTemplate = append(alert.commandTemplate, template.Must(
 				template.New(alert.Name+string(i)).Parse(cmdPart),
 			))
 		}
-		log.Printf("Template built: %v", alert.commandTemplate)
 	} else if alert.commandShellTemplate == nil && alert.CommandShell != "" {
-		log.Println("Building template for shell command...")
 		alert.commandShellTemplate = template.Must(
 			template.New(alert.Name).Parse(alert.CommandShell),
 		)
@@ -96,7 +93,7 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
 	var output []byte
 	output, err = cmd.CombinedOutput()
 	output_str = string(output)
-	log.Printf("Check %s\n---\n%s\n---", alert.Name, output_str)
+	// log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
 
 	return output_str, err
 }

main.go

@@ -5,45 +5,57 @@ import (
 	"time"
 )
 
-func main() {
+func checkMonitors(config *Config) {
-	config, err := LoadConfig("config.yml")
-	if err != nil {
-		log.Fatalf("Error loading config: %v", err)
-	}
-
-	for {
 	for _, monitor := range config.Monitors {
 		if monitor.ShouldCheck() {
 			_, alertNotice := monitor.Check()
 
 			// Should probably consider refactoring everything below here
 			if alertNotice != nil {
-					//log.Printf("Recieved an alert notice: %v", alertNotice)
+				log.Printf("DEBUG: Recieved an alert notice: %v", alertNotice)
-					var alerts []string
+				alertNames := monitor.GetAlertNames(alertNotice.IsUp)
-					if alertNotice.IsUp {
+				if alertNames == nil {
-						alerts = monitor.AlertUp
+					// TODO: Should this be a panic? Should this be validated against? Probably
-						log.Printf("Alert up: %v", monitor.AlertUp)
+					log.Printf("WARNING: Found alert, but no alert mechanisms exist: %v", alertNotice)
-					} else {
-						alerts = monitor.AlertDown
-						log.Printf("Alert down: %v", monitor.AlertDown)
 				}
-					if alerts == nil {
+				for _, alertName := range alertNames {
-						log.Printf("WARNING: Found alert, but no alert mechanism: %v", alertNotice)
-					}
-					for _, alertName := range alerts {
 					if alert, ok := config.Alerts[alertName]; ok {
-							_, err := alert.Send(*alertNotice)
+						output, err := alert.Send(*alertNotice)
 						if err != nil {
-								panic(err)
+							log.Printf(
+								"ERROR: Alert '%s' failed. result=%v: output=%s",
+								alert.Name,
+								err,
+								output,
+							)
+							// TODO: Maybe return this error instead of panicking here
+							log.Fatalf(
+								"ERROR: Unsuccessfully triggered alert '%s'. "+
+									"Crashing to avoid false negatives: %v",
+								alert.Name,
+								err,
+							)
 						}
 					} else {
-							log.Printf("WARNING: Could not find alert for %s", alertName)
+						// TODO: Maybe panic here. Also, probably validate up front
+						log.Printf("ERROR: Alert with name '%s' not found", alertName)
+					}
 				}
 			}
 		}
 	}
 }
 
+func main() {
+	config, err := LoadConfig("config.yml")
+	if err != nil {
+		log.Fatalf("Error loading config: %v", err)
+	}
+
+	// Start main loop
+	for {
+		checkMonitors(&config)
+
 		sleepTime := time.Duration(config.CheckInterval) * time.Second
 		time.Sleep(sleepTime)
 	}

monitor.go

@@ -56,29 +56,27 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
 	}
 
 	output, err := cmd.CombinedOutput()
-	//log.Printf("Check %s\n---\n%s\n---", monitor.Name, string(output))
-
-	isSuccess := (err == nil)
-	if err != nil {
-		log.Printf("ERROR: %v", err)
-	}
-
 	monitor.lastCheck = time.Now()
 	monitor.lastOutput = string(output)
 
 	var alertNotice *AlertNotice
+	isSuccess := (err == nil)
 	if isSuccess {
 		alertNotice = monitor.success()
 	} else {
 		alertNotice = monitor.failure()
 	}
 
+	// log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
+	if err != nil {
+		log.Printf("DEBUG: Command result: %v", err)
+	}
+
 	log.Printf(
-		"Check result for %s: %v, %v at %v",
+		"INFO: %s success=%t, alert=%t",
 		monitor.Name,
 		isSuccess,
-		alertNotice,
+		alertNotice != nil,
-		monitor.lastCheck,
 	)
 
 	return isSuccess, alertNotice
@@ -89,7 +87,6 @@ func (monitor Monitor) isUp() bool {
 }
 
 func (monitor *Monitor) success() (notice *AlertNotice) {
-	log.Printf("Great success!")
 	if !monitor.isUp() {
 		// Alert that we have recovered
 		notice = monitor.createAlertNotice(true)
@@ -102,20 +99,23 @@ func (monitor *Monitor) success() (notice *AlertNotice) {
 }
 
 func (monitor *Monitor) failure() (notice *AlertNotice) {
-	log.Printf("Devastating failure. :(")
 	monitor.failureCount++
 	// If we haven't hit the minimum failures, we can exit
 	if monitor.failureCount < monitor.getAlertAfter() {
 		log.Printf(
-			"Have not hit minimum failures. failures: %v alert after: %v",
+			"DEBUG: %s failed but did not hit minimum failures. "+
+				"Count: %v alert after: %v",
+			monitor.Name,
 			monitor.failureCount,
 			monitor.getAlertAfter(),
 		)
 		return
 	}
 
+	// Take number of failures after minimum
 	failureCount := (monitor.failureCount - monitor.getAlertAfter())
 
+	// Use alert cadence to determine if we should alert
 	if monitor.AlertEvery > 0 {
 		// Handle integer number of failures before alerting
 		if failureCount%monitor.AlertEvery == 0 {
@@ -133,6 +133,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
 		}
 	}
 
+	// If we're going to alert, increment count
 	if notice != nil {
 		monitor.alertCount++
 	}
@@ -150,6 +151,15 @@ func (monitor Monitor) getAlertAfter() int16 {
 	}
 }
 
+// GetAlertNames gives a list of alert names for a given monitor status
+func (monitor Monitor) GetAlertNames(up bool) []string {
+	if up {
+		return monitor.AlertUp
+	} else {
+		return monitor.AlertDown
+	}
+}
+
 func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
 	// TODO: Maybe add something about recovery status here
 	return &AlertNotice{

monitor_test.go

@@ -85,6 +85,31 @@ func TestMonitorIsUp(t *testing.T) {
 	}
 }
 
+// TestMonitorGetAlertNames tests that proper alert names are returned
+func TestMonitorGetAlertNames(t *testing.T) {
+	cases := []struct {
+		monitor  Monitor
+		up       bool
+		expected []string
+		name     string
+	}{
+		{Monitor{}, true, nil, "Empty up"},
+		{Monitor{}, false, nil, "Empty down"},
+		{Monitor{AlertUp: []string{"alert"}}, true, []string{"alert"}, "Return up"},
+		{Monitor{AlertDown: []string{"alert"}}, false, []string{"alert"}, "Return down"},
+	}
+
+	for _, c := range cases {
+		log.Printf("Testing case %s", c.name)
+		actual := c.monitor.GetAlertNames(c.up)
+		if !EqualSliceString(actual, c.expected) {
+			t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
+			log.Printf("Case failed: %s", c.name)
+		}
+		log.Println("-----")
+	}
+}
+
 // TestMonitorSuccess tests the Monitor.success()
 func TestMonitorSuccess(t *testing.T) {
 	cases := []struct {

util.go

@@ -21,3 +21,16 @@ func ShellCommand(command string) *exec.Cmd {
 	//log.Printf("Shell command: %v", shellCommand)
 	return exec.Command(shellCommand[0], shellCommand[1:]...)
 }
+
+// EqualSliceString checks if two string slices are equivalent
+func EqualSliceString(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, val := range a {
+		if val != b[i] {
+			return false
+		}
+	}
+	return true
+}

util_test.go

@@ -0,0 +1,32 @@
+package main
+
+import "testing"
+
+func TestUtilEqualSliceString(t *testing.T) {
+	cases := []struct {
+		a, b     []string
+		expected bool
+	}{
+		// Equal cases
+		{nil, nil, true},
+		{nil, []string{}, true},
+		{[]string{}, nil, true},
+		{[]string{"a"}, []string{"a"}, true},
+		// Inequal cases
+		{nil, []string{"b"}, false},
+		{[]string{"a"}, nil, false},
+		{[]string{"a"}, []string{"b"}, false},
+		{[]string{"a"}, []string{"a", "b"}, false},
+		{[]string{"a", "b"}, []string{"b"}, false},
+	}
+
+	for _, c := range cases {
+		actual := EqualSliceString(c.a, c.b)
+		if actual != c.expected {
+			t.Errorf(
+				"EqualSliceString(%v, %v), expected=%v actual=%v",
+				c.a, c.b, c.expected, actual,
+			)
+		}
+	}
+}