iamthefij/minitor-go
1
0
Fork 0
Code Issues 3 Pull Requests Releases 9 Wiki Activity

Refactor a bit more for testing, update tests

Browse Source
This commit is contained in:
IamTheFij 2019-10-04 14:47:38 -07:00
parent 5c5388d683
commit 6aaeeb32d4
7 changed files with 139 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
alert.go
View File

@ -38,18 +38,15 @@ func (alert Alert) IsValid() bool {
// BuildTemplates compiles command templates for the Alert
func (alert *Alert) BuildTemplates() error {
log.Printf("DEBUG: Building template for alert %s", alert.Name)
if alert.commandTemplate == nil && alert.Command != nil {
// build template
log.Println("Building template for command...")
alert.commandTemplate = []*template.Template{}
for i, cmdPart := range alert.Command {
alert.commandTemplate = append(alert.commandTemplate, template.Must(
template.New(alert.Name+string(i)).Parse(cmdPart),
))
}
log.Printf("Template built: %v", alert.commandTemplate)
} else if alert.commandShellTemplate == nil && alert.CommandShell != "" {
log.Println("Building template for shell command...")
alert.commandShellTemplate = template.Must(
template.New(alert.Name).Parse(alert.CommandShell),
)
@ -96,7 +93,7 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
var output []byte
output, err = cmd.CombinedOutput()
output_str = string(output)
log.Printf("Check %s\n---\n%s\n---", alert.Name, output_str)
// log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
return output_str, err
}

56
main.go
View File

@ -5,44 +5,56 @@ import (
"time"
)
func main() {
config, err := LoadConfig("config.yml")
if err != nil {
log.Fatalf("Error loading config: %v", err)
}
for {
func checkMonitors(config *Config) {
for _, monitor := range config.Monitors {
if monitor.ShouldCheck() {
_, alertNotice := monitor.Check()
// Should probably consider refactoring everything below here
if alertNotice != nil {
//log.Printf("Recieved an alert notice: %v", alertNotice)
var alerts []string
if alertNotice.IsUp {
alerts = monitor.AlertUp
log.Printf("Alert up: %v", monitor.AlertUp)
} else {
alerts = monitor.AlertDown
log.Printf("Alert down: %v", monitor.AlertDown)
log.Printf("DEBUG: Recieved an alert notice: %v", alertNotice)
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
if alertNames == nil {
// TODO: Should this be a panic? Should this be validated against? Probably
log.Printf("WARNING: Found alert, but no alert mechanisms exist: %v", alertNotice)
}
if alerts == nil {
log.Printf("WARNING: Found alert, but no alert mechanism: %v", alertNotice)
}
for _, alertName := range alerts {
for _, alertName := range alertNames {
if alert, ok := config.Alerts[alertName]; ok {
_, err := alert.Send(*alertNotice)
output, err := alert.Send(*alertNotice)
if err != nil {
panic(err)
log.Printf(
"ERROR: Alert '%s' failed. result=%v: output=%s",
alert.Name,
err,
output,
)
// TODO: Maybe return this error instead of panicking here
log.Fatalf(
"ERROR: Unsuccessfully triggered alert '%s'. "+
"Crashing to avoid false negatives: %v",
alert.Name,
err,
)
}
} else {
log.Printf("WARNING: Could not find alert for %s", alertName)
// TODO: Maybe panic here. Also, probably validate up front
log.Printf("ERROR: Alert with name '%s' not found", alertName)
}
}
}
}
}
}
func main() {
config, err := LoadConfig("config.yml")
if err != nil {
log.Fatalf("Error loading config: %v", err)
}
// Start main loop
for {
checkMonitors(&config)
sleepTime := time.Duration(config.CheckInterval) * time.Second
time.Sleep(sleepTime)

36
monitor.go
View File

@ -56,29 +56,27 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
}
output, err := cmd.CombinedOutput()
//log.Printf("Check %s\n---\n%s\n---", monitor.Name, string(output))
isSuccess := (err == nil)
if err != nil {
log.Printf("ERROR: %v", err)
}
monitor.lastCheck = time.Now()
monitor.lastOutput = string(output)
var alertNotice *AlertNotice
isSuccess := (err == nil)
if isSuccess {
alertNotice = monitor.success()
} else {
alertNotice = monitor.failure()
}
// log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
if err != nil {
log.Printf("DEBUG: Command result: %v", err)
}
log.Printf(
"Check result for %s: %v, %v at %v",
"INFO: %s success=%t, alert=%t",
monitor.Name,
isSuccess,
alertNotice,
monitor.lastCheck,
alertNotice != nil,
)
return isSuccess, alertNotice
@ -89,7 +87,6 @@ func (monitor Monitor) isUp() bool {
}
func (monitor *Monitor) success() (notice *AlertNotice) {
log.Printf("Great success!")
if !monitor.isUp() {
// Alert that we have recovered
notice = monitor.createAlertNotice(true)
@ -102,20 +99,23 @@ func (monitor *Monitor) success() (notice *AlertNotice) {
}
func (monitor *Monitor) failure() (notice *AlertNotice) {
log.Printf("Devastating failure. :(")
monitor.failureCount++
// If we haven't hit the minimum failures, we can exit
if monitor.failureCount < monitor.getAlertAfter() {
log.Printf(
"Have not hit minimum failures. failures: %v alert after: %v",
"DEBUG: %s failed but did not hit minimum failures. "+
"Count: %v alert after: %v",
monitor.Name,
monitor.failureCount,
monitor.getAlertAfter(),
)
return
}
// Take number of failures after minimum
failureCount := (monitor.failureCount - monitor.getAlertAfter())
// Use alert cadence to determine if we should alert
if monitor.AlertEvery > 0 {
// Handle integer number of failures before alerting
if failureCount%monitor.AlertEvery == 0 {
@ -133,6 +133,7 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
}
}
// If we're going to alert, increment count
if notice != nil {
monitor.alertCount++
}
@ -150,6 +151,15 @@ func (monitor Monitor) getAlertAfter() int16 {
}
}
// GetAlertNames gives a list of alert names for a given monitor status
func (monitor Monitor) GetAlertNames(up bool) []string {
if up {
return monitor.AlertUp
} else {
return monitor.AlertDown
}
}
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
// TODO: Maybe add something about recovery status here
return &AlertNotice{

25
monitor_test.go
View File

@ -85,6 +85,31 @@ func TestMonitorIsUp(t *testing.T) {
}
}
// TestMonitorGetAlertNames tests that proper alert names are returned
func TestMonitorGetAlertNames(t *testing.T) {
cases := []struct {
monitor Monitor
up bool
expected []string
name string
}{
{Monitor{}, true, nil, "Empty up"},
{Monitor{}, false, nil, "Empty down"},
{Monitor{AlertUp: []string{"alert"}}, true, []string{"alert"}, "Return up"},
{Monitor{AlertDown: []string{"alert"}}, false, []string{"alert"}, "Return down"},
}
for _, c := range cases {
log.Printf("Testing case %s", c.name)
actual := c.monitor.GetAlertNames(c.up)
if !EqualSliceString(actual, c.expected) {
t.Errorf("GetAlertNames(%v), expected=%v actual=%v", c.name, c.expected, actual)
log.Printf("Case failed: %s", c.name)
}
log.Println("-----")
}
}
// TestMonitorSuccess tests the Monitor.success()
func TestMonitorSuccess(t *testing.T) {
cases := []struct {

13
util.go
View File

@ -21,3 +21,16 @@ func ShellCommand(command string) *exec.Cmd {
//log.Printf("Shell command: %v", shellCommand)
return exec.Command(shellCommand[0], shellCommand[1:]...)
}
// EqualSliceString checks if two string slices are equivalent
func EqualSliceString(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, val := range a {
if val != b[i] {
return false
}
}
return true
}

32
util_test.go Normal file
View File

@ -0,0 +1,32 @@
package main
import "testing"
func TestUtilEqualSliceString(t *testing.T) {
cases := []struct {
a, b []string
expected bool
}{
// Equal cases
{nil, nil, true},
{nil, []string{}, true},
{[]string{}, nil, true},
{[]string{"a"}, []string{"a"}, true},
// Inequal cases
{nil, []string{"b"}, false},
{[]string{"a"}, nil, false},
{[]string{"a"}, []string{"b"}, false},
{[]string{"a"}, []string{"a", "b"}, false},
{[]string{"a", "b"}, []string{"b"}, false},
}
for _, c := range cases {
actual := EqualSliceString(c.a, c.b)
if actual != c.expected {
t.Errorf(
"EqualSliceString(%v, %v), expected=%v actual=%v",
c.a, c.b, c.expected, actual,
)
}
}
}