2019-09-21 22:03:26 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"log"
|
2019-10-02 16:37:29 +00:00
|
|
|
"math"
|
2019-09-21 22:03:26 +00:00
|
|
|
"os/exec"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
// Monitor represents a particular periodic check of a command
|
2019-09-21 22:03:26 +00:00
|
|
|
type Monitor struct {
|
|
|
|
// Config values
|
|
|
|
Name string
|
2020-02-16 21:25:11 +00:00
|
|
|
Command CommandOrShell
|
2019-09-21 22:03:26 +00:00
|
|
|
AlertDown []string `yaml:"alert_down"`
|
|
|
|
AlertUp []string `yaml:"alert_up"`
|
|
|
|
CheckInterval float64 `yaml:"check_interval"`
|
|
|
|
AlertAfter int16 `yaml:"alert_after"`
|
2019-10-02 16:37:29 +00:00
|
|
|
AlertEvery int16 `yaml:"alert_every"`
|
2019-09-21 22:03:26 +00:00
|
|
|
// Other values
|
2019-10-02 16:37:29 +00:00
|
|
|
lastCheck time.Time
|
|
|
|
lastOutput string
|
|
|
|
alertCount int16
|
|
|
|
failureCount int16
|
|
|
|
lastSuccess time.Time
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
// IsValid returns a boolean indicating if the Monitor has been correctly
|
|
|
|
// configured
|
2019-09-21 22:03:26 +00:00
|
|
|
func (monitor Monitor) IsValid() bool {
|
2020-02-16 21:25:11 +00:00
|
|
|
return (!monitor.Command.Empty() &&
|
2019-10-04 23:23:48 +00:00
|
|
|
monitor.getAlertAfter() > 0 &&
|
|
|
|
monitor.AlertDown != nil)
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
// ShouldCheck returns a boolean indicating if the Monitor is ready to be
|
|
|
|
// be checked again
|
2019-09-21 22:03:26 +00:00
|
|
|
func (monitor Monitor) ShouldCheck() bool {
|
2019-10-02 16:37:29 +00:00
|
|
|
if monitor.lastCheck.IsZero() {
|
2019-09-21 22:03:26 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
sinceLastCheck := time.Now().Sub(monitor.lastCheck).Seconds()
|
2019-09-21 22:03:26 +00:00
|
|
|
return sinceLastCheck >= monitor.CheckInterval
|
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
// Check will run the command configured by the Monitor and return a status
|
|
|
|
// and a possible AlertNotice
|
|
|
|
func (monitor *Monitor) Check() (bool, *AlertNotice) {
|
2019-09-21 22:03:26 +00:00
|
|
|
var cmd *exec.Cmd
|
2020-02-16 21:25:11 +00:00
|
|
|
if monitor.Command.Command != nil {
|
|
|
|
cmd = exec.Command(monitor.Command.Command[0], monitor.Command.Command[1:]...)
|
2019-09-21 22:03:26 +00:00
|
|
|
} else {
|
2020-02-16 21:25:11 +00:00
|
|
|
cmd = ShellCommand(monitor.Command.ShellCommand)
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
output, err := cmd.CombinedOutput()
|
2019-10-02 16:37:29 +00:00
|
|
|
monitor.lastCheck = time.Now()
|
|
|
|
monitor.lastOutput = string(output)
|
2019-09-21 22:03:26 +00:00
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
var alertNotice *AlertNotice
|
2019-10-04 21:47:38 +00:00
|
|
|
isSuccess := (err == nil)
|
2019-10-02 16:37:29 +00:00
|
|
|
if isSuccess {
|
|
|
|
alertNotice = monitor.success()
|
2019-09-21 22:03:26 +00:00
|
|
|
} else {
|
2019-10-02 16:37:29 +00:00
|
|
|
alertNotice = monitor.failure()
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-04 23:05:25 +00:00
|
|
|
if LogDebug {
|
|
|
|
log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
|
|
|
|
}
|
2019-10-04 21:47:38 +00:00
|
|
|
if err != nil {
|
2019-10-04 23:05:25 +00:00
|
|
|
if LogDebug {
|
|
|
|
log.Printf("DEBUG: Command result: %v", err)
|
|
|
|
}
|
2019-10-04 21:47:38 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
log.Printf(
|
2019-10-04 21:47:38 +00:00
|
|
|
"INFO: %s success=%t, alert=%t",
|
2019-10-02 16:37:29 +00:00
|
|
|
monitor.Name,
|
|
|
|
isSuccess,
|
2019-10-04 21:47:38 +00:00
|
|
|
alertNotice != nil,
|
2019-10-02 16:37:29 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
return isSuccess, alertNotice
|
|
|
|
}
|
|
|
|
|
|
|
|
func (monitor Monitor) isUp() bool {
|
|
|
|
return monitor.alertCount == 0
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
func (monitor *Monitor) success() (notice *AlertNotice) {
|
|
|
|
if !monitor.isUp() {
|
|
|
|
// Alert that we have recovered
|
|
|
|
notice = monitor.createAlertNotice(true)
|
|
|
|
}
|
|
|
|
monitor.failureCount = 0
|
|
|
|
monitor.alertCount = 0
|
|
|
|
monitor.lastSuccess = time.Now()
|
|
|
|
|
|
|
|
return
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
func (monitor *Monitor) failure() (notice *AlertNotice) {
|
|
|
|
monitor.failureCount++
|
|
|
|
// If we haven't hit the minimum failures, we can exit
|
2019-10-03 03:52:57 +00:00
|
|
|
if monitor.failureCount < monitor.getAlertAfter() {
|
2019-10-04 23:05:25 +00:00
|
|
|
if LogDebug {
|
|
|
|
log.Printf(
|
|
|
|
"DEBUG: %s failed but did not hit minimum failures. "+
|
|
|
|
"Count: %v alert after: %v",
|
|
|
|
monitor.Name,
|
|
|
|
monitor.failureCount,
|
|
|
|
monitor.getAlertAfter(),
|
|
|
|
)
|
|
|
|
}
|
2019-10-02 16:37:29 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-10-04 21:47:38 +00:00
|
|
|
// Take number of failures after minimum
|
2019-10-03 03:52:57 +00:00
|
|
|
failureCount := (monitor.failureCount - monitor.getAlertAfter())
|
2019-10-02 16:37:29 +00:00
|
|
|
|
2019-10-04 21:47:38 +00:00
|
|
|
// Use alert cadence to determine if we should alert
|
2019-10-02 16:37:29 +00:00
|
|
|
if monitor.AlertEvery > 0 {
|
|
|
|
// Handle integer number of failures before alerting
|
|
|
|
if failureCount%monitor.AlertEvery == 0 {
|
|
|
|
notice = monitor.createAlertNotice(false)
|
|
|
|
}
|
|
|
|
} else if monitor.AlertEvery == 0 {
|
|
|
|
// Handle alerting on first failure only
|
2019-10-03 03:52:57 +00:00
|
|
|
if failureCount == 0 {
|
2019-10-02 16:37:29 +00:00
|
|
|
notice = monitor.createAlertNotice(false)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Handle negative numbers indicating an exponential backoff
|
|
|
|
if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) {
|
|
|
|
notice = monitor.createAlertNotice(false)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-10-04 21:47:38 +00:00
|
|
|
// If we're going to alert, increment count
|
2019-10-02 16:37:29 +00:00
|
|
|
if notice != nil {
|
|
|
|
monitor.alertCount++
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-10-03 03:52:57 +00:00
|
|
|
func (monitor Monitor) getAlertAfter() int16 {
|
|
|
|
// TODO: Come up with a better way than this method
|
|
|
|
// Zero is one!
|
|
|
|
if monitor.AlertAfter == 0 {
|
|
|
|
return 1
|
|
|
|
}
|
2020-01-07 18:28:14 +00:00
|
|
|
return monitor.AlertAfter
|
2019-10-03 03:52:57 +00:00
|
|
|
}
|
|
|
|
|
2019-10-04 21:47:38 +00:00
|
|
|
// GetAlertNames gives a list of alert names for a given monitor status
|
|
|
|
func (monitor Monitor) GetAlertNames(up bool) []string {
|
|
|
|
if up {
|
|
|
|
return monitor.AlertUp
|
|
|
|
}
|
2020-01-07 18:28:14 +00:00
|
|
|
return monitor.AlertDown
|
2019-10-04 21:47:38 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
|
|
|
|
// TODO: Maybe add something about recovery status here
|
|
|
|
return &AlertNotice{
|
|
|
|
MonitorName: monitor.Name,
|
|
|
|
AlertCount: monitor.alertCount,
|
|
|
|
FailureCount: monitor.failureCount,
|
|
|
|
LastCheckOutput: monitor.lastOutput,
|
|
|
|
LastSuccess: monitor.lastSuccess,
|
|
|
|
IsUp: isUp,
|
|
|
|
}
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|