minitor-go/monitor.go

187 lines
4.6 KiB
Go
Raw Normal View History

2019-09-21 22:03:26 +00:00
package main
import (
2019-10-02 16:37:29 +00:00
"math"
2019-09-21 22:03:26 +00:00
"os/exec"
"time"
2021-05-11 03:12:18 +00:00
"git.iamthefij.com/iamthefij/slog"
2019-09-21 22:03:26 +00:00
)
2019-10-02 16:37:29 +00:00
// Monitor represents a particular periodic check of a command
2021-05-11 04:00:58 +00:00
type Monitor struct { //nolint:maligned
2019-09-21 22:03:26 +00:00
// Config values
AlertAfter int16 `yaml:"alert_after"`
AlertEvery int16 `yaml:"alert_every"`
CheckInterval SecondsOrDuration `yaml:"check_interval"`
2019-09-21 22:03:26 +00:00
Name string
AlertDown []string `yaml:"alert_down"`
AlertUp []string `yaml:"alert_up"`
2021-05-11 04:00:58 +00:00
Command CommandOrShell
2019-09-21 22:03:26 +00:00
// Other values
2021-05-11 17:41:22 +00:00
alertCount int16
failureCount int16
lastCheck time.Time
lastSuccess time.Time
lastOutput string
lastCheckDuration time.Duration
2019-09-21 22:03:26 +00:00
}
2019-10-02 16:37:29 +00:00
// IsValid returns a boolean indicating if the Monitor has been correctly
// configured
2019-09-21 22:03:26 +00:00
func (monitor Monitor) IsValid() bool {
return (!monitor.Command.Empty() &&
2019-10-04 23:23:48 +00:00
monitor.getAlertAfter() > 0 &&
monitor.AlertDown != nil)
2019-09-21 22:03:26 +00:00
}
2019-10-02 16:37:29 +00:00
// ShouldCheck returns a boolean indicating if the Monitor is ready to be
// be checked again
2019-09-21 22:03:26 +00:00
func (monitor Monitor) ShouldCheck() bool {
2019-10-02 16:37:29 +00:00
if monitor.lastCheck.IsZero() {
2019-09-21 22:03:26 +00:00
return true
}
2021-05-11 04:39:52 +00:00
sinceLastCheck := time.Since(monitor.lastCheck)
2021-05-11 04:00:58 +00:00
return sinceLastCheck >= monitor.CheckInterval.Value()
2019-09-21 22:03:26 +00:00
}
2019-10-02 16:37:29 +00:00
// Check will run the command configured by the Monitor and return a status
// and a possible AlertNotice
func (monitor *Monitor) Check() (bool, *AlertNotice) {
2019-09-21 22:03:26 +00:00
var cmd *exec.Cmd
if monitor.Command.Command != nil {
cmd = exec.Command(monitor.Command.Command[0], monitor.Command.Command[1:]...)
2019-09-21 22:03:26 +00:00
} else {
cmd = ShellCommand(monitor.Command.ShellCommand)
2019-09-21 22:03:26 +00:00
}
2021-05-11 17:41:22 +00:00
checkStartTime := time.Now()
2019-09-21 22:03:26 +00:00
output, err := cmd.CombinedOutput()
2019-10-02 16:37:29 +00:00
monitor.lastCheck = time.Now()
monitor.lastOutput = string(output)
2021-05-11 17:41:22 +00:00
monitor.lastCheckDuration = monitor.lastCheck.Sub(checkStartTime)
2019-09-21 22:03:26 +00:00
2019-10-02 16:37:29 +00:00
var alertNotice *AlertNotice
2021-05-11 04:00:58 +00:00
isSuccess := (err == nil)
2019-10-02 16:37:29 +00:00
if isSuccess {
alertNotice = monitor.success()
2019-09-21 22:03:26 +00:00
} else {
2019-10-02 16:37:29 +00:00
alertNotice = monitor.failure()
2019-09-21 22:03:26 +00:00
}
2021-05-11 03:12:18 +00:00
slog.Debugf("Command output: %s", monitor.lastOutput)
slog.OnErrWarnf(err, "Command result: %v", err)
2021-05-11 03:12:18 +00:00
slog.Infof(
"%s success=%t, alert=%t",
2019-10-02 16:37:29 +00:00
monitor.Name,
isSuccess,
alertNotice != nil,
2019-10-02 16:37:29 +00:00
)
return isSuccess, alertNotice
}
// IsUp returns the status of the current monitor
func (monitor Monitor) IsUp() bool {
2019-10-02 16:37:29 +00:00
return monitor.alertCount == 0
2019-09-21 22:03:26 +00:00
}
2021-05-11 17:41:22 +00:00
// LastCheckMilliseconds gives number of miliseconds the last check ran for
func (monitor Monitor) LastCheckMilliseconds() int64 {
return monitor.lastCheckDuration.Milliseconds()
}
2019-10-02 16:37:29 +00:00
func (monitor *Monitor) success() (notice *AlertNotice) {
if !monitor.IsUp() {
2019-10-02 16:37:29 +00:00
// Alert that we have recovered
notice = monitor.createAlertNotice(true)
}
2021-05-11 04:00:58 +00:00
2019-10-02 16:37:29 +00:00
monitor.failureCount = 0
monitor.alertCount = 0
monitor.lastSuccess = time.Now()
return
2019-09-21 22:03:26 +00:00
}
2019-10-02 16:37:29 +00:00
func (monitor *Monitor) failure() (notice *AlertNotice) {
monitor.failureCount++
// If we haven't hit the minimum failures, we can exit
if monitor.failureCount < monitor.getAlertAfter() {
2021-05-11 03:12:18 +00:00
slog.Debugf(
"%s failed but did not hit minimum failures. "+
"Count: %v alert after: %v",
monitor.Name,
monitor.failureCount,
monitor.getAlertAfter(),
)
2019-10-02 16:37:29 +00:00
return
}
// Take number of failures after minimum
failureCount := (monitor.failureCount - monitor.getAlertAfter())
2019-10-02 16:37:29 +00:00
// Use alert cadence to determine if we should alert
2021-05-11 04:00:58 +00:00
switch {
case monitor.AlertEvery > 0:
2019-10-02 16:37:29 +00:00
// Handle integer number of failures before alerting
if failureCount%monitor.AlertEvery == 0 {
notice = monitor.createAlertNotice(false)
}
2021-05-11 04:00:58 +00:00
case monitor.AlertEvery == 0:
2019-10-02 16:37:29 +00:00
// Handle alerting on first failure only
if failureCount == 0 {
2019-10-02 16:37:29 +00:00
notice = monitor.createAlertNotice(false)
}
2021-05-11 04:00:58 +00:00
default:
2019-10-02 16:37:29 +00:00
// Handle negative numbers indicating an exponential backoff
2021-05-11 04:00:58 +00:00
if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) { //nolint:gomnd
2019-10-02 16:37:29 +00:00
notice = monitor.createAlertNotice(false)
}
}
// If we're going to alert, increment count
2019-10-02 16:37:29 +00:00
if notice != nil {
monitor.alertCount++
}
2021-05-11 04:00:58 +00:00
return notice
2019-10-02 16:37:29 +00:00
}
func (monitor Monitor) getAlertAfter() int16 {
// TODO: Come up with a better way than this method
// Zero is one!
if monitor.AlertAfter == 0 {
return 1
}
2021-05-11 04:00:58 +00:00
return monitor.AlertAfter
}
// GetAlertNames gives a list of alert names for a given monitor status
func (monitor Monitor) GetAlertNames(up bool) []string {
if up {
return monitor.AlertUp
}
2021-05-11 04:00:58 +00:00
return monitor.AlertDown
}
2019-10-02 16:37:29 +00:00
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {
// TODO: Maybe add something about recovery status here
return &AlertNotice{
MonitorName: monitor.Name,
AlertCount: monitor.alertCount,
FailureCount: monitor.failureCount,
LastCheckOutput: monitor.lastOutput,
LastSuccess: monitor.lastSuccess,
IsUp: isUp,
}
2019-09-21 22:03:26 +00:00
}