package main import ( "math" "os/exec" "time" "git.iamthefij.com/iamthefij/slog" ) // Monitor represents a particular periodic check of a command type Monitor struct { //nolint:maligned // Config values AlertAfter int16 `yaml:"alert_after"` AlertEvery *int16 `yaml:"alert_every"` CheckInterval time.Duration `yaml:"check_interval"` Name string AlertDown []string `yaml:"alert_down"` AlertUp []string `yaml:"alert_up"` Command CommandOrShell // Other values alertCount int16 failureCount int16 lastCheck time.Time lastSuccess time.Time lastOutput string lastCheckDuration time.Duration } // IsValid returns a boolean indicating if the Monitor has been correctly // configured func (monitor Monitor) IsValid() bool { return (!monitor.Command.Empty() && monitor.getAlertAfter() > 0 && monitor.AlertDown != nil) } // ShouldCheck returns a boolean indicating if the Monitor is ready to be // be checked again func (monitor Monitor) ShouldCheck() bool { if monitor.lastCheck.IsZero() { return true } sinceLastCheck := time.Since(monitor.lastCheck) return sinceLastCheck >= monitor.CheckInterval } // Check will run the command configured by the Monitor and return a status // and a possible AlertNotice func (monitor *Monitor) Check() (bool, *AlertNotice) { var cmd *exec.Cmd if monitor.Command.Command != nil { cmd = exec.Command(monitor.Command.Command[0], monitor.Command.Command[1:]...) } else { cmd = ShellCommand(monitor.Command.ShellCommand) } checkStartTime := time.Now() output, err := cmd.CombinedOutput() monitor.lastCheck = time.Now() monitor.lastOutput = string(output) monitor.lastCheckDuration = monitor.lastCheck.Sub(checkStartTime) var alertNotice *AlertNotice isSuccess := (err == nil) if isSuccess { alertNotice = monitor.success() } else { alertNotice = monitor.failure() } slog.Debugf("Command output: %s", monitor.lastOutput) slog.OnErrWarnf(err, "Command result: %v", err) slog.Infof( "%s success=%t, alert=%t", monitor.Name, isSuccess, alertNotice != nil, ) return isSuccess, alertNotice } // IsUp returns the status of the current monitor func (monitor Monitor) IsUp() bool { return monitor.alertCount == 0 } // LastCheckMilliseconds gives number of miliseconds the last check ran for func (monitor Monitor) LastCheckMilliseconds() int64 { return monitor.lastCheckDuration.Milliseconds() } func (monitor *Monitor) success() (notice *AlertNotice) { if !monitor.IsUp() { // Alert that we have recovered notice = monitor.createAlertNotice(true) } monitor.failureCount = 0 monitor.alertCount = 0 monitor.lastSuccess = time.Now() return } func (monitor *Monitor) failure() (notice *AlertNotice) { monitor.failureCount++ // If we haven't hit the minimum failures, we can exit if monitor.failureCount < monitor.getAlertAfter() { slog.Debugf( "%s failed but did not hit minimum failures. "+ "Count: %v alert after: %v", monitor.Name, monitor.failureCount, monitor.getAlertAfter(), ) return } // Take number of failures after minimum failureCount := (monitor.failureCount - monitor.getAlertAfter()) // Use alert cadence to determine if we should alert switch { case monitor.AlertEvery == nil, *monitor.AlertEvery == 0: // Handle alerting on first failure only if failureCount == 0 { notice = monitor.createAlertNotice(false) } case *monitor.AlertEvery > 0: // Handle integer number of failures before alerting if failureCount%*monitor.AlertEvery == 0 { notice = monitor.createAlertNotice(false) } default: // Handle negative numbers indicating an exponential backoff if failureCount >= int16(math.Pow(2, float64(monitor.alertCount))-1) { //nolint:gomnd notice = monitor.createAlertNotice(false) } } // If we're going to alert, increment count if notice != nil { monitor.alertCount++ } return notice } func (monitor Monitor) getAlertAfter() int16 { // TODO: Come up with a better way than this method // Zero is one! if monitor.AlertAfter == 0 { return 1 } return monitor.AlertAfter } // GetAlertNames gives a list of alert names for a given monitor status func (monitor Monitor) GetAlertNames(up bool) []string { if up { return monitor.AlertUp } return monitor.AlertDown } func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice { // TODO: Maybe add something about recovery status here return &AlertNotice{ MonitorName: monitor.Name, AlertCount: monitor.alertCount, FailureCount: monitor.failureCount, LastCheckOutput: monitor.lastOutput, LastSuccess: monitor.lastSuccess, IsUp: isUp, } }