minitor-go/main.go

168 lines
4.4 KiB
Go
Raw Normal View History

2019-09-21 22:03:26 +00:00
package main
import (
2021-05-11 04:00:58 +00:00
"errors"
"flag"
"fmt"
"strings"
2019-09-21 22:03:26 +00:00
"time"
2021-05-11 03:12:18 +00:00
"git.iamthefij.com/iamthefij/slog"
2019-09-21 22:03:26 +00:00
)
2019-10-04 23:17:20 +00:00
var (
// ExportMetrics will track whether or not we want to export metrics to prometheus
ExportMetrics = false
// MetricsPort is the port to expose metrics on
MetricsPort = 8080
// Metrics contains all active metrics
Metrics = NewMetrics()
2020-02-20 01:31:04 +00:00
// PyCompat enables support for legacy Python templates
PyCompat = false
2019-10-04 23:17:20 +00:00
// version of minitor being run
version = "dev"
2021-05-11 04:00:58 +00:00
errUnknownAlert = errors.New("unknown alert")
2019-10-04 23:17:20 +00:00
)
2021-05-11 04:00:58 +00:00
func sendAlerts(config *Config, monitor *Monitor, alertNotice *AlertNotice) error {
slog.Debugf("Received an alert notice from %s", alertNotice.MonitorName)
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
if alertNames == nil {
// This should only happen for a recovery alert. AlertDown is validated not empty
slog.Warningf(
"Received alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
alertNotice.MonitorName, alertNotice.IsUp,
)
2021-09-02 17:19:03 +00:00
return nil
2021-05-11 04:00:58 +00:00
}
for _, alertName := range alertNames {
if alert, ok := config.Alerts[alertName]; ok {
output, err := alert.Send(*alertNotice)
if err != nil {
slog.Errorf(
"Alert '%s' failed. result=%v: output=%s",
alert.Name,
err,
output,
)
return err
}
// Count alert metrics
Metrics.CountAlert(monitor.Name, alert.Name)
} else {
// This case should never actually happen since we validate against it
slog.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
return fmt.Errorf("unknown alert for monitor %s: %s: %w", alertNotice.MonitorName, alertName, errUnknownAlert)
}
}
return nil
}
2019-10-04 23:17:36 +00:00
func checkMonitors(config *Config) error {
2021-09-02 17:19:03 +00:00
// TODO: Run this in goroutines and capture exceptions
for _, monitor := range config.Monitors {
if monitor.ShouldCheck() {
success, alertNotice := monitor.Check()
hasAlert := alertNotice != nil
// Track status metrics
Metrics.SetMonitorStatus(monitor.Name, monitor.IsUp())
2021-05-11 17:41:22 +00:00
Metrics.CountCheck(monitor.Name, success, monitor.LastCheckMilliseconds(), hasAlert)
if alertNotice != nil {
err := sendAlerts(config, monitor, alertNotice)
// If there was an error in sending an alert, exit early and bubble it up
if err != nil {
return err
}
}
}
}
2019-10-04 23:17:36 +00:00
return nil
}
func sendStartupAlerts(config *Config, alertNames []string) error {
for _, alertName := range alertNames {
var err error
alert, ok := config.Alerts[alertName]
if !ok {
err = fmt.Errorf("unknown alert %s: %w", alertName, errUnknownAlert)
}
if err == nil {
_, err = alert.Send(AlertNotice{
AlertCount: 0,
FailureCount: 0,
IsUp: true,
LastSuccess: time.Now(),
MonitorName: fmt.Sprintf("First Run Alert Test: %s", alert.Name),
LastCheckOutput: "",
})
}
if err != nil {
return err
}
}
return nil
}
2019-09-21 22:03:26 +00:00
func main() {
2021-05-11 03:12:18 +00:00
showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
2021-05-11 03:12:18 +00:00
flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
2020-02-20 01:31:04 +00:00
flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
2021-09-02 17:19:03 +00:00
flag.IntVar(&MetricsPort, "metrics-port", MetricsPort, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
flag.Parse()
2019-10-04 23:17:20 +00:00
// Print version if flag is provided
if *showVersion {
2021-05-11 03:12:18 +00:00
fmt.Println("Minitor version:", version)
2019-10-04 23:17:20 +00:00
return
}
// Load configuration
config, err := LoadConfig(*configPath)
2021-05-11 03:12:18 +00:00
slog.OnErrFatalf(err, "Error loading config: %v", err)
2019-09-21 22:03:26 +00:00
// Serve metrics exporter, if specified
if ExportMetrics {
2021-05-11 17:41:22 +00:00
slog.Infof("Exporting metrics to Prometheus on port %d", MetricsPort)
2021-05-11 03:12:18 +00:00
go ServeMetrics()
}
if *startupAlerts != "" {
alertNames := strings.Split(*startupAlerts, ",")
err = sendStartupAlerts(&config, alertNames)
slog.OnErrPanicf(err, "Error running startup alerts")
}
// Start main loop
2019-09-21 22:03:26 +00:00
for {
2019-10-04 23:17:36 +00:00
err = checkMonitors(&config)
2021-09-02 17:19:03 +00:00
slog.OnErrPanicf(err, "Error checking monitors")
2019-09-21 22:03:26 +00:00
time.Sleep(config.CheckInterval.Value())
2019-09-21 22:03:26 +00:00
}
}