Replace log with slog
This commit is contained in:
parent
f0e179851f
commit
fda9e1bfc3
@ -149,7 +149,6 @@ For the time being, legacy configs for the Python version of Minitor should be c
|
|||||||
|
|
||||||
Future, potentially breaking changes
|
Future, potentially breaking changes
|
||||||
|
|
||||||
- [ ] Implement leveled logging (maybe glog or logrus)
|
|
||||||
- [ ] Consider value of templating vs injecting values into Env variables
|
- [ ] Consider value of templating vs injecting values into Env variables
|
||||||
- [ ] Async checking
|
- [ ] Async checking
|
||||||
- [ ] Revisit metrics and see if they all make sense
|
- [ ] Revisit metrics and see if they all make sense
|
||||||
|
16
alert.go
16
alert.go
@ -3,11 +3,12 @@ package main
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"strings"
|
"strings"
|
||||||
"text/template"
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.iamthefij.com/iamthefij/slog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Alert is a config driven mechanism for sending a notice
|
// Alert is a config driven mechanism for sending a notice
|
||||||
@ -45,9 +46,9 @@ func (alert *Alert) BuildTemplates() error {
|
|||||||
"{last_success}", "{{.LastSuccess}}",
|
"{last_success}", "{{.LastSuccess}}",
|
||||||
"{monitor_name}", "{{.MonitorName}}",
|
"{monitor_name}", "{{.MonitorName}}",
|
||||||
)
|
)
|
||||||
if LogDebug {
|
|
||||||
log.Printf("DEBUG: Building template for alert %s", alert.Name)
|
slog.Debugf("Building template for alert %s", alert.Name)
|
||||||
}
|
|
||||||
if alert.commandTemplate == nil && alert.Command.Command != nil {
|
if alert.commandTemplate == nil && alert.Command.Command != nil {
|
||||||
alert.commandTemplate = []*template.Template{}
|
alert.commandTemplate = []*template.Template{}
|
||||||
for i, cmdPart := range alert.Command.Command {
|
for i, cmdPart := range alert.Command.Command {
|
||||||
@ -75,7 +76,8 @@ func (alert *Alert) BuildTemplates() error {
|
|||||||
|
|
||||||
// Send will send an alert notice by executing the command template
|
// Send will send an alert notice by executing the command template
|
||||||
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
||||||
log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName)
|
slog.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
|
||||||
|
|
||||||
var cmd *exec.Cmd
|
var cmd *exec.Cmd
|
||||||
if alert.commandTemplate != nil {
|
if alert.commandTemplate != nil {
|
||||||
command := []string{}
|
command := []string{}
|
||||||
@ -110,9 +112,7 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
|||||||
var output []byte
|
var output []byte
|
||||||
output, err = cmd.CombinedOutput()
|
output, err = cmd.CombinedOutput()
|
||||||
outputStr = string(output)
|
outputStr = string(output)
|
||||||
if LogDebug {
|
slog.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
|
||||||
log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
|
|
||||||
}
|
|
||||||
|
|
||||||
return outputStr, err
|
return outputStr, err
|
||||||
}
|
}
|
||||||
|
22
config.go
22
config.go
@ -3,8 +3,8 @@ package main
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
|
||||||
|
|
||||||
|
"git.iamthefij.com/iamthefij/slog"
|
||||||
"gopkg.in/yaml.v2"
|
"gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -53,32 +53,36 @@ func (config Config) IsValid() (isValid bool) {
|
|||||||
// Validate alerts
|
// Validate alerts
|
||||||
if config.Alerts == nil || len(config.Alerts) == 0 {
|
if config.Alerts == nil || len(config.Alerts) == 0 {
|
||||||
// This should never happen because there is a default alert named 'log' for now
|
// This should never happen because there is a default alert named 'log' for now
|
||||||
log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert")
|
slog.Errorf("Invalid alert configuration: Must provide at least one alert")
|
||||||
|
|
||||||
isValid = false
|
isValid = false
|
||||||
}
|
}
|
||||||
for _, alert := range config.Alerts {
|
for _, alert := range config.Alerts {
|
||||||
if !alert.IsValid() {
|
if !alert.IsValid() {
|
||||||
log.Printf("ERROR: Invalid alert configuration: %s", alert.Name)
|
slog.Errorf("Invalid alert configuration: %s", alert.Name)
|
||||||
|
|
||||||
isValid = false
|
isValid = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate monitors
|
// Validate monitors
|
||||||
if config.Monitors == nil || len(config.Monitors) == 0 {
|
if config.Monitors == nil || len(config.Monitors) == 0 {
|
||||||
log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor")
|
slog.Errorf("Invalid monitor configuration: Must provide at least one monitor")
|
||||||
|
|
||||||
isValid = false
|
isValid = false
|
||||||
}
|
}
|
||||||
for _, monitor := range config.Monitors {
|
for _, monitor := range config.Monitors {
|
||||||
if !monitor.IsValid() {
|
if !monitor.IsValid() {
|
||||||
log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
|
slog.Errorf("Invalid monitor configuration: %s", monitor.Name)
|
||||||
|
|
||||||
isValid = false
|
isValid = false
|
||||||
}
|
}
|
||||||
// Check that all Monitor alerts actually exist
|
// Check that all Monitor alerts actually exist
|
||||||
for _, isUp := range []bool{true, false} {
|
for _, isUp := range []bool{true, false} {
|
||||||
for _, alertName := range monitor.GetAlertNames(isUp) {
|
for _, alertName := range monitor.GetAlertNames(isUp) {
|
||||||
if _, ok := config.Alerts[alertName]; !ok {
|
if _, ok := config.Alerts[alertName]; !ok {
|
||||||
log.Printf(
|
slog.Errorf(
|
||||||
"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
|
"Invalid monitor configuration: %s. Unknown alert %s",
|
||||||
monitor.Name, alertName,
|
monitor.Name, alertName,
|
||||||
)
|
)
|
||||||
isValid = false
|
isValid = false
|
||||||
@ -114,9 +118,7 @@ func LoadConfig(filePath string) (config Config, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if LogDebug {
|
slog.Debugf("Config values:\n%v\n", config)
|
||||||
log.Printf("DEBUG: Config values:\n%v\n", config)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add log alert if not present
|
// Add log alert if not present
|
||||||
if PyCompat {
|
if PyCompat {
|
||||||
|
1
go.mod
1
go.mod
@ -3,6 +3,7 @@ module git.iamthefij.com/iamthefij/minitor-go
|
|||||||
go 1.15
|
go 1.15
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
git.iamthefij.com/iamthefij/slog v1.3.0
|
||||||
github.com/prometheus/client_golang v1.2.1
|
github.com/prometheus/client_golang v1.2.1
|
||||||
gopkg.in/yaml.v2 v2.2.4
|
gopkg.in/yaml.v2 v2.2.4
|
||||||
)
|
)
|
||||||
|
2
go.sum
2
go.sum
@ -1,3 +1,5 @@
|
|||||||
|
git.iamthefij.com/iamthefij/slog v1.3.0 h1:4Hu5PQvDrW5e3FrTS3q2iIXW0iPvhNY/9qJsqDR3K3I=
|
||||||
|
git.iamthefij.com/iamthefij/slog v1.3.0/go.mod h1:1RUj4hcCompZkAxXCRfUX786tb3cM/Zpkn97dGfUfbg=
|
||||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||||
|
39
main.go
39
main.go
@ -3,14 +3,12 @@ package main
|
|||||||
import (
|
import (
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.iamthefij.com/iamthefij/slog"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// LogDebug will control whether debug messsages should be logged
|
|
||||||
LogDebug = false
|
|
||||||
|
|
||||||
// ExportMetrics will track whether or not we want to export metrics to prometheus
|
// ExportMetrics will track whether or not we want to export metrics to prometheus
|
||||||
ExportMetrics = false
|
ExportMetrics = false
|
||||||
// MetricsPort is the port to expose metrics on
|
// MetricsPort is the port to expose metrics on
|
||||||
@ -38,14 +36,12 @@ func checkMonitors(config *Config) error {
|
|||||||
|
|
||||||
// Should probably consider refactoring everything below here
|
// Should probably consider refactoring everything below here
|
||||||
if alertNotice != nil {
|
if alertNotice != nil {
|
||||||
if LogDebug {
|
slog.Debugf("Received an alert notice from %s", alertNotice.MonitorName)
|
||||||
log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
|
|
||||||
}
|
|
||||||
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
|
alertNames := monitor.GetAlertNames(alertNotice.IsUp)
|
||||||
if alertNames == nil {
|
if alertNames == nil {
|
||||||
// This should only happen for a recovery alert. AlertDown is validated not empty
|
// This should only happen for a recovery alert. AlertDown is validated not empty
|
||||||
log.Printf(
|
slog.Warningf(
|
||||||
"WARNING: Recieved alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
|
"Received alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
|
||||||
alertNotice.MonitorName, alertNotice.IsUp,
|
alertNotice.MonitorName, alertNotice.IsUp,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@ -53,8 +49,8 @@ func checkMonitors(config *Config) error {
|
|||||||
if alert, ok := config.Alerts[alertName]; ok {
|
if alert, ok := config.Alerts[alertName]; ok {
|
||||||
output, err := alert.Send(*alertNotice)
|
output, err := alert.Send(*alertNotice)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf(
|
slog.Errorf(
|
||||||
"ERROR: Alert '%s' failed. result=%v: output=%s",
|
"Alert '%s' failed. result=%v: output=%s",
|
||||||
alert.Name,
|
alert.Name,
|
||||||
err,
|
err,
|
||||||
output,
|
output,
|
||||||
@ -71,7 +67,8 @@ func checkMonitors(config *Config) error {
|
|||||||
Metrics.CountAlert(monitor.Name, alert.Name)
|
Metrics.CountAlert(monitor.Name, alert.Name)
|
||||||
} else {
|
} else {
|
||||||
// This case should never actually happen since we validate against it
|
// This case should never actually happen since we validate against it
|
||||||
log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
slog.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
||||||
|
|
||||||
return fmt.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
return fmt.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -83,30 +80,30 @@ func checkMonitors(config *Config) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
// Get debug flag
|
showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
|
||||||
flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
|
configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
|
||||||
|
|
||||||
|
flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
|
||||||
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
|
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
|
||||||
flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
|
flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
|
||||||
flag.IntVar(&MetricsPort, "metrics-port", 8080, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
|
flag.IntVar(&MetricsPort, "metrics-port", 8080, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
|
||||||
var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
|
|
||||||
var configPath = flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// Print version if flag is provided
|
// Print version if flag is provided
|
||||||
if *showVersion {
|
if *showVersion {
|
||||||
log.Println("Minitor version:", version)
|
fmt.Println("Minitor version:", version)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load configuration
|
// Load configuration
|
||||||
config, err := LoadConfig(*configPath)
|
config, err := LoadConfig(*configPath)
|
||||||
if err != nil {
|
slog.OnErrFatalf(err, "Error loading config: %v", err)
|
||||||
log.Fatalf("Error loading config: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Serve metrics exporter, if specified
|
// Serve metrics exporter, if specified
|
||||||
if ExportMetrics {
|
if ExportMetrics {
|
||||||
log.Println("INFO: Exporting metrics to Prometheus")
|
slog.Infof("Exporting metrics to Prometheus")
|
||||||
|
|
||||||
go ServeMetrics()
|
go ServeMetrics()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
24
monitor.go
24
monitor.go
@ -1,10 +1,11 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"log"
|
|
||||||
"math"
|
"math"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.iamthefij.com/iamthefij/slog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Monitor represents a particular periodic check of a command
|
// Monitor represents a particular periodic check of a command
|
||||||
@ -66,17 +67,11 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
|
|||||||
alertNotice = monitor.failure()
|
alertNotice = monitor.failure()
|
||||||
}
|
}
|
||||||
|
|
||||||
if LogDebug {
|
slog.Debugf("Command output: %s", monitor.lastOutput)
|
||||||
log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
|
slog.OnErrWarnf(err, "Command result: %v", err)
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if LogDebug {
|
|
||||||
log.Printf("DEBUG: Command result: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf(
|
slog.Infof(
|
||||||
"INFO: %s success=%t, alert=%t",
|
"%s success=%t, alert=%t",
|
||||||
monitor.Name,
|
monitor.Name,
|
||||||
isSuccess,
|
isSuccess,
|
||||||
alertNotice != nil,
|
alertNotice != nil,
|
||||||
@ -106,15 +101,14 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
|
|||||||
monitor.failureCount++
|
monitor.failureCount++
|
||||||
// If we haven't hit the minimum failures, we can exit
|
// If we haven't hit the minimum failures, we can exit
|
||||||
if monitor.failureCount < monitor.getAlertAfter() {
|
if monitor.failureCount < monitor.getAlertAfter() {
|
||||||
if LogDebug {
|
slog.Debugf(
|
||||||
log.Printf(
|
"%s failed but did not hit minimum failures. "+
|
||||||
"DEBUG: %s failed but did not hit minimum failures. "+
|
|
||||||
"Count: %v alert after: %v",
|
"Count: %v alert after: %v",
|
||||||
monitor.Name,
|
monitor.Name,
|
||||||
monitor.failureCount,
|
monitor.failureCount,
|
||||||
monitor.getAlertAfter(),
|
monitor.getAlertAfter(),
|
||||||
)
|
)
|
||||||
}
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
2
util.go
2
util.go
@ -8,7 +8,7 @@ import (
|
|||||||
// ShellCommand takes a string and executes it as a command using `sh`
|
// ShellCommand takes a string and executes it as a command using `sh`
|
||||||
func ShellCommand(command string) *exec.Cmd {
|
func ShellCommand(command string) *exec.Cmd {
|
||||||
shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
|
shellCommand := []string{"sh", "-c", strings.TrimSpace(command)}
|
||||||
//log.Printf("Shell command: %v", shellCommand)
|
|
||||||
return exec.Command(shellCommand[0], shellCommand[1:]...)
|
return exec.Command(shellCommand[0], shellCommand[1:]...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user