WIP: Update logging to improve formatting a bit
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
IamTheFij 2019-11-22 15:07:18 -08:00
parent f6ccd9a3bd
commit f1451166e6
7 changed files with 60 additions and 65 deletions

View File

@ -3,10 +3,11 @@ package main
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"log"
"os/exec" "os/exec"
"text/template" "text/template"
"time" "time"
log "github.com/sirupsen/logrus"
) )
// Alert is a config driven mechanism for sending a notice // Alert is a config driven mechanism for sending a notice
@ -38,9 +39,7 @@ func (alert Alert) IsValid() bool {
// BuildTemplates compiles command templates for the Alert // BuildTemplates compiles command templates for the Alert
func (alert *Alert) BuildTemplates() error { func (alert *Alert) BuildTemplates() error {
if LogDebug { log.Debugf("Building template for alert %s", alert.Name)
log.Printf("DEBUG: Building template for alert %s", alert.Name)
}
if alert.commandTemplate == nil && alert.Command != nil { if alert.commandTemplate == nil && alert.Command != nil {
alert.commandTemplate = []*template.Template{} alert.commandTemplate = []*template.Template{}
for i, cmdPart := range alert.Command { for i, cmdPart := range alert.Command {
@ -60,8 +59,8 @@ func (alert *Alert) BuildTemplates() error {
} }
// Send will send an alert notice by executing the command template // Send will send an alert notice by executing the command template
func (alert Alert) Send(notice AlertNotice) (output_str string, err error) { func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
log.Printf("INFO: Sending alert %s for %s", alert.Name, notice.MonitorName) log.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
var cmd *exec.Cmd var cmd *exec.Cmd
if alert.commandTemplate != nil { if alert.commandTemplate != nil {
command := []string{} command := []string{}
@ -95,10 +94,8 @@ func (alert Alert) Send(notice AlertNotice) (output_str string, err error) {
var output []byte var output []byte
output, err = cmd.CombinedOutput() output, err = cmd.CombinedOutput()
output_str = string(output) outputStr = string(output)
if LogDebug { log.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
log.Printf("DEBUG: Alert output for: %s\n---\n%s\n---", alert.Name, output_str)
}
return output_str, err return outputStr, err
} }

View File

@ -1,8 +1,9 @@
package main package main
import ( import (
"log"
"testing" "testing"
log "github.com/sirupsen/logrus"
) )
func TestAlertIsValid(t *testing.T) { func TestAlertIsValid(t *testing.T) {
@ -22,13 +23,13 @@ func TestAlertIsValid(t *testing.T) {
} }
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Debugf("Testing case %s", c.name)
actual := c.alert.IsValid() actual := c.alert.IsValid()
if actual != c.expected { if actual != c.expected {
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual) t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expected, actual)
log.Printf("Case failed: %s", c.name) log.Debugf("Case failed: %s", c.name)
} }
log.Println("-----") log.Debugf("-----")
} }
} }
@ -71,19 +72,19 @@ func TestAlertSend(t *testing.T) {
} }
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Debugf("Testing case %s", c.name)
c.alert.BuildTemplates() c.alert.BuildTemplates()
output, err := c.alert.Send(c.notice) output, err := c.alert.Send(c.notice)
hasErr := (err != nil) hasErr := (err != nil)
if output != c.expectedOutput { if output != c.expectedOutput {
t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output) t.Errorf("Send(%v output), expected=%v actual=%v", c.name, c.expectedOutput, output)
log.Printf("Case failed: %s", c.name) log.Debugf("Case failed: %s", c.name)
} }
if hasErr != c.expectErr { if hasErr != c.expectErr {
t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err) t.Errorf("Send(%v err), expected=%v actual=%v", c.name, "Err", err)
log.Printf("Case failed: %s", c.name) log.Debugf("Case failed: %s", c.name)
} }
log.Println("-----") log.Debugf("-----")
} }
} }
@ -94,7 +95,7 @@ func TestAlertSendNoTemplates(t *testing.T) {
if err == nil { if err == nil {
t.Errorf("Send(no template), expected=%v actual=%v", "Err", output) t.Errorf("Send(no template), expected=%v actual=%v", "Err", output)
} }
log.Println("-----") log.Debugf("-----")
} }
func TestAlertBuildTemplate(t *testing.T) { func TestAlertBuildTemplate(t *testing.T) {
@ -109,13 +110,13 @@ func TestAlertBuildTemplate(t *testing.T) {
} }
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Debugf("Testing case %s", c.name)
err := c.alert.BuildTemplates() err := c.alert.BuildTemplates()
hasErr := (err != nil) hasErr := (err != nil)
if hasErr != c.expectErr { if hasErr != c.expectErr {
t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err) t.Errorf("IsValid(%v), expected=%t actual=%t", c.name, c.expectErr, err)
log.Printf("Case failed: %s", c.name) log.Debugf("Case failed: %s", c.name)
} }
log.Println("-----") log.Debugf("-----")
} }
} }

View File

@ -3,9 +3,9 @@ package main
import ( import (
"errors" "errors"
"io/ioutil" "io/ioutil"
"log"
"os" "os"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
) )
@ -22,20 +22,20 @@ func (config Config) IsValid() (isValid bool) {
// Validate monitors // Validate monitors
if config.Monitors == nil || len(config.Monitors) == 0 { if config.Monitors == nil || len(config.Monitors) == 0 {
log.Printf("ERROR: Invalid monitor configuration: Must provide at least one monitor") log.Errorf("Invalid monitor configuration: Must provide at least one monitor")
isValid = false isValid = false
} }
for _, monitor := range config.Monitors { for _, monitor := range config.Monitors {
if !monitor.IsValid() { if !monitor.IsValid() {
log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name) log.Errorf("Invalid monitor configuration: %s", monitor.Name)
isValid = false isValid = false
} }
// Check that all Monitor alerts actually exist // Check that all Monitor alerts actually exist
for _, isUp := range []bool{true, false} { for _, isUp := range []bool{true, false} {
for _, alertName := range monitor.GetAlertNames(isUp) { for _, alertName := range monitor.GetAlertNames(isUp) {
if _, ok := config.Alerts[alertName]; !ok { if _, ok := config.Alerts[alertName]; !ok {
log.Printf( log.Errorf(
"ERROR: Invalid monitor configuration: %s. Unknown alert %s", "Invalid monitor configuration: %s. Unknown alert %s",
monitor.Name, alertName, monitor.Name, alertName,
) )
isValid = false isValid = false
@ -46,12 +46,12 @@ func (config Config) IsValid() (isValid bool) {
// Validate alerts // Validate alerts
if config.Alerts == nil || len(config.Alerts) == 0 { if config.Alerts == nil || len(config.Alerts) == 0 {
log.Printf("ERROR: Invalid alert configuration: Must provide at least one alert") log.Errorf("Invalid alert configuration: Must provide at least one alert")
isValid = false isValid = false
} }
for _, alert := range config.Alerts { for _, alert := range config.Alerts {
if !alert.IsValid() { if !alert.IsValid() {
log.Printf("ERROR: Invalid alert configuration: %s", alert.Name) log.Errorf("Invalid alert configuration: %s", alert.Name)
isValid = false isValid = false
} }
} }
@ -85,9 +85,7 @@ func LoadConfig(filePath string) (config Config, err error) {
return return
} }
if LogDebug { log.Debugf("Config values:\n%v\n", config)
log.Printf("DEBUG: Config values:\n%v\n", config)
}
if !config.IsValid() { if !config.IsValid() {
err = errors.New("Invalid configuration") err = errors.New("Invalid configuration")

1
go.mod
View File

@ -4,5 +4,6 @@ go 1.12
require ( require (
github.com/prometheus/client_golang v1.2.1 github.com/prometheus/client_golang v1.2.1
github.com/sirupsen/logrus v1.4.2
gopkg.in/yaml.v2 v2.2.4 gopkg.in/yaml.v2 v2.2.4
) )

2
go.sum
View File

@ -53,6 +53,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -68,6 +69,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

17
main.go
View File

@ -3,14 +3,12 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"log"
"time" "time"
log "github.com/sirupsen/logrus"
) )
var ( var (
// LogDebug will control whether debug messsages should be logged
LogDebug = false
// ExportMetrics will track whether or not we want to export metrics to prometheus // ExportMetrics will track whether or not we want to export metrics to prometheus
ExportMetrics = false ExportMetrics = false
// MetricsPort is the port to expose metrics on // MetricsPort is the port to expose metrics on
@ -35,9 +33,7 @@ func checkMonitors(config *Config) error {
// Should probably consider refactoring everything below here // Should probably consider refactoring everything below here
if alertNotice != nil { if alertNotice != nil {
if LogDebug { log.Debugf("Recieved an alert notice from %s", alertNotice.MonitorName)
log.Printf("DEBUG: Recieved an alert notice from %s", alertNotice.MonitorName)
}
alertNames := monitor.GetAlertNames(alertNotice.IsUp) alertNames := monitor.GetAlertNames(alertNotice.IsUp)
if alertNames == nil { if alertNames == nil {
// This should only happen for a recovery alert. AlertDown is validated not empty // This should only happen for a recovery alert. AlertDown is validated not empty
@ -81,11 +77,16 @@ func checkMonitors(config *Config) error {
func main() { func main() {
// Get debug flag // Get debug flag
flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)") var debug = flag.Bool("debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)") flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
var showVersion = flag.Bool("version", false, "Display the version of minitor and exit") var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
flag.Parse() flag.Parse()
// Set debug if flag is set
if *debug {
log.SetLevel(log.DebugLevel)
}
// Print version if flag is provided // Print version if flag is provided
if *showVersion { if *showVersion {
log.Println("Minitor version:", version) log.Println("Minitor version:", version)

View File

@ -1,10 +1,11 @@
package main package main
import ( import (
"log"
"math" "math"
"os/exec" "os/exec"
"time" "time"
log "github.com/sirupsen/logrus"
) )
// Monitor represents a particular periodic check of a command // Monitor represents a particular periodic check of a command
@ -70,20 +71,18 @@ func (monitor *Monitor) Check() (bool, *AlertNotice) {
alertNotice = monitor.failure() alertNotice = monitor.failure()
} }
if LogDebug { log.Debugf("Command output: %s", monitor.lastOutput)
log.Printf("DEBUG: Command output: %s", monitor.lastOutput)
}
if err != nil { if err != nil {
if LogDebug { log.Debugf("Command result: %v", err)
log.Printf("DEBUG: Command result: %v", err)
}
} }
log.Printf( log.WithFields(log.Fields{
"INFO: %s success=%t, alert=%t", "monitor": monitor.Name,
"success": isSuccess,
"alert": alertNotice != nil,
}).Infof(
"%s checked",
monitor.Name, monitor.Name,
isSuccess,
alertNotice != nil,
) )
return isSuccess, alertNotice return isSuccess, alertNotice
@ -109,15 +108,13 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
monitor.failureCount++ monitor.failureCount++
// If we haven't hit the minimum failures, we can exit // If we haven't hit the minimum failures, we can exit
if monitor.failureCount < monitor.getAlertAfter() { if monitor.failureCount < monitor.getAlertAfter() {
if LogDebug { log.Debugf(
log.Printf( "%s failed but did not hit minimum failures. "+
"DEBUG: %s failed but did not hit minimum failures. "+ "Count: %v alert after: %v",
"Count: %v alert after: %v", monitor.Name,
monitor.Name, monitor.failureCount,
monitor.failureCount, monitor.getAlertAfter(),
monitor.getAlertAfter(), )
)
}
return return
} }
@ -155,18 +152,16 @@ func (monitor Monitor) getAlertAfter() int16 {
// Zero is one! // Zero is one!
if monitor.AlertAfter == 0 { if monitor.AlertAfter == 0 {
return 1 return 1
} else {
return monitor.AlertAfter
} }
return monitor.AlertAfter
} }
// GetAlertNames gives a list of alert names for a given monitor status // GetAlertNames gives a list of alert names for a given monitor status
func (monitor Monitor) GetAlertNames(up bool) []string { func (monitor Monitor) GetAlertNames(up bool) []string {
if up { if up {
return monitor.AlertUp return monitor.AlertUp
} else {
return monitor.AlertDown
} }
return monitor.AlertDown
} }
func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice { func (monitor Monitor) createAlertNotice(isUp bool) *AlertNotice {