Add prometheus metrics exporter
This should add metrics parity to the Python version
This commit is contained in:
parent
1f6193e486
commit
457e19af9b
6
Makefile
6
Makefile
@ -1,6 +1,7 @@
|
||||
.PHONY: all
|
||||
DOCKER_TAG ?= minitor-go-${USER}
|
||||
|
||||
.PHONY: test
|
||||
.PHONY: default
|
||||
default: test
|
||||
|
||||
.PHONY: build
|
||||
@ -12,7 +13,7 @@ minitor-go:
|
||||
|
||||
.PHONY: run
|
||||
run: minitor-go build
|
||||
./minitor-go -debug
|
||||
./minitor-go -debug -metrics
|
||||
|
||||
.PHONY: test
|
||||
test:
|
||||
@ -24,7 +25,6 @@ test:
|
||||
@go tool cover -func=coverage.out | awk -v target=80.0% \
|
||||
'/^total:/ { print "Total coverage: " $$3 " Minimum coverage: " target; if ($$3+0.0 >= target+0.0) print "ok"; else { print "fail"; exit 1; } }'
|
||||
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f ./minitor-go
|
||||
|
||||
@ -84,7 +84,9 @@ func LoadConfig(filePath string) (config Config, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("config:\n%v\n", config)
|
||||
if LogDebug {
|
||||
log.Printf("DEBUG: Config values:\n%v\n", config)
|
||||
}
|
||||
|
||||
if !config.IsValid() {
|
||||
err = errors.New("Invalid configuration")
|
||||
|
||||
5
go.mod
5
go.mod
@ -2,4 +2,7 @@ module git.iamthefij.com/iamthefij/minitor-go
|
||||
|
||||
go 1.12
|
||||
|
||||
require gopkg.in/yaml.v2 v2.2.4
|
||||
require (
|
||||
github.com/prometheus/client_golang v1.2.1
|
||||
gopkg.in/yaml.v2 v2.2.4
|
||||
)
|
||||
|
||||
75
go.sum
75
go.sum
@ -1,3 +1,78 @@
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
|
||||
github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
|
||||
github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
|
||||
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
|
||||
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
27
main.go
27
main.go
@ -11,6 +11,13 @@ var (
|
||||
// LogDebug will control whether debug messsages should be logged
|
||||
LogDebug = false
|
||||
|
||||
// ExportMetrics will track whether or not we want to export metrics to prometheus
|
||||
ExportMetrics = false
|
||||
// MetricsPort is the port to expose metrics on
|
||||
MetricsPort = 8080
|
||||
// Metrics contains all active metrics
|
||||
Metrics = NewMetrics()
|
||||
|
||||
// version of minitor being run
|
||||
version = "dev"
|
||||
)
|
||||
@ -18,7 +25,13 @@ var (
|
||||
func checkMonitors(config *Config) error {
|
||||
for _, monitor := range config.Monitors {
|
||||
if monitor.ShouldCheck() {
|
||||
_, alertNotice := monitor.Check()
|
||||
success, alertNotice := monitor.Check()
|
||||
|
||||
hasAlert := alertNotice != nil
|
||||
|
||||
// Track status metrics
|
||||
Metrics.SetMonitorStatus(monitor.Name, success)
|
||||
Metrics.CountCheck(monitor.Name, success, hasAlert)
|
||||
|
||||
// Should probably consider refactoring everything below here
|
||||
if alertNotice != nil {
|
||||
@ -50,6 +63,9 @@ func checkMonitors(config *Config) error {
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
// Count alert metrics
|
||||
Metrics.CountAlert(monitor.Name, alert.Name)
|
||||
} else {
|
||||
// This case should never actually happen since we validate against it
|
||||
log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
|
||||
@ -66,12 +82,13 @@ func checkMonitors(config *Config) error {
|
||||
func main() {
|
||||
// Get debug flag
|
||||
flag.BoolVar(&LogDebug, "debug", false, "Enables debug logs (default: false)")
|
||||
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
|
||||
var showVersion = flag.Bool("version", false, "Display the version of minitor and exit")
|
||||
flag.Parse()
|
||||
|
||||
// Print version if flag is provided
|
||||
if *showVersion {
|
||||
fmt.Println("Minitor version:", version)
|
||||
log.Println("Minitor version:", version)
|
||||
return
|
||||
}
|
||||
|
||||
@ -81,6 +98,12 @@ func main() {
|
||||
log.Fatalf("Error loading config: %v", err)
|
||||
}
|
||||
|
||||
// Serve metrics exporter, if specified
|
||||
if ExportMetrics {
|
||||
log.Println("INFO: Exporting metrics to Prometheus")
|
||||
go ServeMetrics()
|
||||
}
|
||||
|
||||
// Start main loop
|
||||
for {
|
||||
err = checkMonitors(&config)
|
||||
|
||||
101
metrics.go
Normal file
101
metrics.go
Normal file
@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
// "github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// TODO: Not sure if this is the best way to handle. A global instance for
|
||||
// metrics isn't bad, but it might be nice to curry versions of the metrics
|
||||
// for each monitor. Especially since every monitor has it's own. Perhaps
|
||||
// another new function that essentially curries each metric for a given
|
||||
// monitor name would do. This could be run when validating monitors and
|
||||
// initializing alert templates.
|
||||
|
||||
// MinitorMetrics contains all counters and metrics that Minitor will need to access
|
||||
type MinitorMetrics struct {
|
||||
alertCount *prometheus.CounterVec
|
||||
checkCount *prometheus.CounterVec
|
||||
monitorStatus *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
// NewMetrics creates and initializes all metrics
|
||||
func NewMetrics() *MinitorMetrics {
|
||||
// Initialize all metrics
|
||||
metrics := &MinitorMetrics{
|
||||
alertCount: prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "minitor_alert_total",
|
||||
Help: "Number of Minitor alerts",
|
||||
},
|
||||
[]string{"alert", "monitor"},
|
||||
),
|
||||
checkCount: prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "minitor_check_total",
|
||||
Help: "Number of Minitor checks",
|
||||
},
|
||||
[]string{"monitor", "status", "is_alert"},
|
||||
),
|
||||
monitorStatus: prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "minitor_monitor_up_count",
|
||||
Help: "Status of currently responsive monitors",
|
||||
},
|
||||
[]string{"monitor"},
|
||||
),
|
||||
}
|
||||
|
||||
// Register newly created metrics
|
||||
prometheus.MustRegister(metrics.alertCount)
|
||||
prometheus.MustRegister(metrics.checkCount)
|
||||
prometheus.MustRegister(metrics.monitorStatus)
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
// SetMonitorStatus sets the current status of Monitor
|
||||
func (metrics *MinitorMetrics) SetMonitorStatus(monitor string, isUp bool) {
|
||||
val := 0.0
|
||||
if isUp {
|
||||
val = 1.0
|
||||
}
|
||||
metrics.monitorStatus.With(prometheus.Labels{"monitor": monitor}).Set(val)
|
||||
}
|
||||
|
||||
// CountCheck counts the result of a particular Monitor check
|
||||
func (metrics *MinitorMetrics) CountCheck(monitor string, isSuccess bool, isAlert bool) {
|
||||
status := "failure"
|
||||
if isSuccess {
|
||||
status = "success"
|
||||
}
|
||||
|
||||
alertVal := "false"
|
||||
if isAlert {
|
||||
alertVal = "true"
|
||||
}
|
||||
|
||||
metrics.checkCount.With(
|
||||
prometheus.Labels{"monitor": monitor, "status": status, "is_alert": alertVal},
|
||||
).Inc()
|
||||
}
|
||||
|
||||
// CountAlert counts an alert
|
||||
func (metrics *MinitorMetrics) CountAlert(monitor string, alert string) {
|
||||
metrics.alertCount.With(
|
||||
prometheus.Labels{
|
||||
"alert": alert,
|
||||
"monitor": monitor,
|
||||
},
|
||||
).Inc()
|
||||
}
|
||||
|
||||
// ServeMetrics starts an http server with a Prometheus metrics handler
|
||||
func ServeMetrics() {
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
host := fmt.Sprintf(":%d", MetricsPort)
|
||||
_ = http.ListenAndServe(host, nil)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user