Compare commits

..

1 Commits

Author SHA1 Message Date
e4e6618af1 Adds ability to run specified alerts on startup
All checks were successful
continuous-integration/drone/push Build is passing
This is helpful to determine if your alerts are valid before an actual failure
2024-04-03 11:50:49 -07:00
10 changed files with 104 additions and 102 deletions

View File

@ -60,15 +60,51 @@ steps:
when:
event: tag
- name: Build and publish docker images
image: thegeeklab/drone-docker-buildx
- name: push image - arm
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
platforms:
- linux/amd64
- linux/arm64
- linux/arm
auto_tag_suffix: linux-arm
username:
from_secret: docker_username
password:
from_secret: docker_password
build_args:
- ARCH=arm
- REPO=arm32v7
- name: push image - arm64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-arm64
username:
from_secret: docker_username
password:
from_secret: docker_password
build_args:
- ARCH=arm64
- REPO=arm64v8
- name: push image - amd64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-amd64
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: publish manifest
image: plugins/manifest
settings:
spec: manifest.tmpl
auto_tag: true
ignore_missing: true
username:
from_secret: docker_username
password:

View File

@ -1,11 +1,11 @@
FROM alpine:3.18
ARG REPO=library
FROM ${REPO}/alpine:3.18
RUN mkdir /app
WORKDIR /app/
# Add common checking tools
# hadolint ignore=DL3018
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
# Add minitor user for running as non-root
RUN addgroup -S minitor && adduser -S minitor -G minitor
@ -15,9 +15,8 @@ COPY ./scripts /app/scripts
RUN chmod -R 755 /app/scripts
# Copy minitor in
ARG TARGETOS
ARG TARGETARCH
COPY ./dist/minitor-${TARGETOS}-${TARGETARCH} ./minitor
ARG ARCH=amd64
COPY ./dist/minitor-linux-${ARCH} ./minitor
# Drop to non-root user
USER minitor

View File

@ -1,3 +1,4 @@
ARG REPO=library
FROM golang:1.20 AS builder
RUN mkdir /app
@ -8,13 +9,12 @@ RUN go mod download
COPY ./*.go /app/
ARG TARGETOS
ARG TARGETARCH
ARG ARCH=amd64
ARG VERSION=dev
ENV CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=${TARGETARCH}
ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
FROM alpine:3.18
FROM ${REPO}/alpine:3.18
RUN mkdir /app
WORKDIR /app/
@ -22,8 +22,7 @@ WORKDIR /app/
COPY --from=builder /app/minitor .
# Add common checking tools
# hadolint ignore=DL3018
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata
RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2023c
# Add minitor user for running as non-root
RUN addgroup -S minitor && adduser -S minitor -G minitor

View File

@ -79,11 +79,11 @@ $(TARGET_ALIAS):
# Arch specific docker build targets
.PHONY: docker-build-arm
docker-build-arm: dist/minitor-linux-arm
docker build --platform linux/arm . -t ${DOCKER_TAG}-linux-arm
docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
.PHONY: docker-build-arm64
docker-build-arm64: dist/minitor-linux-arm64
docker build --platform linux/arm64 . -t ${DOCKER_TAG}-linux-arm64
docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
# Cross run on host architechture
.PHONY: docker-run-arm

View File

@ -58,7 +58,6 @@ The global configurations are:
|---|---|
|`check_interval`|Maximum frequency to run checks for each monitor as duration, eg. 1m2s.|
|`default_alert_after`|A default value used as an `alert_after` value for a monitor if not specified or 0.|
|`default_alert_every`|A default value used as an `alert_every` value for a monitor if not specified.|
|`default_alert_down`|Default down alerts to used by a monitor in case none are provided.|
|`default_alert_up`|Default up alerts to used by a monitor in case none are provided.|
|`monitors`|List of all monitors. Detailed description below|
@ -119,16 +118,6 @@ To provide flexible formatting, the following non-standard functions are availab
For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants).
#### Running alerts on startup
It's not the best feeling to find out your alerts are broken when you're expecting to be alerted about another failure. To avoid this and provide early insight into broken alerts, it is possible to specify a list of alerts to run when Minitor starts up. This can be done using the command line flag `-startup-alerts`. This flag accepts a comma separated list of strings and will run a test of each of those alerts. Minitor will then respond as it typically does for any failed alert. This can be used to allow you time to correct when initially launching, and to allow schedulers to more easily detect a failed deployment of Minitor.
Eg.
```bash
minitor -startup-alerts=log_down,log_up -config ./config.yml
```
### Metrics
Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.

16
main.go
View File

@ -92,7 +92,7 @@ func checkMonitors(config *Config) error {
return nil
}
func sendStartupAlerts(config *Config, alertNames []string) error {
func sendFirstRunAlerts(config *Config, alertNames []string) error {
for _, alertName := range alertNames {
var err error
@ -123,7 +123,7 @@ func sendStartupAlerts(config *Config, alertNames []string) error {
func main() {
showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
firstRunAlerts := flag.String("first-run-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
@ -138,10 +138,6 @@ func main() {
return
}
if PyCompat {
slog.Warningf("Python compatibility mode is enabled. This will be removed in the next major release. Please update your configuration.")
}
// Load configuration
config, err := LoadConfig(*configPath)
slog.OnErrFatalf(err, "Error loading config: %v", err)
@ -153,12 +149,12 @@ func main() {
go ServeMetrics()
}
if *startupAlerts != "" {
alertNames := strings.Split(*startupAlerts, ",")
if *firstRunAlerts != "" {
alertNames := strings.Split(*firstRunAlerts, ",")
err = sendStartupAlerts(&config, alertNames)
err = sendFirstRunAlerts(&config, alertNames)
slog.OnErrPanicf(err, "Error running startup alerts")
slog.OnErrPanicf(err, "Error running first run alerts")
}
// Start main loop

View File

@ -137,22 +137,22 @@ func TestCheckMonitors(t *testing.T) {
func TestFirstRunAlerts(t *testing.T) {
cases := []struct {
config Config
expectErr bool
startupAlerts []string
name string
config Config
expectErr bool
firstRunAlerts []string
name string
}{
{
config: Config{},
expectErr: false,
startupAlerts: []string{},
name: "Empty",
config: Config{},
expectErr: false,
firstRunAlerts: []string{},
name: "Empty",
},
{
config: Config{},
expectErr: true,
startupAlerts: []string{"missing"},
name: "Unknown",
config: Config{},
expectErr: true,
firstRunAlerts: []string{"missing"},
name: "Unknown",
},
{
config: Config{
@ -162,9 +162,9 @@ func TestFirstRunAlerts(t *testing.T) {
},
},
},
expectErr: false,
startupAlerts: []string{"good"},
name: "Successful alert",
expectErr: false,
firstRunAlerts: []string{"good"},
name: "Successful alert",
},
{
config: Config{
@ -175,9 +175,9 @@ func TestFirstRunAlerts(t *testing.T) {
},
},
},
expectErr: true,
startupAlerts: []string{"bad"},
name: "Failed alert",
expectErr: true,
firstRunAlerts: []string{"bad"},
name: "Failed alert",
},
}
@ -187,7 +187,7 @@ func TestFirstRunAlerts(t *testing.T) {
t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err)
}
err = sendStartupAlerts(&c.config, c.startupAlerts)
err = sendFirstRunAlerts(&c.config, c.firstRunAlerts)
if err == nil && c.expectErr {
t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name)
} else if err != nil && !c.expectErr {

25
manifest.tmpl Normal file
View File

@ -0,0 +1,25 @@
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7

View File

@ -149,7 +149,6 @@ func (monitor *Monitor) failure() (notice *AlertNotice) {
// If we're going to alert, increment count
if notice != nil {
monitor.alertCount++
notice.AlertCount = monitor.alertCount
}
return notice

View File

@ -138,47 +138,6 @@ func TestMonitorSuccess(t *testing.T) {
}
}
func TestMonitorAlertCount(t *testing.T) {
var alertEvery int16 = 1
cases := []struct {
checkSuccess bool
alertCount int16
name string
}{
{false, 1, "First failure and first alert"},
{false, 2, "Second failure and first alert"},
{true, 2, "Success should preserve past alert count"},
{false, 1, "First failure and first alert after success"},
}
// Unlike previous tests, this one requires a static Monitor with repeated
// calls to the failure method
monitor := Monitor{failureCount: 0, AlertAfter: 1, AlertEvery: &alertEvery}
for _, c := range cases {
log.Printf("Testing case %s", c.name)
var notice *AlertNotice
if c.checkSuccess {
notice = monitor.success()
} else {
notice = monitor.failure()
}
if notice == nil {
t.Errorf("failure(%v) expected notice, got nil", c.name)
}
if notice.AlertCount != c.alertCount {
t.Errorf("failure(%v), expected=%v actual=%v", c.name, c.alertCount, notice.AlertCount)
log.Printf("Case failed: %s", c.name)
}
log.Println("-----")
}
}
// TestMonitorFailureAlertAfter tests that alerts will not trigger until
// hitting the threshold provided by AlertAfter
func TestMonitorFailureAlertAfter(t *testing.T) {