Compare commits

..

10 Commits

Author SHA1 Message Date
7604138c9e Use buildx
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/tag Build is passing
2024-04-03 12:15:11 -07:00
f58b4c1495 Adds ability to run specified alerts on startup
Some checks failed
continuous-integration/drone/push Build is failing
This is helpful to determine if your alerts are valid before an actual failure
2024-04-03 12:03:17 -07:00
6a2b44673e Upgrade prometheus client and protobuf
Some checks failed
continuous-integration/drone/push Build is failing
2024-04-03 11:28:01 -07:00
01cca50532 Add tzdata
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/tag Build is passing
Allows setting container timezone using TZ env variable
2023-08-11 06:20:35 -07:00
2789aa63e4 More loosely pins apk packages 2023-08-11 06:20:15 -07:00
37db4b2db0 Update error string when failing to send alert
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/tag Build is passing
Wrap both originating errors
2023-08-10 16:23:02 -04:00
41a1dbeceb Add date format functions 2023-08-10 16:22:30 -04:00
c02d64d674 Update go to 1.20
All checks were successful
continuous-integration/drone/push Build is passing
2023-08-10 16:21:33 -04:00
46f4561bea Update alpine and system package versions
All checks were successful
continuous-integration/drone/push Build is passing
Bump to alpine 3.18
2023-06-14 16:52:04 -07:00
a1e0e9698b Add dig and nslookup
Some checks failed
continuous-integration/drone/push Build is failing
continuous-integration/drone/tag Build is passing
2023-05-05 14:07:53 -07:00
17 changed files with 306 additions and 111 deletions

View File

@ -4,7 +4,7 @@ name: test
steps: steps:
- name: test - name: test
image: golang:1.17 image: golang:1.20
environment: environment:
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}} VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
commands: commands:
@ -30,7 +30,7 @@ trigger:
steps: steps:
- name: build all binaries - name: build all binaries
image: golang:1.17 image: golang:1.20
environment: environment:
VERSION: ${DRONE_TAG:-${DRONE_COMMIT}} VERSION: ${DRONE_TAG:-${DRONE_COMMIT}}
commands: commands:
@ -60,51 +60,15 @@ steps:
when: when:
event: tag event: tag
- name: push image - arm - name: Build and publish docker images
image: plugins/docker image: thegeeklab/drone-docker-buildx
settings: settings:
repo: iamthefij/minitor-go repo: iamthefij/minitor-go
auto_tag: true auto_tag: true
auto_tag_suffix: linux-arm platforms:
username: - linux/amd64
from_secret: docker_username - linux/arm64
password: - linux/arm
from_secret: docker_password
build_args:
- ARCH=arm
- REPO=arm32v7
- name: push image - arm64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-arm64
username:
from_secret: docker_username
password:
from_secret: docker_password
build_args:
- ARCH=arm64
- REPO=arm64v8
- name: push image - amd64
image: plugins/docker
settings:
repo: iamthefij/minitor-go
auto_tag: true
auto_tag_suffix: linux-amd64
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: publish manifest
image: plugins/manifest
settings:
spec: manifest.tmpl
auto_tag: true
ignore_missing: true
username: username:
from_secret: docker_username from_secret: docker_username
password: password:

View File

@ -1,11 +1,10 @@
ARG REPO=library FROM alpine:3.18
FROM ${REPO}/alpine:3.17
RUN mkdir /app RUN mkdir /app
WORKDIR /app/ WORKDIR /app/
# Add common checking tools # Add common checking tools
RUN apk --no-cache add bash=~5 curl=~8 jq=~1.6 bind-tools~=9 RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2024a
# Add minitor user for running as non-root # Add minitor user for running as non-root
RUN addgroup -S minitor && adduser -S minitor -G minitor RUN addgroup -S minitor && adduser -S minitor -G minitor
@ -15,8 +14,9 @@ COPY ./scripts /app/scripts
RUN chmod -R 755 /app/scripts RUN chmod -R 755 /app/scripts
# Copy minitor in # Copy minitor in
ARG ARCH=amd64 ARG TARGETOS
COPY ./dist/minitor-linux-${ARCH} ./minitor ARG TARGETARCH
COPY ./dist/minitor-${TARGETOS}-${TARGETARCH} ./minitor
# Drop to non-root user # Drop to non-root user
USER minitor USER minitor

View File

@ -1,5 +1,4 @@
ARG REPO=library FROM golang:1.20 AS builder
FROM golang:1.17 AS builder
RUN mkdir /app RUN mkdir /app
WORKDIR /app WORKDIR /app
@ -9,12 +8,13 @@ RUN go mod download
COPY ./*.go /app/ COPY ./*.go /app/
ARG ARCH=amd64 ARG TARGETOS
ARG TARGETARCH
ARG VERSION=dev ARG VERSION=dev
ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} ENV CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=${TARGETARCH}
RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor . RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
FROM ${REPO}/alpine:3.17 FROM alpine:3.18
RUN mkdir /app RUN mkdir /app
WORKDIR /app/ WORKDIR /app/
@ -22,7 +22,7 @@ WORKDIR /app/
COPY --from=builder /app/minitor . COPY --from=builder /app/minitor .
# Add common checking tools # Add common checking tools
RUN apk --no-cache add bash=~5 curl=~8 jq=~1.6 bind-tools~=9 RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2024a
# Add minitor user for running as non-root # Add minitor user for running as non-root
RUN addgroup -S minitor && adduser -S minitor -G minitor RUN addgroup -S minitor && adduser -S minitor -G minitor

View File

@ -79,11 +79,11 @@ $(TARGET_ALIAS):
# Arch specific docker build targets # Arch specific docker build targets
.PHONY: docker-build-arm .PHONY: docker-build-arm
docker-build-arm: dist/minitor-linux-arm docker-build-arm: dist/minitor-linux-arm
docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm docker build --platform linux/arm . -t ${DOCKER_TAG}-linux-arm
.PHONY: docker-build-arm64 .PHONY: docker-build-arm64
docker-build-arm64: dist/minitor-linux-arm64 docker-build-arm64: dist/minitor-linux-arm64
docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64 docker build --platform linux/arm64 . -t ${DOCKER_TAG}-linux-arm64
# Cross run on host architechture # Cross run on host architechture
.PHONY: docker-run-arm .PHONY: docker-run-arm

View File

@ -46,6 +46,8 @@ docker run -v $PWD/config.yml:/app/config.yml iamthefij/minitor-go:latest
Images are provided for `amd64`, `arm`, and `arm64` architechtures. Images are provided for `amd64`, `arm`, and `arm64` architechtures.
Timezone configuration for the container is set by passing the `TZ` env variable. Eg. `TZ=America/Los_Angeles`.
## Configuring ## Configuring
In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file. In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file.
@ -94,10 +96,38 @@ Also, when alerts are executed, they will be passed through Go's format function
|`{{.AlertCount}}`|Number of times this monitor has alerted| |`{{.AlertCount}}`|Number of times this monitor has alerted|
|`{{.FailureCount}}`|The total number of sequential failed checks for this monitor| |`{{.FailureCount}}`|The total number of sequential failed checks for this monitor|
|`{{.LastCheckOutput}}`|The last returned value from the check command to either stderr or stdout| |`{{.LastCheckOutput}}`|The last returned value from the check command to either stderr or stdout|
|`{{.LastSuccess}}`|The ISO datetime of the last successful check| |`{{.LastSuccess}}`|The datetime of the last successful check as a go Time struct|
|`{{.MonitorName}}`|The name of the monitor that failed and triggered the alert| |`{{.MonitorName}}`|The name of the monitor that failed and triggered the alert|
|`{{.IsUp}}`|Indicates if the monitor that is alerting is up or not. Can be used in a conditional message template| |`{{.IsUp}}`|Indicates if the monitor that is alerting is up or not. Can be used in a conditional message template|
To provide flexible formatting, the following non-standard functions are available in templates:
|func|description|
|---|---|
|`ANSIC <Time>`|Formats provided time in ANSIC format|
|`UnixDate <Time>`|Formats provided time in UnixDate format|
|`RubyDate <Time>`|Formats provided time in RubyDate format|
|`RFC822Z <Time>`|Formats provided time in RFC822Z format|
|`RFC850 <Time>`|Formats provided time in RFC850 format|
|`RFC1123 <Time>`|Formats provided time in RFC1123 format|
|`RFC1123Z <Time>`|Formats provided time in RFC1123Z format|
|`RFC3339 <Time>`|Formats provided time in RFC3339 format|
|`RFC3339Nano <Time>`|Formats provided time in RFC3339Nano format|
|`FormatTime <Time> <string template>`|Formats provided time according to provided template|
|`InTZ <Time> <string timezone name>`|Converts provided time to parsed timezone from the provided name|
For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants).
#### Running alerts on startup
It's not the best feeling to find out your alerts are broken when you're expecting to be alerted about another failure. To avoid this and provide early insight into broken alerts, it is possible to specify a list of alerts to run when Minitor starts up. This can be done using the command line flag `-startup-alerts`. This flag accepts a comma separated list of strings and will run a test of each of those alerts. Minitor will then respond as it typically does for any failed alert. This can be used to allow you time to correct when initially launching, and to allow schedulers to more easily detect a failed deployment of Minitor.
Eg.
```bash
minitor -startup-alerts=log_down,log_up -config ./config.yml
```
### Metrics ### Metrics
Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana. Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.
@ -158,7 +188,7 @@ minitor-go:
check_interval: 1m30s check_interval: 1m30s
``` ```
The `-py-compat` flag has been removed. Any existing Python oriented configuration needs to be migrated to the new templates. For the time being, legacy configs for the Python version of Minitor should be compatible if you apply the `-py-compat` flag when running Minitor. Eventually, this flag will go away when later breaking changes are introduced.
## Future ## Future

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"os/exec" "os/exec"
"strings"
"text/template" "text/template"
"time" "time"
@ -44,21 +45,70 @@ func (alert Alert) IsValid() bool {
// BuildTemplates compiles command templates for the Alert // BuildTemplates compiles command templates for the Alert
func (alert *Alert) BuildTemplates() error { func (alert *Alert) BuildTemplates() error {
// TODO: Remove legacy template support later after 1.0
legacy := strings.NewReplacer(
"{alert_count}", "{{.AlertCount}}",
"{alert_message}", "{{.MonitorName}} check has failed {{.FailureCount}} times",
"{failure_count}", "{{.FailureCount}}",
"{last_output}", "{{.LastCheckOutput}}",
"{last_success}", "{{.LastSuccess}}",
"{monitor_name}", "{{.MonitorName}}",
)
slog.Debugf("Building template for alert %s", alert.Name) slog.Debugf("Building template for alert %s", alert.Name)
// Time format func factory
tff := func(formatString string) func(time.Time) string {
return func(t time.Time) string {
return t.Format(formatString)
}
}
// Create some functions for formatting datetimes in popular formats
timeFormatFuncs := template.FuncMap{
"ANSIC": tff(time.ANSIC),
"UnixDate": tff(time.UnixDate),
"RubyDate": tff(time.RubyDate),
"RFC822Z": tff(time.RFC822Z),
"RFC850": tff(time.RFC850),
"RFC1123": tff(time.RFC1123),
"RFC1123Z": tff(time.RFC1123Z),
"RFC3339": tff(time.RFC3339),
"RFC3339Nano": tff(time.RFC3339Nano),
"FormatTime": func(t time.Time, timeFormat string) string {
return t.Format(timeFormat)
},
"InTZ": func(t time.Time, tzName string) (time.Time, error) {
tz, err := time.LoadLocation(tzName)
if err != nil {
return t, fmt.Errorf("failed to convert time to specified tz: %w", err)
}
return t.In(tz), nil
},
}
switch { switch {
case alert.commandTemplate == nil && alert.Command.Command != nil: case alert.commandTemplate == nil && alert.Command.Command != nil:
alert.commandTemplate = []*template.Template{} alert.commandTemplate = []*template.Template{}
for i, cmdPart := range alert.Command.Command { for i, cmdPart := range alert.Command.Command {
if PyCompat {
cmdPart = legacy.Replace(cmdPart)
}
alert.commandTemplate = append(alert.commandTemplate, template.Must( alert.commandTemplate = append(alert.commandTemplate, template.Must(
template.New(alert.Name+fmt.Sprint(i)).Parse(cmdPart), template.New(alert.Name+fmt.Sprint(i)).Funcs(timeFormatFuncs).Parse(cmdPart),
)) ))
} }
case alert.commandShellTemplate == nil && alert.Command.ShellCommand != "": case alert.commandShellTemplate == nil && alert.Command.ShellCommand != "":
shellCmd := alert.Command.ShellCommand shellCmd := alert.Command.ShellCommand
if PyCompat {
shellCmd = legacy.Replace(shellCmd)
}
alert.commandShellTemplate = template.Must( alert.commandShellTemplate = template.Must(
template.New(alert.Name).Parse(shellCmd), template.New(alert.Name).Funcs(timeFormatFuncs).Parse(shellCmd),
) )
default: default:
return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate) return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate)
@ -118,7 +168,7 @@ func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
if err != nil { if err != nil {
err = fmt.Errorf( err = fmt.Errorf(
"Alert '%s' failed to send. Returned %v: %w", "Alert %s failed to send. Returned %w: %w",
alert.Name, alert.Name,
err, err,
ErrAlertFailed, ErrAlertFailed,

View File

@ -70,6 +70,14 @@ func TestAlertSend(t *testing.T) {
"Command shell with bad template", "Command shell with bad template",
false, false,
}, },
{
Alert{Command: CommandOrShell{ShellCommand: "echo {alert_message}"}},
AlertNotice{MonitorName: "test", FailureCount: 1},
"test check has failed 1 times\n",
false,
"Command shell with legacy template",
true,
},
// Test default log alert down // Test default log alert down
{ {
*NewLogAlert(), *NewLogAlert(),
@ -92,6 +100,8 @@ func TestAlertSend(t *testing.T) {
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
// Set PyCompat to value of compat flag
PyCompat = c.pyCompat
err := c.alert.BuildTemplates() err := c.alert.BuildTemplates()
if err != nil { if err != nil {
@ -111,6 +121,9 @@ func TestAlertSend(t *testing.T) {
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
// Set PyCompat back to default value
PyCompat = false
log.Println("-----") log.Println("-----")
} }
} }

View File

@ -13,11 +13,11 @@ var errInvalidConfig = errors.New("Invalid configuration")
// Config type is contains all provided user configuration // Config type is contains all provided user configuration
type Config struct { type Config struct {
CheckInterval time.Duration `yaml:"check_interval"` CheckInterval SecondsOrDuration `yaml:"check_interval"`
DefaultAlertAfter int16 `yaml:"default_alert_after"` DefaultAlertAfter int16 `yaml:"default_alert_after"`
DefaultAlertEvery *int16 `yaml:"default_alert_every"` DefaultAlertEvery *int16 `yaml:"default_alert_every"`
DefaultAlertDown []string `yaml:"default_alert_down"` DefaultAlertDown []string `yaml:"default_alert_down"`
DefaultAlertUp []string `yaml:"default_alert_up"` DefaultAlertUp []string `yaml:"default_alert_up"`
Monitors []*Monitor Monitors []*Monitor
Alerts map[string]*Alert Alerts map[string]*Alert
} }
@ -56,6 +56,34 @@ func (cos *CommandOrShell) UnmarshalYAML(unmarshal func(interface{}) error) erro
return nil return nil
} }
// SecondsOrDuration wraps a duration value for parsing a duration or seconds from YAML
// NOTE: This should be removed in favor of only parsing durations once compatibility is broken
type SecondsOrDuration struct {
value time.Duration
}
// Value returns a duration value
func (sod SecondsOrDuration) Value() time.Duration {
return sod.value
}
// UnmarshalYAML allows unmarshalling a duration value or seconds if an int was provided
func (sod *SecondsOrDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
var seconds int64
err := unmarshal(&seconds)
if err == nil {
sod.value = time.Second * time.Duration(seconds)
return nil
}
// Error indicates that we don't have an int
err = unmarshal(&sod.value)
return err
}
// IsValid checks config validity and returns true if valid // IsValid checks config validity and returns true if valid
func (config Config) IsValid() (isValid bool) { func (config Config) IsValid() (isValid bool) {
isValid = true isValid = true
@ -154,6 +182,18 @@ func LoadConfig(filePath string) (config Config, err error) {
slog.Debugf("Config values:\n%v\n", config) slog.Debugf("Config values:\n%v\n", config)
// Add log alert if not present
if PyCompat {
// Initialize alerts list if not present
if config.Alerts == nil {
config.Alerts = map[string]*Alert{}
}
if _, ok := config.Alerts["log"]; !ok {
config.Alerts["log"] = NewLogAlert()
}
}
// Finish initializing configuration // Finish initializing configuration
if err = config.Init(); err != nil { if err = config.Init(); err != nil {
return return

View File

@ -15,6 +15,7 @@ func TestLoadConfig(t *testing.T) {
}{ }{
{"./test/valid-config.yml", false, "Valid config file", false}, {"./test/valid-config.yml", false, "Valid config file", false},
{"./test/valid-config-default-values.yml", false, "Valid config file with default values", false}, {"./test/valid-config-default-values.yml", false, "Valid config file with default values", false},
{"./test/valid-default-log-alert.yml", false, "Valid config file with default log alert PyCompat", true},
{"./test/valid-default-log-alert.yml", true, "Invalid config file no log alert", false}, {"./test/valid-default-log-alert.yml", true, "Invalid config file no log alert", false},
{"./test/does-not-exist", true, "Invalid config path", false}, {"./test/does-not-exist", true, "Invalid config path", false},
{"./test/invalid-config-type.yml", true, "Invalid config type for key", false}, {"./test/invalid-config-type.yml", true, "Invalid config type for key", false},
@ -24,6 +25,8 @@ func TestLoadConfig(t *testing.T) {
for _, c := range cases { for _, c := range cases {
log.Printf("Testing case %s", c.name) log.Printf("Testing case %s", c.name)
// Set PyCompat based on compatibility mode
PyCompat = c.pyCompat
_, err := LoadConfig(c.configPath) _, err := LoadConfig(c.configPath)
hasErr := (err != nil) hasErr := (err != nil)
@ -31,6 +34,9 @@ func TestLoadConfig(t *testing.T) {
t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err) t.Errorf("LoadConfig(%v), expected_error=%v actual=%v", c.name, c.expectErr, err)
log.Printf("Case failed: %s", c.name) log.Printf("Case failed: %s", c.name)
} }
// Set PyCompat to default value
PyCompat = false
} }
} }
@ -47,15 +53,15 @@ func TestIntervalParsing(t *testing.T) {
oneMinute := time.Minute oneMinute := time.Minute
// validate top level interval seconds represented as an int // validate top level interval seconds represented as an int
if config.CheckInterval != oneSecond { if config.CheckInterval.Value() != oneSecond {
t.Errorf("Incorrectly parsed int seconds. expected=%v actual=%v", oneSecond, config.CheckInterval) t.Errorf("Incorrectly parsed int seconds. expected=%v actual=%v", oneSecond, config.CheckInterval)
} }
if config.Monitors[0].CheckInterval != tenSeconds { if config.Monitors[0].CheckInterval.Value() != tenSeconds {
t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval) t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
} }
if config.Monitors[1].CheckInterval != oneMinute { if config.Monitors[1].CheckInterval.Value() != oneMinute {
t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval) t.Errorf("Incorrectly parsed seconds duration. expected=%v actual=%v", oneSecond, config.CheckInterval)
} }

14
go.mod
View File

@ -1,10 +1,10 @@
module git.iamthefij.com/iamthefij/minitor-go module git.iamthefij.com/iamthefij/minitor-go
go 1.17 go 1.20
require ( require (
git.iamthefij.com/iamthefij/slog v1.3.0 git.iamthefij.com/iamthefij/slog v1.3.0
github.com/prometheus/client_golang v1.15.0 github.com/prometheus/client_golang v1.19.0
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
) )
@ -14,10 +14,10 @@ require (
github.com/golang/protobuf v1.5.3 // indirect github.com/golang/protobuf v1.5.3 // indirect
github.com/kr/text v0.2.0 // indirect github.com/kr/text v0.2.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect
golang.org/x/sys v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect google.golang.org/protobuf v1.33.0 // indirect
) )

14
go.sum
View File

@ -182,12 +182,16 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
@ -195,6 +199,8 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+
github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
@ -203,6 +209,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@ -365,6 +373,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@ -504,6 +514,10 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

44
main.go
View File

@ -4,6 +4,7 @@ import (
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"strings"
"time" "time"
"git.iamthefij.com/iamthefij/slog" "git.iamthefij.com/iamthefij/slog"
@ -17,6 +18,9 @@ var (
// Metrics contains all active metrics // Metrics contains all active metrics
Metrics = NewMetrics() Metrics = NewMetrics()
// PyCompat enables support for legacy Python templates
PyCompat = false
// version of minitor being run // version of minitor being run
version = "dev" version = "dev"
@ -88,12 +92,42 @@ func checkMonitors(config *Config) error {
return nil return nil
} }
func sendStartupAlerts(config *Config, alertNames []string) error {
for _, alertName := range alertNames {
var err error
alert, ok := config.Alerts[alertName]
if !ok {
err = fmt.Errorf("unknown alert %s: %w", alertName, errUnknownAlert)
}
if err == nil {
_, err = alert.Send(AlertNotice{
AlertCount: 0,
FailureCount: 0,
IsUp: true,
LastSuccess: time.Now(),
MonitorName: fmt.Sprintf("First Run Alert Test: %s", alert.Name),
LastCheckOutput: "",
})
}
if err != nil {
return err
}
}
return nil
}
func main() { func main() {
showVersion := flag.Bool("version", false, "Display the version of minitor and exit") showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)") configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)") flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)") flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
flag.BoolVar(&PyCompat, "py-compat", false, "Enables support for legacy Python Minitor config. Will eventually be removed. (default: false)")
flag.IntVar(&MetricsPort, "metrics-port", MetricsPort, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)") flag.IntVar(&MetricsPort, "metrics-port", MetricsPort, "The port that Prometheus metrics should be exported on, if enabled. (default: 8080)")
flag.Parse() flag.Parse()
@ -115,11 +149,19 @@ func main() {
go ServeMetrics() go ServeMetrics()
} }
if *startupAlerts != "" {
alertNames := strings.Split(*startupAlerts, ",")
err = sendStartupAlerts(&config, alertNames)
slog.OnErrPanicf(err, "Error running startup alerts")
}
// Start main loop // Start main loop
for { for {
err = checkMonitors(&config) err = checkMonitors(&config)
slog.OnErrPanicf(err, "Error checking monitors") slog.OnErrPanicf(err, "Error checking monitors")
time.Sleep(config.CheckInterval) time.Sleep(config.CheckInterval.Value())
} }
} }

View File

@ -134,3 +134,64 @@ func TestCheckMonitors(t *testing.T) {
} }
} }
} }
func TestFirstRunAlerts(t *testing.T) {
cases := []struct {
config Config
expectErr bool
startupAlerts []string
name string
}{
{
config: Config{},
expectErr: false,
startupAlerts: []string{},
name: "Empty",
},
{
config: Config{},
expectErr: true,
startupAlerts: []string{"missing"},
name: "Unknown",
},
{
config: Config{
Alerts: map[string]*Alert{
"good": {
Command: CommandOrShell{Command: []string{"true"}},
},
},
},
expectErr: false,
startupAlerts: []string{"good"},
name: "Successful alert",
},
{
config: Config{
Alerts: map[string]*Alert{
"bad": {
Name: "bad",
Command: CommandOrShell{Command: []string{"false"}},
},
},
},
expectErr: true,
startupAlerts: []string{"bad"},
name: "Failed alert",
},
}
for _, c := range cases {
err := c.config.Init()
if err != nil {
t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err)
}
err = sendStartupAlerts(&c.config, c.startupAlerts)
if err == nil && c.expectErr {
t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name)
} else if err != nil && !c.expectErr {
t.Errorf("sendFirstRunAlerts(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
}
}
}

View File

@ -1,25 +0,0 @@
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
-
image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7

View File

@ -11,9 +11,9 @@ import (
// Monitor represents a particular periodic check of a command // Monitor represents a particular periodic check of a command
type Monitor struct { //nolint:maligned type Monitor struct { //nolint:maligned
// Config values // Config values
AlertAfter int16 `yaml:"alert_after"` AlertAfter int16 `yaml:"alert_after"`
AlertEvery *int16 `yaml:"alert_every"` AlertEvery *int16 `yaml:"alert_every"`
CheckInterval time.Duration `yaml:"check_interval"` CheckInterval SecondsOrDuration `yaml:"check_interval"`
Name string Name string
AlertDown []string `yaml:"alert_down"` AlertDown []string `yaml:"alert_down"`
AlertUp []string `yaml:"alert_up"` AlertUp []string `yaml:"alert_up"`
@ -45,7 +45,7 @@ func (monitor Monitor) ShouldCheck() bool {
sinceLastCheck := time.Since(monitor.lastCheck) sinceLastCheck := time.Since(monitor.lastCheck)
return sinceLastCheck >= monitor.CheckInterval return sinceLastCheck >= monitor.CheckInterval.Value()
} }
// Check will run the command configured by the Monitor and return a status // Check will run the command configured by the Monitor and return a status

View File

@ -45,9 +45,9 @@ func TestMonitorShouldCheck(t *testing.T) {
name string name string
}{ }{
{Monitor{}, true, "Empty"}, {Monitor{}, true, "Empty"},
{Monitor{lastCheck: timeNow, CheckInterval: time.Second * 15}, false, "Just checked"}, {Monitor{lastCheck: timeNow, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "Just checked"},
{Monitor{lastCheck: timeTenSecAgo, CheckInterval: time.Second * 15}, false, "-10s"}, {Monitor{lastCheck: timeTenSecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, false, "-10s"},
{Monitor{lastCheck: timeTwentySecAgo, CheckInterval: time.Second * 15}, true, "-20s"}, {Monitor{lastCheck: timeTwentySecAgo, CheckInterval: SecondsOrDuration{time.Second * 15}}, true, "-20s"},
} }
for _, c := range cases { for _, c := range cases {

View File

@ -1,5 +1,5 @@
--- ---
check_interval: 1s check_interval: 1
monitors: monitors:
- name: Command - name: Command